feat: customize comet rpc and p2p ports (#485)

* feat: customize comet rpc and p2p ports

* config set p2p addr

* add test

* unnecessary cast

* fix test

Author: agouin

api/v1/cosmosfullnode_types.go

@@ -17,6 +17,9 @@ limitations under the License.
 package v1
 
 import (
+	"strconv"
+	"strings"
+
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -596,6 +599,14 @@ type ChainVersion struct {
 
 // CometConfig configures the config.toml.
 type CometConfig struct {
+	// RPC listen address. Defaults to tcp://0.0.0.0:26657
+	// +optional
+	RPCListenAddress string `json:"rpcListenAddress"`
+
+	// P2P listen address. Defaults to tcp://0.0.0.0:26656
+	// +optional
+	P2PListenAddress string `json:"p2pListenAddress"`
+
 	// Comma delimited list of p2p nodes in <ID>@<IP>:<PORT> format to keep persistent p2p connections.
 	// +kubebuilder:validation:MinLength:=1
 	// +optional
@@ -642,6 +653,39 @@ type CometConfig struct {
 	TomlOverrides *string `json:"overrides"`
 }
 
+const (
+	defaultRPCPort = 26657
+	defaultP2PPort = 26656
+)
+
+func (c CometConfig) RPCPort() int32 {
+	if c.RPCListenAddress == "" {
+		return defaultRPCPort
+	}
+	portSplit := strings.Split(c.RPCListenAddress, ":")
+
+	port, err := strconv.ParseInt(portSplit[len(portSplit)-1], 10, 32)
+	if err != nil {
+		return defaultRPCPort
+	}
+
+	return int32(port)
+}
+
+func (c CometConfig) P2PPort() int32 {
+	if c.P2PListenAddress == "" {
+		return defaultP2PPort
+	}
+	portSplit := strings.Split(c.P2PListenAddress, ":")
+
+	port, err := strconv.ParseInt(portSplit[len(portSplit)-1], 10, 32)
+	if err != nil {
+		return defaultP2PPort
+	}
+
+	return int32(port)
+}
+
 // SDKAppConfig configures the cosmos sdk application app.toml.
 type SDKAppConfig struct {
 	// The minimum gas prices a validator is willing to accept for processing a

config/crd/bases/cosmos.strange.love_cosmosfullnodes.yaml

@@ -222,6 +222,9 @@ spec:
                           WARNING: Overriding may clobber some values that the operator sets such as persistent_peers, private_peer_ids,
                           and unconditional_peer_ids. Use the dedicated fields for these values which will merge values.
                         type: string
+                      p2pListenAddress:
+                        description: P2P listen address. Defaults to tcp://0.0.0.0:26656
+                        type: string
                       peers:
                         description: Comma delimited list of p2p nodes in <ID>@<IP>:<PORT>
                           format to keep persistent p2p connections.
@@ -231,6 +234,9 @@ spec:
                         description: Comma delimited list of node/peer IDs to keep
                           private (will not be gossiped to other peers)
                         type: string
+                      rpcListenAddress:
+                        description: RPC listen address. Defaults to tcp://0.0.0.0:26657
+                        type: string
                       seeds:
                         description: Comma delimited list of p2p seed nodes in <ID>@<IP>:<PORT>
                           format.

internal/cosmos/status_collector.go

@@ -47,7 +47,19 @@ func (coll StatusCollector) Collect(ctx context.Context, pods []corev1.Pod) Stat
 				statuses[i].Err = errors.New("pod has no IP")
 				return nil
 			}
-			host := fmt.Sprintf("http://%s:26657", ip)
+			var rpcPort int32 = 26657
+			for _, c := range pod.Spec.Containers {
+				if c.Name == "node" {
+					for _, p := range c.Ports {
+						if p.Name == "rpc" {
+							rpcPort = p.ContainerPort
+							break
+						}
+					}
+					break
+				}
+			}
+			host := fmt.Sprintf("http://%s:%d", ip, rpcPort)
 			cctx, cancel := context.WithTimeout(ctx, coll.timeout)
 			defer cancel()
 			resp, err := coll.comet.Status(cctx, host)

internal/fullnode/configmap_builder.go

@@ -192,15 +192,28 @@ func addConfigToml(buf *bytes.Buffer, cmData map[string]string, crd *cosmosv1.Co
 		}
 	}
 
+	if comet.P2PListenAddress != "" {
+		p2p["laddr"] = comet.P2PListenAddress
+	}
+
 	base["p2p"] = p2p
 
+	var rpcLaddr = "tcp://0.0.0.0:26657"
+	if comet.RPCListenAddress != "" {
+		rpcLaddr = comet.RPCListenAddress
+	}
+
+	rpc := decodedToml{
+		"laddr": rpcLaddr,
+	}
+
 	if v := comet.CorsAllowedOrigins; v != nil {
-		base["rpc"] = decodedToml{
-			"cors_allowed_origins": v,
-			"cors-allowed-origins": v,
-		}
+		rpc["cors_allowed_origins"] = v
+		rpc["cors-allowed-origins"] = v
 	}
 
+	base["rpc"] = rpc
+
 	dst := defaultComet()
 
 	mergemap.Merge(dst, base)

internal/fullnode/peer_collector.go

@@ -16,6 +16,7 @@ import (
 
 // Peer contains information about a peer.
 type Peer struct {
+	P2PPort         int32
 	NodeID          string
 	PrivateAddress  string // Only the private address my-service.namespace.svc.cluster.local:<port>
 	ExternalAddress string // Only the address <external-ip-or-hostname>:<port>. Not all peers will be external.
@@ -31,7 +32,7 @@ func (peer Peer) PrivatePeer() string {
 // ExternalPeer returns the full external address of the peer in the format <node_id>@<external_address>:<port>.
 func (peer Peer) ExternalPeer() string {
 	if peer.ExternalAddress == "" {
-		return peer.NodeID + "@" + net.JoinHostPort("0.0.0.0", strconv.Itoa(p2pPort))
+		return peer.NodeID + "@" + net.JoinHostPort("0.0.0.0", strconv.Itoa(int(peer.P2PPort)))
 	}
 	return peer.NodeID + "@" + peer.ExternalAddress
 }
@@ -119,7 +120,9 @@ func (c PeerCollector) Collect(ctx context.Context, crd *cosmosv1.CosmosFullNode
 		}
 
 		svcName := p2pServiceName(crd, i)
+		p2pPort := crd.Spec.ChainSpec.Comet.P2PPort()
 		peers[c.objectKey(crd, i)] = Peer{
+			P2PPort:        p2pPort,
 			NodeID:         nodeKey.NodeKey.ID(),
 			PrivateAddress: fmt.Sprintf("%s.%s.svc.%s:%d", svcName, crd.Namespace, clusterDomain, p2pPort),
 		}
@@ -159,7 +162,7 @@ func (c PeerCollector) addExternalAddress(ctx context.Context, peers Peers, crd
 	lb := ingress[0]
 	host := lo.Ternary(lb.IP != "", lb.IP, lb.Hostname)
 	if host != "" {
-		info.ExternalAddress = net.JoinHostPort(host, strconv.Itoa(p2pPort))
+		info.ExternalAddress = net.JoinHostPort(host, strconv.FormatInt(int64(crd.Spec.ChainSpec.Comet.P2PPort()), 10))
 	}
 	return nil
 }

internal/fullnode/pod_builder.go

@@ -84,7 +84,7 @@ func NewPodBuilder(crd *cosmosv1.CosmosFullNode) PodBuilder {
 					Command:         []string{startCmd},
 					Args:            startArgs,
 					Env:             envVars(crd),
-					Ports:           buildPorts(crd.Spec.Type),
+					Ports:           buildPorts(crd),
 					Resources:       tpl.Resources,
 					ReadinessProbe:  probes[0],
 					ImagePullPolicy: tpl.ImagePullPolicy,
@@ -96,7 +96,7 @@ func NewPodBuilder(crd *cosmosv1.CosmosFullNode) PodBuilder {
 					// Available images: https://github.com/orgs/strangelove-ventures/packages?repo_name=cosmos-operator
 					// IMPORTANT: Must use v0.6.2 or later.
 					Image:   "ghcr.io/strangelove-ventures/cosmos-operator:" + version.DockerTag(),
-					Command: []string{"/manager", "healthcheck"},
+					Command: []string{"/manager", "healthcheck", "--rpc-host", fmt.Sprintf("http://localhost:%d", crd.Spec.ChainSpec.Comet.RPCPort())},
 					Ports:   []corev1.ContainerPort{{ContainerPort: healthCheckPort, Protocol: corev1.ProtocolTCP}},
 					Resources: corev1.ResourceRequirements{
 						Requests: corev1.ResourceList{
@@ -148,7 +148,7 @@ func podReadinessProbes(crd *cosmosv1.CosmosFullNode) []*corev1.Probe {
 		ProbeHandler: corev1.ProbeHandler{
 			HTTPGet: &corev1.HTTPGetAction{
 				Path:   "/health",
-				Port:   intstr.FromInt(rpcPort),
+				Port:   intstr.FromInt(int(crd.Spec.ChainSpec.Comet.RPCPort())),
 				Scheme: corev1.URISchemeHTTP,
 			},
 		},

internal/fullnode/pod_builder_test.go

@@ -147,6 +147,36 @@ func TestPodBuilder(t *testing.T) {
 		}
 	})
 
+	t.Run("override ports", func(t *testing.T) {
+		crd := defaultCRD()
+		crd.Spec.ChainSpec.Comet.RPCListenAddress = "tcp://0.0.0.0:27147"
+		crd.Spec.ChainSpec.Comet.P2PListenAddress = "tcp://0.0.0.0:27146"
+		pod, err := NewPodBuilder(&crd).Build()
+		require.NoError(t, err)
+		ports := pod.Spec.Containers[0].Ports
+
+		require.Equal(t, 7, len(ports))
+
+		for i, tt := range []struct {
+			Name string
+			Port int32
+		}{
+			{"api", 1317},
+			{"rosetta", 8080},
+			{"grpc", 9090},
+			{"prometheus", 26660},
+			{"p2p", 27146},
+			{"rpc", 27147},
+			{"grpc-web", 9091},
+		} {
+			port := ports[i]
+			require.Equal(t, tt.Name, port.Name, tt)
+			require.Equal(t, corev1.ProtocolTCP, port.Protocol)
+			require.Equal(t, tt.Port, port.ContainerPort)
+			require.Zero(t, port.HostPort)
+		}
+	})
+
 	t.Run("ports - sentry", func(t *testing.T) {
 		crd := defaultCRD()
 		crd.Spec.Type = cosmosv1.Sentry
@@ -269,7 +299,7 @@ func TestPodBuilder(t *testing.T) {
 		healthContainer := pod.Spec.Containers[1]
 		require.Equal(t, "healthcheck", healthContainer.Name)
 		require.Equal(t, "ghcr.io/strangelove-ventures/cosmos-operator:latest", healthContainer.Image)
-		require.Equal(t, []string{"/manager", "healthcheck"}, healthContainer.Command)
+		require.Equal(t, []string{"/manager", "healthcheck", "--rpc-host", "http://localhost:26657"}, healthContainer.Command)
 		require.Empty(t, healthContainer.Args)
 		require.Empty(t, healthContainer.ImagePullPolicy)
 		require.NotEmpty(t, healthContainer.Resources)

internal/fullnode/ports.go

@@ -9,60 +9,62 @@ const (
 	apiPort     = 1317
 	grpcPort    = 9090
 	grpcWebPort = 9091
-	p2pPort     = 26656
 	privvalPort = 1234
 	promPort    = 26660
 	rosettaPort = 8080
+	p2pPort     = 26656
 	rpcPort     = 26657
 )
 
-func buildPorts(nodeType cosmosv1.FullNodeType) []corev1.ContainerPort {
-	switch nodeType {
+func buildPorts(crd *cosmosv1.CosmosFullNode) []corev1.ContainerPort {
+	switch crd.Spec.Type {
 	case cosmosv1.Sentry:
-		return append(defaultPorts[:], corev1.ContainerPort{
+		return append(ports(crd), corev1.ContainerPort{
 			Name:          "privval",
 			ContainerPort: privvalPort,
 			Protocol:      corev1.ProtocolTCP,
 		})
 	default:
-		return defaultPorts[:]
+		return ports(crd)
 	}
 }
 
-var defaultPorts = [...]corev1.ContainerPort{
-	{
-		Name:          "api",
-		Protocol:      corev1.ProtocolTCP,
-		ContainerPort: apiPort,
-	},
-	{
-		Name:          "rosetta",
-		Protocol:      corev1.ProtocolTCP,
-		ContainerPort: rosettaPort,
-	},
-	{
-		Name:          "grpc",
-		Protocol:      corev1.ProtocolTCP,
-		ContainerPort: grpcPort,
-	},
-	{
-		Name:          "prometheus",
-		Protocol:      corev1.ProtocolTCP,
-		ContainerPort: promPort,
-	},
-	{
-		Name:          "p2p",
-		Protocol:      corev1.ProtocolTCP,
-		ContainerPort: p2pPort,
-	},
-	{
-		Name:          "rpc",
-		Protocol:      corev1.ProtocolTCP,
-		ContainerPort: rpcPort,
-	},
-	{
-		Name:          "grpc-web",
-		Protocol:      corev1.ProtocolTCP,
-		ContainerPort: grpcWebPort,
-	},
+func ports(crd *cosmosv1.CosmosFullNode) []corev1.ContainerPort {
+	return []corev1.ContainerPort{
+		{
+			Name:          "api",
+			Protocol:      corev1.ProtocolTCP,
+			ContainerPort: apiPort,
+		},
+		{
+			Name:          "rosetta",
+			Protocol:      corev1.ProtocolTCP,
+			ContainerPort: rosettaPort,
+		},
+		{
+			Name:          "grpc",
+			Protocol:      corev1.ProtocolTCP,
+			ContainerPort: grpcPort,
+		},
+		{
+			Name:          "prometheus",
+			Protocol:      corev1.ProtocolTCP,
+			ContainerPort: promPort,
+		},
+		{
+			Name:          "p2p",
+			Protocol:      corev1.ProtocolTCP,
+			ContainerPort: crd.Spec.ChainSpec.Comet.P2PPort(),
+		},
+		{
+			Name:          "rpc",
+			Protocol:      corev1.ProtocolTCP,
+			ContainerPort: crd.Spec.ChainSpec.Comet.RPCPort(),
+		},
+		{
+			Name:          "grpc-web",
+			Protocol:      corev1.ProtocolTCP,
+			ContainerPort: grpcWebPort,
+		},
+	}
 }