Bootstrap flag for connection limit overload manager resource monitor

apis/projectcontour/v1alpha1/contourconfig.go

@@ -302,6 +302,16 @@ type EnvoyConfig struct {
 	// Network holds various configurable Envoy network values.
 	// +optional
 	Network *NetworkParameters `json:"network,omitempty"`
+
+	// OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+	// the envoy overload manager actions, such as global connection limits, enforced.
+	//
+	// The configured values must be different from the endpoints
+	// configured by [EnvoyConfig.Metrics] and [EnvoyConfig.Health]
+	//
+	// This is disabled by default
+	// +optional
+	OMEnforcedHealth *HealthConfig `json:"omEnforcedHealth,omitempty"`
 }
 
 // DebugConfig contains Contour specific troubleshooting options.

apis/projectcontour/v1alpha1/contourconfig_helpers.go

@@ -112,6 +112,10 @@ func (e *EnvoyConfig) Validate() error {
 		return fmt.Errorf("invalid envoy configuration: %v", err)
 	}
 
+	if err := healthEndpointsInConflict(e.Metrics, e.Health, e.OMEnforcedHealth); err != nil {
+		return fmt.Errorf("invalid envoy configuration: %v", err)
+	}
+
 	if err := e.Logging.Validate(); err != nil {
 		return err
 	}
@@ -330,3 +334,17 @@ func endpointsInConfict(health *HealthConfig, metrics *MetricsConfig) error {
 	}
 	return nil
 }
+
+// healthEndpointsInConflict returns an error if the same address and port are used between the overload manager enforced health listener and metrics
+// _or_ the stats listener. Since the metrics would be validated against the stats listener already we only need to check that the overload manager
+// health listener is not in conflict with either metrics or stats
+func healthEndpointsInConflict(metrics *MetricsConfig, health, omEnforcedHealth *HealthConfig) error {
+	switch {
+	case omEnforcedHealth != nil && health != nil && omEnforcedHealth.Address == health.Address && omEnforcedHealth.Port == health.Port:
+		return fmt.Errorf("cannot use the same port for health checks and overload-manager enforced health checks")
+	case omEnforcedHealth != nil && metrics != nil && omEnforcedHealth.Address == metrics.Address && omEnforcedHealth.Port == metrics.Port:
+		return fmt.Errorf("cannot use the same port for metrics and overload-manager enforced health checks")
+	default:
+		return nil
+	}
+}

apis/projectcontour/v1alpha1/contourdeployment.go

@@ -245,6 +245,15 @@ type EnvoySettings struct {
 	//
 	// +optional
 	OverloadMaxHeapSize uint64 `json:"overloadMaxHeapSize,omitempty"`
+
+	// OverloadMaxDownstreamConn defines the envoy global downstream connection limit controlled by the overload manager.
+	// When the value is greater than 0 the overload manager is enabled and listeners
+	// will begin rejecting connections when the the connection threshold is hit.
+	// Metrics and health listeners are not subject to the connection limits, however,
+	// they still count against the global limit.
+	//
+	// +optional
+	OverloadMaxDownstreamConnections uint64 `json:"overloadMaxDownstreamConnections,omitempty"`
 }
 
 // WorkloadType is the type of Kubernetes workload to use for a component.

changelogs/unreleased/6794-seth-epps-minor.md

@@ -0,0 +1,52 @@
+## Overload Manager - Max Global Connections
+
+Introduces an envoy bootstrap flag to enable the [global downstream connection limit overload manager resource monitors](https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/resource_monitors/downstream_connections/v3/downstream_connections.proto#envoy-v3-api-msg-extensions-resource-monitors-downstream-connections-v3-downstreamconnectionsconfig).
+
+The new flag can be passed as an integer flag to the contour bootstrap subcommand, `overload-downstream-max-conn`.
+
+```sh
+contour bootstrap --help
+INFO[0000] maxprocs: Leaving GOMAXPROCS=10: CPU quota undefined
+usage: contour bootstrap [<flags>] <path>
+
+Generate bootstrap configuration.
+
+
+Flags:
+  -h, --[no-]help                Show context-sensitive help (also try --help-long and --help-man).
+      --log-format=text          Log output format for Contour. Either text or json.
+      --admin-address="/admin/admin.sock"
+                                 Path to Envoy admin unix domain socket.
+      --admin-port=ADMIN-PORT    DEPRECATED: Envoy admin interface port.
+      --dns-lookup-family=DNS-LOOKUP-FAMILY
+                                 Defines what DNS Resolution Policy to use for Envoy -> Contour cluster name lookup. Either v4, v6, auto, or all.
+      --envoy-cafile=ENVOY-CAFILE
+                                 CA Filename for Envoy secure xDS gRPC communication. ($ENVOY_CAFILE)
+      --envoy-cert-file=ENVOY-CERT-FILE
+                                 Client certificate filename for Envoy secure xDS gRPC communication. ($ENVOY_CERT_FILE)
+      --envoy-key-file=ENVOY-KEY-FILE
+                                 Client key filename for Envoy secure xDS gRPC communication. ($ENVOY_KEY_FILE)
+      --namespace="projectcontour"
+                                 The namespace the Envoy container will run in. ($CONTOUR_NAMESPACE)
+      --overload-downstream-max-conn=OVERLOAD-DOWNSTREAM-MAX-CONN
+                                 Defines the Envoy global downstream connection limit
+      --overload-max-heap=OVERLOAD-MAX-HEAP
+                                 Defines the maximum heap size in bytes until overload manager stops accepting new connections.
+      --resources-dir=RESOURCES-DIR
+                                 Directory where configuration files will be written to.
+      --xds-address=XDS-ADDRESS  xDS gRPC API address.
+      --xds-port=XDS-PORT        xDS gRPC API port.
+      --xds-resource-version="v3"
+                                 The versions of the xDS resources to request from Contour.
+
+Args:
+  <path>  Configuration file ('-' for standard output).
+```
+
+As part of this change, we also set the `ignore_global_conn_limit` flag to `true` on the existing admin listeners such
+that envoy remains live, ready, and serving stats even though it is rejecting downstream connections.
+
+To add some flexibility for health checks, in addition to adding a new bootstrap flag, there is a new configuration
+option for the envoy health config to enforce the envoy overload manager actions, namely rejecting requests. This
+"advanced" configuration gives the operator the ability to configure readiness and liveness to handle taking pods out
+of the pool of pods that can serve traffic.

cmd/contour/bootstrap.go

@@ -34,6 +34,7 @@ func registerBootstrap(app *kingpin.Application) (*kingpin.CmdClause, *envoy.Boo
 	bootstrap.Flag("envoy-cert-file", "Client certificate filename for Envoy secure xDS gRPC communication.").Envar("ENVOY_CERT_FILE").StringVar(&config.GrpcClientCert)
 	bootstrap.Flag("envoy-key-file", "Client key filename for Envoy secure xDS gRPC communication.").Envar("ENVOY_KEY_FILE").StringVar(&config.GrpcClientKey)
 	bootstrap.Flag("namespace", "The namespace the Envoy container will run in.").Envar("CONTOUR_NAMESPACE").Default("projectcontour").StringVar(&config.Namespace)
+	bootstrap.Flag("overload-downstream-max-conn", "Defines the Envoy global downstream connection limit.").Int64Var(&config.GlobalDownstreamConnectionLimit)
 	bootstrap.Flag("overload-max-heap", "Defines the maximum heap size in bytes until overload manager stops accepting new connections.").Uint64Var(&config.MaximumHeapSizeBytes)
 	bootstrap.Flag("resources-dir", "Directory where configuration files will be written to.").StringVar(&config.ResourcesDir)
 	bootstrap.Flag("xds-address", "xDS gRPC API address.").StringVar(&config.XDSAddress)

cmd/contour/contour.go

@@ -104,6 +104,9 @@
 		if err := envoy.ValidAdminAddress(bootstrapCtx.AdminAddress); err != nil {
 			log.WithField("flag", "--admin-address").WithError(err).Fatal("failed to parse bootstrap args")
 		}
+		if err := envoy.ValidConnectionLimit(bootstrapCtx.GlobalDownstreamConnectionLimit); err != nil {
+			log.WithField("flag", "--overload-downstream-max-conn").WithError(err).Fatal("failed to parse bootstrap args")
+		}
 		envoyGen := envoy_v3.NewEnvoyGen(envoy_v3.EnvoyGenOpt{
 			XDSClusterName: envoy_v3.DefaultXDSClusterName,
 		})

cmd/contour/serve.go

@@ -501,7 +501,7 @@
 	})
 
 	resources := []xdscache.ResourceCache{
-		xdscache_v3.NewListenerCache(listenerConfig, *contourConfiguration.Envoy.Metrics, *contourConfiguration.Envoy.Health, *contourConfiguration.Envoy.Network.EnvoyAdminPort, envoyGen),
+		xdscache_v3.NewListenerCache(listenerConfig, *contourConfiguration.Envoy.Metrics, *contourConfiguration.Envoy.Health, contourConfiguration.Envoy.OMEnforcedHealth, *contourConfiguration.Envoy.Network.EnvoyAdminPort, envoyGen),
 		xdscache_v3.NewSecretsCache(envoy_v3.StatsSecrets(contourConfiguration.Envoy.Metrics.TLS)),
 		&xdscache_v3.RouteCache{},
 		xdscache_v3.NewClusterCache(envoyGen),

cmd/contour/servecontext.go

@@ -515,6 +515,14 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_v1alpha1.Co
 		Port:    ctx.statsPort,
 	}
 
+	var envoyOMEnforcedHealthListenerConfig *contour_v1alpha1.HealthConfig
+	if ctx.Config.OMEnforcedHealthListener != nil {
+		envoyOMEnforcedHealthListenerConfig = &contour_v1alpha1.HealthConfig{
+			Address: ctx.Config.OMEnforcedHealthListener.Address,
+			Port:    ctx.Config.OMEnforcedHealthListener.Port,
+		}
+	}
+
 	// Override metrics endpoint info from config files
 	//
 	// Note!
@@ -603,6 +611,7 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_v1alpha1.Co
 				EnvoyAdminPort:            &ctx.Config.Network.EnvoyAdminPort,
 				EnvoyStripTrailingHostDot: &ctx.Config.Network.EnvoyStripTrailingHostDot,
 			},
+			OMEnforcedHealth: envoyOMEnforcedHealthListenerConfig,
 		},
 		Gateway: gatewayConfig,
 		HTTPProxy: &contour_v1alpha1.HTTPProxyConfig{

cmd/contour/servecontext_test.go

@@ -486,6 +486,7 @@ func defaultContourConfiguration() contour_v1alpha1.ContourConfigurationSpec {
 				XffNumTrustedHops:         ptr.To(uint32(0)),
 				EnvoyStripTrailingHostDot: ptr.To(false),
 			},
+			OMEnforcedHealth: nil,
 		},
 		Gateway: nil,
 		HTTPProxy: &contour_v1alpha1.HTTPProxyConfig{
@@ -902,6 +903,22 @@ func TestConvertServeContext(t *testing.T) {
 				return cfg
 			},
 		},
+		"envoy overload manager health checks": {
+			getServeContext: func(ctx *serveContext) *serveContext {
+				ctx.Config.OMEnforcedHealthListener = &config.OMEnforcedHealthListenerConfig{
+					Address: "0.0.0.0",
+					Port:    8005,
+				}
+				return ctx
+			},
+			getContourConfiguration: func(cfg contour_v1alpha1.ContourConfigurationSpec) contour_v1alpha1.ContourConfigurationSpec {
+				cfg.Envoy.OMEnforcedHealth = &contour_v1alpha1.HealthConfig{
+					Address: "0.0.0.0",
+					Port:    8005,
+				}
+				return cfg
+			},
+		},
 	}
 
 	for name, tc := range cases {

examples/contour/01-contour-config.yaml

@@ -179,3 +179,7 @@ data:
     #  socket-options:
     #    tos: 64
     #    traffic-class: 64
+    #
+    # omEnforcedHealthListener:
+    #   address: 0.0.0.0
+    #   port: 8003

examples/contour/01-crds.yaml

@@ -549,6 +549,22 @@ spec:
                           Contour's default is false.
                         type: boolean
                     type: object
+                  omEnforcedHealth:
+                    description: |-
+                      OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+                      the envoy overload manager actions, such as global connection limits, enforced.
+                      The configured values must be different from the endpoints
+                      configured by [EnvoyConfig.Metrics] and [EnvoyConfig.Health]
+                      This is disabled by default
+                    properties:
+                      address:
+                        description: Defines the health address interface.
+                        minLength: 1
+                        type: string
+                      port:
+                        description: Defines the health port.
+                        type: integer
+                    type: object
                   service:
                     description: |-
                       Service holds Envoy service parameters for setting Ingress status.
@@ -3781,6 +3797,15 @@ spec:
                           type: object
                         type: array
                     type: object
+                  overloadMaxDownstreamConnections:
+                    description: |-
+                      OverloadMaxDownstreamConn defines the envoy global downstream connection limit controlled by the overload manager.
+                      When the value is greater than 0 the overload manager is enabled and listeners
+                      will begin rejecting connections when the the connection threshold is hit.
+                      Metrics and health listeners are not subject to the connection limits, however,
+                      they still count against the global limit.
+                    format: int64
+                    type: integer
                   overloadMaxHeapSize:
                     description: |-
                       OverloadMaxHeapSize defines the maximum heap memory of the envoy controlled by the overload manager.
@@ -4392,6 +4417,22 @@ spec:
                               Contour's default is false.
                             type: boolean
                         type: object
+                      omEnforcedHealth:
+                        description: |-
+                          OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+                          the envoy overload manager actions, such as global connection limits, enforced.
+                          The configured values must be different from the endpoints
+                          configured by [EnvoyConfig.Metrics] and [EnvoyConfig.Health]
+                          This is disabled by default
+                        properties:
+                          address:
+                            description: Defines the health address interface.
+                            minLength: 1
+                            type: string
+                          port:
+                            description: Defines the health port.
+                            type: integer
+                        type: object
                       service:
                         description: |-
                           Service holds Envoy service parameters for setting Ingress status.

examples/render/contour-deployment.yaml

@@ -212,6 +212,10 @@ data:
     #  socket-options:
     #    tos: 64
     #    traffic-class: 64
+    #
+    # omEnforcedHealthListener:
+    #   address: 0.0.0.0
+    #   port: 8003
 
 ---
 apiVersion: apiextensions.k8s.io/v1
@@ -764,6 +768,22 @@ spec:
                           Contour's default is false.
                         type: boolean
                     type: object
+                  omEnforcedHealth:
+                    description: |-
+                      OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+                      the envoy overload manager actions, such as global connection limits, enforced.
+                      The configured values must be different from the endpoints
+                      configured by [EnvoyConfig.Metrics] and [EnvoyConfig.Health]
+                      This is disabled by default
+                    properties:
+                      address:
+                        description: Defines the health address interface.
+                        minLength: 1
+                        type: string
+                      port:
+                        description: Defines the health port.
+                        type: integer
+                    type: object
                   service:
                     description: |-
                       Service holds Envoy service parameters for setting Ingress status.
@@ -3996,6 +4016,15 @@ spec:
                           type: object
                         type: array
                     type: object
+                  overloadMaxDownstreamConnections:
+                    description: |-
+                      OverloadMaxDownstreamConn defines the envoy global downstream connection limit controlled by the overload manager.
+                      When the value is greater than 0 the overload manager is enabled and listeners
+                      will begin rejecting connections when the the connection threshold is hit.
+                      Metrics and health listeners are not subject to the connection limits, however,
+                      they still count against the global limit.
+                    format: int64
+                    type: integer
                   overloadMaxHeapSize:
                     description: |-
                       OverloadMaxHeapSize defines the maximum heap memory of the envoy controlled by the overload manager.
@@ -4607,6 +4636,22 @@ spec:
                               Contour's default is false.
                             type: boolean
                         type: object
+                      omEnforcedHealth:
+                        description: |-
+                          OMEnforcedHealth defines the endpoint Envoy uses to serve health checks with
+                          the envoy overload manager actions, such as global connection limits, enforced.
+                          The configured values must be different from the endpoints
+                          configured by [EnvoyConfig.Metrics] and [EnvoyConfig.Health]
+                          This is disabled by default
+                        properties:
+                          address:
+                            description: Defines the health address interface.
+                            minLength: 1
+                            type: string
+                          port:
+                            description: Defines the health port.
+                            type: integer
+                        type: object
                       service:
                         description: |-
                           Service holds Envoy service parameters for setting Ingress status.