replace nginz disco with native upstream resolver WPB-15302

changelog.d/5-internal/nginz-disco-replacement

@@ -0,0 +1 @@
+charts/nginz: remove nginz_disco script and sidecar container, and replace outside upstreams.txt file by an inline block, making use of 'resolve' keyword to directly reference DNS names inside the kubernetes cluster.

charts/nginz/README.md

@@ -1,6 +1,6 @@
 # nginz chart
 
-Deploys [nginz - nginx with authentication module](https://github.com/wireapp/wire-server/services/nginz), along with a little sidecar container
+Deploys [nginz - nginx with authentication module](https://github.com/wireapp/wire-server/services/nginz).
 
 ## Configuring zauth
 
@@ -18,7 +18,3 @@ This only needs to be done when you wish to bypass normal authentication for som
 * `htpasswd -cb myfile.txt myuser mypassword && cat myfile.txt` (`htpasswd` is from `httpd-tools` or `apache-utils`) generates a hashed user:password line which you can pass to the nginz chart under `nginz.secrets.basicAuth` (see also wire-server-deploy/values/wire-server/secrets.yaml )
 * generate the base64 value of the original user:password (*not* of the myfile contents): `echo '<user>:<password>' | base64`
 * deploy and try a request by passing a header `Authorization: Basic <base64-encoded-value>`
-
-## Sidecar container nginz-disco
-
-Due to nginx not supporting DNS names for its list of upstream servers (unless you pay extra), the [nginz-disco](https://github.com/wireapp/wire-server/tree/develop/tools/nginz_disco) container is a simple bash script to do DNS lookups and write the resulting IPs to a file. Nginz reloads on changes to this file.

charts/nginz/templates/conf/_nginx.conf.tpl

@@ -189,8 +189,22 @@ http {
   #
   #  Proxied Upstream Services
   #
+{{- $clusterDomain := .Values.nginx_conf.cluster_domain }}
 
-  include {{ .Values.nginx_conf.upstream_config }};
+  resolver kube-dns.kube-system.svc.{{ $clusterDomain }} valid=30s ipv6=off;
+  resolver_timeout 5s;
+
+{{- $validUpstreams := include "valid_upstreams" . | fromJson }}
+{{- range $name, $_ := $validUpstreams }}
+  upstream {{ $name }} {
+      zone {{ $name }} 64k; # needed for dynamic DNS updates
+      least_conn;
+      keepalive 32;
+
+      {{- $ns := index $.Values.nginx_conf.upstream_namespace $name | default $.Release.Namespace }}
+      server {{ $name }}.{{ $ns }}.svc.{{ $clusterDomain }}:8080 max_fails=3 resolve;
+  }
+{{ end }}
 
   #
   # Mapping for websocket connections

charts/nginz/templates/configmap.yaml

@@ -18,7 +18,6 @@ metadata:
 data:
   nginx.conf: |2
     {{- include "nginz_nginx.conf" . | indent 4 }}
-  upstreams.txt: "{{- include "nginz_upstreams.txt" . | trim }}"
 
 {{ (.Files.Glob "static/conf/*").AsConfig | indent 2 }}
 ---

charts/nginz/templates/deployment.yaml

@@ -37,19 +37,6 @@ spec:
             matchLabels:
               app: nginz
       containers:
-      - name: nginz-disco
-        image: "{{ .Values.images.nginzDisco.repository }}:{{ .Values.images.nginzDisco.tag }}"
-        {{- if eq (include "includeSecurityContext" .) "true" }}
-        securityContext:
-          {{- toYaml .Values.podSecurityContext | nindent 10 }}
-        {{- end }}
-        volumeMounts:
-        - name: config
-          mountPath: /etc/wire/nginz/conf
-          readOnly: true
-        - name: upstreams
-          mountPath: /etc/wire/nginz/upstreams
-          readOnly: false
       - name: nginz
         image: "{{ .Values.images.nginz.repository }}:{{ .Values.images.nginz.tag }}"
         {{- if eq (include "includeSecurityContext" .) "true" }}
@@ -63,9 +50,6 @@ spec:
         - name: config
           mountPath: /etc/wire/nginz/conf
           readOnly: true
-        - name: upstreams
-          mountPath: /etc/wire/nginz/upstreams
-          readOnly: true
         - name: deeplink
           mountPath: /etc/wire/nginz/deeplink
           readOnly: true
@@ -88,6 +72,12 @@ spec:
             path: /status
             port: {{ .Values.config.http.httpPort }}
             scheme: HTTP
+        lifecycle:
+          preStop:
+            exec:
+              # kubernetes by default sends a SIGTERM to the container,
+              # we can be more graceful with in-flight requests using quit
+              command: ["sh", "-c", "nginx -c /etc/wire/nginz/conf/nginx.conf -s quit && sleep 25"]
         resources:
 {{ toYaml .Values.resources | indent 12 }}
       dnsPolicy: ClusterFirst
@@ -99,8 +89,6 @@ spec:
       - name: secrets
         secret:
           secretName: nginz
-      - name: upstreams
-        emptyDir: {}
       - name: deeplink
         configMap:
           name: nginz-deeplink

charts/nginz/values.yaml

@@ -9,9 +9,6 @@ metrics:
   serviceMonitor:
     enabled: false
 images:
-  nginzDisco:
-    repository: quay.io/wire/nginz_disco
-    tag: do-not-use
   nginz:
     repository: quay.io/wire/nginz
     tag: do-not-use
@@ -22,9 +19,9 @@ config:
   ws:
     wsPort: 8081
     useProxyProtocol: true
-terminationGracePeriodSeconds: 30
+terminationGracePeriodSeconds: 90
 nginx_conf:
-  upstream_config: /etc/wire/nginz/upstreams/upstreams.conf
+  cluster_domain: cluster.local
   zauth_keystore: /etc/wire/nginz/secrets/zauth.conf
   zauth_acl: /etc/wire/nginz/conf/zauth.acl
   basic_auth_file: /etc/wire/nginz/secrets/basic_auth.txt
@@ -124,8 +121,7 @@ nginx_conf:
   ignored_upstreams: []
 
   # If an upstream runs in a different namespace than nginz, its namespace must
-  # be specified here otherwise nginz_disco will fail to find the upstream and
-  # nginx will think that the upstream is down.
+  # be specified here otherwise nginz will think that the upstream is down.
   upstream_namespace: {
     # galeb: integrations
   }

hack/bin/kind-upload-images.sh

@@ -27,7 +27,5 @@ nix -v --show-trace -L build -f "$ROOT_DIR/nix" wireServer.imagesList -o "$image
 
 xargs -I {} -P 10 "$SCRIPT_DIR/kind-upload-image.sh" "wireServer.$IMAGES_ATTR.{}" < "$image_list_file"
 
-for image_name in nginz nginz-disco; do
-    printf '*** Unploading image %s\n' "$image_name"
-    "$SCRIPT_DIR/kind-upload-image.sh" "$image_name"
-done
+printf '*** Unploading image %s\n' nginz
+"$SCRIPT_DIR/kind-upload-image.sh" nginz

hack/bin/upload-images.sh

@@ -30,7 +30,5 @@ nix -v --show-trace -L build -f "$ROOT_DIR/nix" "wireServer.$IMAGES_ATTR" --no-l
 
 xargs -I {} -P 10 "$SCRIPT_DIR/upload-image.sh" "wireServer.$IMAGES_ATTR.{}" < "$image_list_file"
 
-for image_name in nginz nginz-disco; do
-    printf '*** Uploading image %s\n' "$image_name"
-    "$SCRIPT_DIR/upload-image.sh" "$image_name"
-done
+printf '*** Uploading image %s\n' nginz
+"$SCRIPT_DIR/upload-image.sh" nginz

integration/default.nix

@@ -45,6 +45,7 @@
 , HsOpenSSL
 , http-client
 , http-types
+, interpolate
 , kan-extensions
 , lens
 , lens-aeson
@@ -149,6 +150,7 @@ mkDerivation {
     HsOpenSSL
     http-client
     http-types
+    interpolate
     kan-extensions
     lens
     lens-aeson

integration/integration.cabal

@@ -259,6 +259,7 @@ library
     , HsOpenSSL
     , http-client
     , http-types
+    , interpolate
     , kan-extensions
     , lens
     , lens-aeson