Validate server TLS certificate between federators

.gitignore

@@ -99,4 +99,4 @@ i.yaml
 b.yaml
 telepresence.log
 
-/.ghci
+/.ghci

CHANGELOG.md

@@ -45,6 +45,7 @@ Upgrade nginz (#1658)
 * Extend feature config API (#1658)
 * `fileSharing` feature config (#1652, #1654, #1655)
 * Add user_id to csv export (#1663)
+* Validate server TLS certificate between federators (#1662)
 
 ## Bug fixes and other updates
 

Makefile

@@ -439,6 +439,11 @@ kind-restart-all: .local/kind-kubeconfig
 	kubectl delete pod -n $(NAMESPACE) -l release=$(NAMESPACE)-wire-server && \
 	kubectl delete pod -n $(NAMESPACE)-fed2 -l release=$(NAMESPACE)-fed2-wire-server
 
+kind-restart-nginx-ingress: .local/kind-kubeconfig
+	export KUBECONFIG=$(CURDIR)/.local/kind-kubeconfig && \
+	kubectl delete pod -n $(NAMESPACE) -l app=nginx-ingress && \
+	kubectl delete pod -n $(NAMESPACE)-fed2 -l app=nginx-ingress
+
 kind-restart-%: .local/kind-kubeconfig
 	export KUBECONFIG=$(CURDIR)/.local/kind-kubeconfig && \
 	kubectl delete pod -n $(NAMESPACE) -l wireService=$(*) && \

charts/federator/templates/configmap-ca.yaml

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: "federator-ca"
+  labels:
+    wireService: federator
+    chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+data:
+  # TODO: add validation and fail early during templating: either contents should be provided; or explicitly system trust store enabled
+  {{- if .Values.remoteCAContents }}
+  remote-ca.pem: {{ .Values.remoteCAContents | b64enc | quote }}
+  {{- end }}

charts/federator/templates/configmap.yaml

@@ -3,7 +3,7 @@ kind: ConfigMap
 metadata:
   name: "federator"
   labels:
-    wireService: fedrator
+    wireService: federator
     chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }}
     release: {{ .Release.Name }}
     heritage: {{ .Release.Service }}
@@ -40,12 +40,18 @@ data:
 
     {{- with .optSettings }}
     optSettings:
-      setFederationStrategy:
-        {{- if .setFederationStrategy.allowAll }}
+      # Filepath to one or more PEM-encoded server certificates to use as a trust
+      # store when making grpc requests to remote backends
+      {{- if $.Values.remoteCAContents }}
+      remoteCAStore: "/etc/wire/federator/ca/remote-ca.pem"
+      {{- end }}
+      useSystemCAStore: {{ .useSystemCAStore }}
+      federationStrategy:
+        {{- if .federationStrategy.allowAll }}
         allowAll:
-        {{- else if .setFederationStrategy.allowedDomains }}
+        {{- else if .federationStrategy.allowedDomains }}
         allowedDomains:
-        {{- range $domain := .setFederationStrategy.allowedDomains }}
+        {{- range $domain := .federationStrategy.allowedDomains }}
           - {{ $domain | quote }}
         {{- end }}
         {{- else }}

charts/federator/templates/deployment.yaml

@@ -25,19 +25,27 @@ spec:
       annotations:
         # An annotation of the configmap checksum ensures changes to the configmap cause a redeployment upon `helm upgrade`
         checksum/configmap: {{ include (print .Template.BasePath "/configmap.yaml") . | sha256sum }}
+        checksum/configmap-ca: {{ include (print .Template.BasePath "/configmap-ca.yaml") . | sha256sum }}
         fluentbit.io/parser: json
     spec:
       volumes:
         - name: "federator-config"
           configMap:
             name: "federator"
+        # federator-ca holds CA certificates to use as a trust store
+        # when making requests to remote backends
+        - name: "federator-ca"
+          secret:
+            secretName: "federator-ca"
       containers:
         - name: federator
           image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
           imagePullPolicy: {{ default "" .Values.imagePullPolicy | quote }}
           volumeMounts:
             - name: "federator-config"
               mountPath: "/etc/wire/federator/conf"
+            - name: "federator-ca"
+              mountPath: "/etc/wire/federator/ca"
           ports:
             - name: internal
               containerPort: {{ .Values.service.internalFederatorPort }}

charts/federator/templates/tests/configmap.yaml

@@ -16,3 +16,6 @@ data:
     galley:
       host: galley
       port: 8080
+    nginxIngress:
+      host: federation-test-helper.{{ .Release.Namespace }}.svc.cluster.local
+      port: 443

charts/federator/templates/tests/federator-integration.yaml

@@ -13,6 +13,10 @@ spec:
     - name: "federator-config"
       configMap:
         name: "federator"
+    # integration tests need access to the CA
+    - name: "federator-ca"
+      secret:
+        secretName: "federator-ca"
   containers:
   - name: integration
     command: ["federator-integration"]
@@ -22,4 +26,6 @@ spec:
       mountPath: "/etc/wire/integration"
     - name: "federator-config"
       mountPath: "/etc/wire/federator/conf"
+    - name: "federator-ca"
+      mountPath: "/etc/wire/federator/ca"
   restartPolicy: Never

charts/federator/values.yaml

@@ -20,5 +20,16 @@ config:
   logLevel: Debug
   logFormat: JSON
   optSettings:
-    setFederationStrategy:
+    # Defaults to using system CA store in the federator image for making
+    # connections to remote federators.
+    # A custom CA certificate can be provided by specifying
+    #   federator.remoteCAContents
+    # e.g. from a pem file myca.pem:
+    #   { echo "federator:"; echo "  remoteCAContents: |"; sed -e 's/^/    /' myca.pem; } > myca.yaml
+    # then use '-f myca.yaml' as additional flag to your helm commands.
+    #
+    # Using custom CA doesn't automatically disable system CA store, it should
+    # be disabled explicitly by setting useSystemCAStore to false.
+    useSystemCAStore: true
+    federationStrategy:
       allowedDomains: []

charts/nginx-ingress-services/templates/certificate.yaml

@@ -36,4 +36,7 @@ spec:
     {{- if .Values.accountPages.enabled }}
     - {{ .Values.config.dns.accountPages }}
     {{- end }}
+    {{- if .Values.federator.enabled }}
+    - {{ .Values.config.dns.federator }}
+    {{- end }}
 {{- end -}}

deploy/services-demo/conf/nginz/nginx.conf

@@ -124,6 +124,7 @@ http {
     # This applies only locally, as for kubernetes (helm chart) based deployments,
     # TLS is terminated at the ingress level, not at nginz level
     listen 8443 ssl http2;
+    listen [::]:8443 ssl http2;
 
     # self-signed certificates generated using wire-server/hack/bin/selfsigned.sh
     ssl_certificate integration-leaf.pem;

docs/reference/config-options.md

@@ -153,12 +153,12 @@ optSettings:
 
 ### Federation allow list
 
-As of 2021-02, federation (whatever is implemented by the time you read this) is turned off by default by means of having an empty allow list:
+As of 2021-07, federation (whatever is implemented by the time you read this) is turned off by default by means of having an empty allow list:
 
 ```yaml
 # federator.yaml
 optSettings:
-  setFederationStrategy:
+  federationStrategy:
     allowedDomains: []
 ```
 
@@ -168,7 +168,7 @@ You can choose to federate with a specific list of allowed servers:
 ```yaml
 # federator.yaml
 optSettings:
-  setFederationStrategy:
+  federationStrategy:
     allowedDomains:
       - server1.example.com
       - server2.example.com
@@ -179,14 +179,29 @@ or, you can federate with everyone:
 ```yaml
 # federator.yaml
 optSettings:
-  setFederationStrategy:
+  federationStrategy:
     # note the 'empty' value after 'allowAll'
     allowAll:
 
 # when configuring helm charts, this becomes (note 'true' after 'allowAll')
 # inside helm_vars/wire-server:
 federator:
   optSettings:
-    setFederationStrategy:
+    federationStrategy:
       allowAll: true
 ```
+
+### Federation TLS Config
+
+When a federator connects with another federator, it does so over HTTPS. There
+are two options to configure the CA for this:
+1. `useSystemCAStore`: Boolean. If set to `True` it will use the system CA.
+1. `remoteCAStore`: Maybe Filepath. This config option can be used to specify
+   multiple certificates from either a single file (multiple PEM formatted
+   certificates concatenated) or directory (one certificate per file, file names
+   are hashes from certificate).
+
+Both of these options can be specified, in this case the stores are concatenated
+and used for verifying certificates. When `useSystemCAStore` is `False` and
+`remoteCAStore` is not set, then all outbound connections will fail with TLS
+error as there will be no CA to verify.

hack/bin/buildah-make-images.sh

@@ -9,7 +9,7 @@ EXECUTABLES=${EXECUTABLES:-"cannon brig cargohold galley gundeck federator brig-
 CONTAINER_NAME="output"
 DOCKER_TAG=${DOCKER_TAG:-$USER}
 
-buildah containers | awk '{print $5}' | grep "$CONTAINER_NAME" || \
+buildah containers | awk '{print $5}' | grep "$CONTAINER_NAME" ||
     buildah from --name "$CONTAINER_NAME" -v "${TOP_LEVEL}":/src --pull quay.io/wire/alpine-deps:develop
 
 # Only brig needs these templates, but for simplicity we add them to all resulting images (optimization FUTUREWORK)

hack/bin/integration-setup.sh

@@ -4,22 +4,22 @@ USAGE="Usage: $0"
 
 set -e
 
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 TOP_LEVEL="$DIR/../.."
 CHARTS_DIR="${TOP_LEVEL}/.local/charts"
 
 NAMESPACE=${NAMESPACE:-test-integration}
 ENABLE_KIND_VALUES=${ENABLE_KIND_VALUES:-0}
 
-kubectl create namespace "${NAMESPACE}" > /dev/null 2>&1 || true
+kubectl create namespace "${NAMESPACE}" >/dev/null 2>&1 || true
 
 ${DIR}/integration-cleanup.sh
 
-charts=( fake-aws databases-ephemeral wire-server nginx-ingress-controller nginx-ingress-services )
+charts=(fake-aws databases-ephemeral wire-server nginx-ingress-controller nginx-ingress-services)
 
 echo "updating recursive dependencies ..."
 for chart in "${charts[@]}"; do
-  "$DIR/update.sh" "$CHARTS_DIR/$chart"
+    "$DIR/update.sh" "$CHARTS_DIR/$chart"
 done
 
 echo "Installing charts..."
@@ -37,19 +37,26 @@ function printLogs() {
 
 trap printLogs ERR
 
-FEDERATION_DOMAIN="federation-test-helper.$NAMESPACE.svc.cluster.local"
+export FEDERATION_DOMAIN_BASE="$NAMESPACE.svc.cluster.local"
+FEDERATION_DOMAIN="federation-test-helper.$FEDERATION_DOMAIN_BASE"
+"$DIR/selfsigned-kubernetes.sh"
 
 for chart in "${charts[@]}"; do
     kubectl -n ${NAMESPACE} get pods
     valuesfile="${DIR}/../helm_vars/${chart}/values.yaml"
     kindValuesfile="${DIR}/../helm_vars/${chart}/kind-values.yaml"
+    certificatesValuesfile="${DIR}/../helm_vars/${chart}/certificates.yaml"
 
     declare -a options=()
 
     if [ -f "$valuesfile" ]; then
         options+=(-f "$valuesfile")
     fi
 
+    if [ -f "$certificatesValuesfile" ]; then
+        options+=(-f "$certificatesValuesfile")
+    fi
+
     if [[ "$chart" == "nginx-ingress-services" ]]; then
         # Federation domain is also the SRV record created by the
         # federation-test-helper service. Maybe we can find a way to make these
@@ -78,7 +85,10 @@ resourcesReady() {
     SNS_POD=$(kubectl -n "${NAMESPACE}" get pods | grep fake-aws-sns | grep Running | awk '{print $1}')
     kubectl -n "${NAMESPACE}" logs "$SNS_POD" -c initiate-fake-aws-sns | grep created
 }
-until resourcesReady; do echo 'waiting for SNS resources'; sleep 1; done
+until resourcesReady; do
+    echo 'waiting for SNS resources'
+    sleep 1
+done
 
 kubectl -n ${NAMESPACE} get pods
 

hack/bin/selfsigned-kubernetes.sh

@@ -0,0 +1,84 @@
+#!/usr/bin/env bash
+
+# Create a self-signed x509 certificate in the hack/helm_vars directories (as helm yaml config).
+# Requires 'cfssl' to be on your PATH (see https://github.com/cloudflare/cfssl)
+# These certificates are only meant for integration tests.
+# (The CA certificates are assumed to be re-used across the domains A and B for end2end integration tests.)
+
+set -ex
+TEMP=${TEMP:-/tmp}
+CSR="$TEMP/csr.json"
+OUTPUTNAME_CA="integration-ca"
+OUTPUTNAME_LEAF_CERT="integration-leaf"
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TOP_LEVEL="$DIR/../.."
+OUTPUT_CONFIG_FEDERATOR="$TOP_LEVEL/hack/helm_vars/wire-server/certificates.yaml"
+OUTPUT_CONFIG_INGRESS="$TOP_LEVEL/hack/helm_vars/nginx-ingress-services/certificates.yaml"
+
+command -v cfssl >/dev/null 2>&1 || {
+    echo >&2 "cfssl is not installed, aborting. See https://github.com/cloudflare/cfssl"
+    exit 1
+}
+command -v cfssljson >/dev/null 2>&1 || {
+    echo >&2 "cfssljson is not installed, aborting. See https://github.com/cloudflare/cfssl"
+    exit 1
+}
+
+FEDERATION_DOMAIN_BASE=${FEDERATION_DOMAIN_BASE:?"you must provide a FEDERATION_DOMAIN_BASE env variable"}
+
+# generate CA key and cert
+if [ ! -f "$OUTPUTNAME_CA.pem" ]; then
+    echo "CA file not found, generating CA..."
+    echo '{
+    "CN": "ca.example.com",
+        "key": {
+            "algo": "rsa",
+            "size": 2048
+        }
+    }' >"$CSR"
+    cfssl gencert -initca "$CSR" | cfssljson -bare "$OUTPUTNAME_CA"
+    rm "$OUTPUTNAME_CA.csr"
+else
+    echo "Re-using previous CA"
+fi
+
+# For federation end2end tests, only the
+# 'federation-test-helper.$FEDERATION_DOMAIN_BASE' is necessary for
+# ingress->federator traffic. For other potential traffic in the integration
+# tests of the future, we use a wildcard certificate here.
+echo '{
+    "key": {
+        "algo": "rsa",
+        "size": 2048
+    }
+}' >"$CSR"
+# generate cert and key based on CA given comma-separated hostnames as SANs
+cfssl gencert -ca "$OUTPUTNAME_CA.pem" -ca-key "$OUTPUTNAME_CA-key.pem" -hostname="*.$FEDERATION_DOMAIN_BASE" "$CSR" | cfssljson -bare "$OUTPUTNAME_LEAF_CERT"
+
+# the following yaml override file is needed as an override to
+# nginx-ingress-services helm chart
+# for domain A, ingress@A needs cert+key for A
+{
+    echo "secrets:"
+    echo "  tlsWildcardCert: |"
+    sed -e 's/^/    /' $OUTPUTNAME_LEAF_CERT.pem
+    echo "  tlsWildcardKey: |"
+    sed -e 's/^/    /' $OUTPUTNAME_LEAF_CERT-key.pem
+} | tee "$OUTPUT_CONFIG_INGRESS"
+
+# the following yaml override file is needed as an override to
+# the wire-server (federator) helm chart
+# e.g. for installing on domain A, federator@A needs the CA for B
+# As a "shortcut" for integration tests, we re-use the same CA for both domains
+# A and B.
+{
+    echo "federator:"
+    echo "  remoteCAContents: |"
+    sed -e 's/^/    /' $OUTPUTNAME_CA.pem
+} | tee "$OUTPUT_CONFIG_FEDERATOR"
+
+# cleanup unneeded files
+rm "$OUTPUTNAME_LEAF_CERT.csr"
+rm "$OUTPUTNAME_LEAF_CERT.pem"
+rm "$OUTPUTNAME_LEAF_CERT-key.pem"
+rm "$CSR"

hack/helm_vars/.gitignore

@@ -0,0 +1 @@
+certificates.yaml