Skip to content

Commit 03797e9

Browse files
jschauljschaul
authored
Forward grpc traffic to federator via ingress (or nginz for local integration tests) (#1386)
* integration tests now also deploy an ingress with a self-signed TLS certificate * changes to allow federation traffic to reach the federator: * locally: nginz and its config to forward grpc traffic to federator (generated certificates locally, open an extra port on nginz, and use nginz as the "ingress" for local tests) * on real environments: nginx-ingress-services changes to terminate TLS & forward grpc traffic from `federator.<domain>` to federator * for integration tests: there is a `federation-test-helper` service which adds an SRV dns record in kubernetes for integration test discovery which redirects traffic to the ingress. * drive-by fixes to nginz config * example grpcurl usage bash script under ./hack/federation * example grpc python client under ./hack/federation * side-effect: you can no longer disable TLS at ingress level.
1 parent 9f14855 commit 03797e9

File tree

28 files changed

+879
-24
lines changed

28 files changed

+879
-24
lines changed

‎Makefile

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ DOCKER_TAG ?= $(USER)
88
# default helm chart version must be 0.0.42 for local development (because 42 is the answer to the universe and everything)
99
HELM_SEMVER ?= 0.0.42
1010
# The list of helm charts needed for integration tests on kubernetes
11-
CHARTS_INTEGRATION := wire-server databases-ephemeral fake-aws
11+
CHARTS_INTEGRATION := wire-server databases-ephemeral fake-aws nginx-ingress-controller nginx-ingress-services
1212
# The list of helm charts to publish on S3
1313
# FUTUREWORK: after we "inline local subcharts",
1414
# (e.g. move charts/brig to charts/wire-server/brig)
@@ -273,8 +273,8 @@ kube-restart-%:
273273
kubectl delete pod -n $(NAMESPACE) -l wireService=$(*)
274274
kubectl delete pod -n $(NAMESPACE)-fed2 -l wireService=$(*)
275275

276-
.PHONY: latest-brig-tag
277-
latest-brig-tag:
276+
.PHONY: latest-tag
277+
latest-tag:
278278
./hack/bin/find-latest-docker-tag.sh
279279

280280
.PHONY: release-chart-%

‎charts/federator/templates/service.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,11 @@ metadata:
1010
spec:
1111
type: ClusterIP
1212
ports:
13-
- name: http
13+
- name: internal
1414
port: {{ .Values.service.internalFederatorPort }}
1515
targetPort: {{ .Values.service.internalFederatorPort }}
1616

17-
- name: wire-server-federator
17+
- name: federator-ext
1818
port: {{ .Values.service.externalFederatorPort }}
1919
targetPort: {{ .Values.service.externalFederatorPort }}
2020
selector:

‎charts/nginx-ingress-services/templates/federation-test-helper.yaml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
# Assumes that the controller is deployed in the same namespace. Only used for
2+
# enabling discovery by creating SRV records while running integration tests.
3+
{{- if (and .Values.federator.enabled .Values.federator.integrationTestHelper) }}
4+
apiVersion: v1
5+
kind: Service
6+
metadata:
7+
name: federation-test-helper
8+
namespace: {{ .Release.namespace }}
9+
spec:
10+
ports:
11+
- name: wire-server-federator
12+
port: 443
13+
protocol: TCP
14+
targetPort: https
15+
selector:
16+
app: nginx-ingress
17+
component: controller
18+
type: ClusterIP
19+
{{- end }}

‎charts/nginx-ingress-services/templates/ingress.yaml

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,8 @@ metadata:
55
annotations:
66
kubernetes.io/ingress.class: "nginx"
77
spec:
8-
# This assumes you have created the given cert
8+
# This assumes you have created the given cert (see secret.yaml)
99
# https://github.com/kubernetes/ingress-nginx/blob/master/docs/examples/PREREQUISITES.md#tls-certificates
10-
{{- if .Values.tls.enabled }}
1110
tls:
1211
- hosts:
1312
- {{ .Values.config.dns.https }}
@@ -21,7 +20,6 @@ spec:
2120
- {{ .Values.config.dns.accountPages }}
2221
{{- end }}
2322
secretName: {{ include "nginx-ingress-services.getCertificateSecretName" . | quote }}
24-
{{- end }}
2523
rules:
2624
- host: {{ .Values.config.dns.https }}
2725
http:

‎charts/nginx-ingress-services/templates/ingress_federator.yaml

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
{{- if .Values.federator.enabled }}
2+
# We use a separate ingress for federator/grpc since we can't forward
3+
# both normal http1 traffic and grpc traffic in the same kubernetes ingress it appears.
4+
# Setting backend-protocol annotation to "GRPC" for everything is likely incorrect.
5+
# see also example https://github.com/kubernetes/ingress-nginx/blob/master/docs/examples/grpc/ingress.yaml
6+
# and docs https://kubernetes.github.io/ingress-nginx/examples/grpc/
7+
apiVersion: extensions/v1beta1
8+
kind: Ingress
9+
metadata:
10+
name: federator-ingress
11+
annotations:
12+
kubernetes.io/ingress.class: "nginx"
13+
nginx.ingress.kubernetes.io/ssl-redirect: "true"
14+
nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
15+
spec:
16+
tls:
17+
- hosts:
18+
- {{ .Values.config.dns.federator }}
19+
secretName: {{ include "nginx-ingress-services.getCertificateSecretName" . | quote }}
20+
rules:
21+
- host: {{ .Values.config.dns.federator }}
22+
http:
23+
paths:
24+
- backend:
25+
serviceName: federator
26+
servicePort: federator-ext # name must be below 15 chars
27+
{{- end }}

‎charts/nginx-ingress-services/templates/secret.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,13 @@ metadata:
77
release: "{{ .Release.Name }}"
88
heritage: "{{ .Release.Service }}"
99
type: kubernetes.io/tls
10-
{{ if and .Values.tls.enabled .Values.tls.useCertManager -}}
11-
{{- /* NOTE: providing `data` (and empty strings) allows to manage this secret resource with Helm if cert-manager is user */ -}}
10+
{{ if .Values.tls.useCertManager -}}
11+
{{- /* NOTE: providing `data` (and empty strings) allows to manage this secret resource with Helm if cert-manager is used */ -}}
1212
data:
1313
tls.crt: ""
1414
tls.key: ""
1515
{{- end -}}
16-
{{- if and .Values.tls.enabled (not .Values.tls.useCertManager) -}}
16+
{{- if (not .Values.tls.useCertManager) -}}
1717
data:
1818
{{- /* for_helm_linting is necessary only since the 'with' block below does not throw an error upon an empty .Values.secrets */}}
1919
for_helm_linting: {{ required "No .secrets found in configuration. Did you forget to helm <command> -f path/to/secrets.yaml ?" .Values.secrets | quote | b64enc | quote }}

‎charts/nginx-ingress-services/values.yaml

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ teamSettings:
66
# Account pages may be useful to enable password reset or email validation done after the initial registration
77
accountPages:
88
enabled: false
9-
9+
federator:
10+
enabled: false
11+
integrationTestHelper: false
1012
# If you want to use TLS termination on the ingress,
1113
# then set this variable to true and ensure that there
1214
# is a valid wildcard TLS certificate
@@ -59,6 +61,8 @@ service:
5961
# ssl: nginz-ssl.<domain>
6062
# webapp: webapp.<domain>
6163
# fakeS3: assets.<domain>
64+
# federator: federator.<domain>
65+
# ^ federator is ignored unless federator.enabled == true
6266
# teamSettings: teams.<domain>
6367
# ^ teamSettings is ignored unless teamSettings.enabled == true
6468
# accountPages: account.<domain>

‎deploy/services-demo/conf/nginz/integration-ca-key.pem

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
-----BEGIN RSA PRIVATE KEY-----
2+
MIIEpQIBAAKCAQEApwf/2d2YraQDpCipPVtYR+7BNu47AgkD7kFvGhoxJhDP7CsU
3+
VdpqU5gsVVo8kvhkh4k1tsJyuWWeKn6piNSXxUCFIc80KkUPgsYf5v+RBXr73Fdg
4+
ezHQNhNi0dRZCh+YG/hN7pOX46+B0PyKwUEMTeUqizkmFU5tILPMMyDAGx1Bp2LB
5+
oJi4u+48fzTDMaWSXnCVF04G9+A4LDzw0fPdDMgKLEiXJ8GPoPs0cNs6MJoFDgpe
6+
gzy1mv7X7otmRVTaafZGd4TTo6lGC2VVSS5tpj4Qfz/PxyCLK7tf5033HNWEJzAw
7+
6izRXp849VferHuYEbP+2lexNk9tl45BsFhkrwIDAQABAoIBAQCFkzYeSsJginuG
8+
+iVttfEBhYPqo9V4qTEFhjqNS0jmwiclHMZkagkB1P4PO9yZRB9Q7H+SKiqI7STx
9+
ot19WVYOHqzY/tUewJ/I2xyEJPkawuFLsmyr2IhD1nj+iKy0FdQU+huIoWukX6SX
10+
Nn7YUWa/nHbLY+Z6v38x2deBQ72dcBtDcOh1vtUR3fVfsiX5uzCcfvNZAw4cCyB2
11+
j8ySDIiP10Ic81da3FIeCm8g2yp3DrnvTa77xsr0IfSykB3UcSrGqDwZxs9pS82Q
12+
1fog//4xAfBYC9LEcnQrCvz2kqLSLICtjkgK+dlzgvY3rZMq9c/OY1nR7Wp2BIyp
13+
kKB5AEnRAoGBANTM3fq4YGzUodf+Xla4MDvQFJsYjQuig/CJboQ7JSFZi2uLnSHX
14+
+7JDiHtQd3uifYMhzSxXXKV82CK7SsJOQlIVoCZ5eTsyYGyAu1fUqfBvfHYN4Gbr
15+
3QyZJE0Hut2rvn5DaT/dpgh7Uy9QWKhpAsmxzhKa/iADUTiNAO8pxxRFAoGBAMjw
16+
iZV43XWLvzP90P5jANHuk9tR/B5cM9zK40aWglNsMlK9cUgW3ovohMzTFce/LQWy
17+
zGZ1WZZcUUcR/pHot3fyjWKeJadZhSZ/7hN/0d/UDuFY5nQ8eGQoy2qrrtY+6MMU
18+
Eiz09EFnKKA7hUoDnbhOH1hCKsfrOVse55RDkTZjAoGABrzRzm1mCCwXT7prDD3a
19+
sRoefOajGJo1qTkAuckRnOOz6VzLRdYLzxIaUSU0E0MKzEsWru+5LDgus7LQZCSM
20+
LwMmRfGUqA4pRWYyCE7gbo9pFmfMEhYnso1qu9Gh1gDpECBcRbxj1GLrOFVH6VUh
21+
1Hb/ulET+LmCKdM1E110Qy0CgYEAimbDHSUGxHPg2pq0XMMsSWyegq3RjcfMIQPN
22+
z0zTr0oSz1KUuCaoWo1pCvtJQS+4fvhMOTYS4rHreZw3T6CO3hs+rvJm1QGf6Iit
23+
HtknYZfaN/TXprAP7Ez87xgZcJAcGmG0syp1Iqc/ID5e7D/ZXpzQkiXg+ZpXAyAi
24+
OcjgOCkCgYEAmsCsqtPn5vgB+/vr0n28UsFS4Of9whlgEPYndNss3nAmVEohQJRg
25+
QlBlJd2iDa7R0TrJZCuAwuqK7TxB/RoHL8UkryUt2nag39GYAyE+lfPM558/AWyt
26+
9yyLQNfiJnqTC2Ne2j7EyicBLha4J9NoBeNE5UqLlzrH4LRJ3fRX9Ps=
27+
-----END RSA PRIVATE KEY-----

‎deploy/services-demo/conf/nginz/integration-ca.pem

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
-----BEGIN CERTIFICATE-----
2+
MIIDAjCCAeqgAwIBAgIUJ+1IpVKexlyGhgZaOZi2Ka01Q2QwDQYJKoZIhvcNAQEL
3+
BQAwGTEXMBUGA1UEAxMOY2EuZXhhbXBsZS5jb20wHhcNMjEwMzA5MjA0MDAwWhcN
4+
MjYwMzA4MjA0MDAwWjAZMRcwFQYDVQQDEw5jYS5leGFtcGxlLmNvbTCCASIwDQYJ
5+
KoZIhvcNAQEBBQADggEPADCCAQoCggEBAKcH/9ndmK2kA6QoqT1bWEfuwTbuOwIJ
6+
A+5BbxoaMSYQz+wrFFXaalOYLFVaPJL4ZIeJNbbCcrllnip+qYjUl8VAhSHPNCpF
7+
D4LGH+b/kQV6+9xXYHsx0DYTYtHUWQofmBv4Te6Tl+OvgdD8isFBDE3lKos5JhVO
8+
bSCzzDMgwBsdQadiwaCYuLvuPH80wzGlkl5wlRdOBvfgOCw88NHz3QzICixIlyfB
9+
j6D7NHDbOjCaBQ4KXoM8tZr+1+6LZkVU2mn2RneE06OpRgtlVUkubaY+EH8/z8cg
10+
iyu7X+dN9xzVhCcwMOos0V6fOPVX3qx7mBGz/tpXsTZPbZeOQbBYZK8CAwEAAaNC
11+
MEAwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFFLJ
12+
Dc8PpKsUGFi33sK+r5Xgi97XMA0GCSqGSIb3DQEBCwUAA4IBAQBtiav0b3r4k3CC
13+
N1DRNNpcvpHR8odPJkxR2r0kCLRqwu2YDxQYxCK5wCPNcpzGDd6nyg/nLWBzl4Vd
14+
UwBIUPSVavDAeQ8VL0YNCNhQlL4x6z5rT1aSRdp0VlRnnl4zjilWSN/IB5Y61i0Q
15+
Ww7Sd5T9hZUCOJm5bAeLo4+cxkOwN6jdT2wmLtgFkj4CFVvS/8nL8ZwC1qvt3mLs
16+
E8q7/KelKgddU5AET8Viim20m6ZxgGNWIX33LBiJ3Rg3QuJMnWyfBI6PCORRrdTb
17+
0AH/F3iUKQL8Zv6twr++S7VPeDllVgEXkq457ZBi2qJPikL5cUprV/0bWQxgf4+1
18+
L70TaNMy
19+
-----END CERTIFICATE-----

‎deploy/services-demo/conf/nginz/integration-leaf-key.pem

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
-----BEGIN RSA PRIVATE KEY-----
2+
MIIEpAIBAAKCAQEA7AcV1TSTJRGMOM+qB2F4wWbsV+M2C/4KWt0LKC6k3EeqYo+R
3+
n9utlcH42S3UgzjB9zVMXYItwtGSCNVbxQ4s/nCNsyZR2JB689gnR1hetnKCFJTY
4+
2ETBsqFgoyNHU1ZkoTIq0ZQVeOEkcn2qnM981xuPvptY1ChdFtD52ybJRhAcgrHX
5+
+P3aWns0N7Hn7EgFWinGDCZ27QXA/iiuSteVr/3Fh39RF7+YYix4Y7TRqXQqRhK/
6+
sp7fZQjLeXB/oECh5tEm/WEFJoX4mDNf96GAVzHynjNysCSZd44iZxbMlgNynYv0
7+
1Q/LLddI1KBz4MqqtBvHgh61ddtcM6vi15iK8QIDAQABAoIBAGMj9r+/fa8sV8Hl
8+
OH4xKw/Rs6SXA7S12JOiFXWjYgxE7gTIWVrUY21sNKLE5WbZCGBWOuVsMNyMOtML
9+
C6kR80RM8rg1eFuHFJ4oRRdqvx3Tq2AJxavR9aqaroTX5LXUrCApemLd2McVisGs
10+
l2WWodGY+iAOEJnM+o0C6nBrMlAC1fHQGDaIeGCdrRWS1RO47z+wMxnd9n0Lmek1
11+
YHg+Cylg16ZbZvcl8KfJ3CzD8QGpu4hF1BMzJwzYg2qmLUPrlhB1bJomW8asbZtu
12+
MyKsr5zl95f7lOtn3DwsxNOYHlQFqYs8u4rogOqb09yIRoYm1n4v6Z1DKdQ1cm/U
13+
yIPcp4ECgYEA+P7ZPF412TLlHo337ryF10SvuWJJCYZLrVaeiTecgbqbbqtUcR9c
14+
NnTzd8LTBMAtl6lcyTzrTi5+RGrGiicKunZiUnWl8g67E7EMUsM/tWrZah3yOPDa
15+
yTNtXtpGkBqa4+sujky3WqCDicBpATeeE9xdIP3KS+OjKUR1gKs1dgUCgYEA8qrZ
16+
LyWGeI07C0ql3I6iFlhwbSMfQ+8mx8yYj+5rCEiY3Ns/D7WbxvqXOgBVD+QK26a9
17+
4f20z6HTXBnPAmYoMlG/uh+MJj2VlgsgElRy4fi33ky+F67NWcniaO5gU0iO5P0m
18+
W6yKBuEg34NAuBZ85Q/HzZoCXGijaugA0H+DyP0CgYEAtSuRWthdq/TZOxsnTCEF
19+
7XT/uw6lf/WkOLJu1f6NtOLXV3X4Efs0eDcuM3ShX7KJOfG0HoRh3df8bcGXRkxU
20+
BQpATilmHjLTsec/xTRltZXSLeEuLnopZv4xVTPS5eVF0BJ+JHHzc1CZhPBoNseG
21+
EINli6y9tewi+tLiAEYStxUCgYBj46Hs/1RgETqpxjuKE4hzDEqTEi8PhsT+36A5
22+
NxoF2eRanUFTFR2fhY10iah6FxPFINjuysF/6owXUGRfB3Aqbm4ujkfhgd4uWjwE
23+
b5CupfQB5LMnCoRrHmg9a0ppXee8KNx+RgK3HDqs5tlgLZmRrLJBVtD7+vKx0fhr
24+
2uGDfQKBgQCVu/KGMBrS1M/05nrg6MS8vrDO0Wwyk3X9/mmGh651omQeW39IvFq+
25+
5rI0w46f6mslksFYqCJhfxXIKHN0sB3cBw/290j6FY3+TxpD2sJLIOeTllr+ivvA
26+
4BBs0XcecrPF7Ykw+E9hOHTo+dBRoTz1yUPHJYsTN5vI8los9XIyBg==
27+
-----END RSA PRIVATE KEY-----

0 commit comments

Comments
 (0)