Skip to content

Commit ce2bbff

Browse files
makkesmakkes
committed
use HTTPS in gitops run S3 bucket server
When installing the dev bucket server on the cluster, it is now configured with a self-signed certificate generated on demand. The certificate is valid for 3 days which should be enough for a typical `gitops run` use case. Instead of using HTTP in the forwarding from the host machine into the bucket server pod, the CLI now uses HTTPS so that traffic can't be intercepted on its way between the developer's machine and the cluster. The certificate, after being generated, is put into a Kubernetes Secret which is in turn mounted into the bucket server pod.
1 parent c5ad717 commit ce2bbff

File tree

7 files changed

+248
-53
lines changed

7 files changed

+248
-53
lines changed

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ GIT_COMMIT?=$(shell which git > /dev/null && git log -n1 --pretty='%h')
99
VERSION?=$(shell which git > /dev/null && git describe --always --match "v*")
1010
FLUX_VERSION=0.37.0
1111
CHART_VERSION=$(shell which yq > /dev/null && yq e '.version' charts/gitops-server/Chart.yaml)
12-
DEV_BUCKET_CONTAINER_IMAGE=ghcr.io/weaveworks/gitops-bucket-server@sha256:8fbb7534e772e14ea598d287a4b54a3f556416cac6621095ce45f78346fda78a
12+
DEV_BUCKET_CONTAINER_IMAGE?=ghcr.io/weaveworks/gitops-bucket-server@sha256:157fa617e893e3ab0239547d8f1e820664b10c849fbd652c7f8738920b842f13
1313
TIER=oss
1414

1515
# Go build args
@@ -31,7 +31,7 @@ LDFLAGS?=-X github.com/weaveworks/weave-gitops/cmd/gitops/version.Branch=$(BRANC
3131
# Docker args
3232
# LDFLAGS is passed so we don't have to copy the entire .git directory into the image
3333
# just to get, e.g. the commit hash
34-
DOCKERARGS:=--build-arg FLUX_VERSION=$(FLUX_VERSION) --build-arg LDFLAGS="$(LDFLAGS)" --build-arg GIT_COMMIT=$(GIT_COMMIT)
34+
DOCKERARGS+=--build-arg FLUX_VERSION=$(FLUX_VERSION) --build-arg LDFLAGS="$(LDFLAGS)" --build-arg GIT_COMMIT=$(GIT_COMMIT)
3535
# We want to be able to reference this in builds & pushes
3636
DEFAULT_DOCKER_REPO=localhost:5001
3737
DOCKER_REGISTRY?=$(DEFAULT_DOCKER_REPO)

‎cmd/gitops/beta/run/cmd.go‎

Lines changed: 15 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,6 @@ import (
2222
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
2323
"github.com/fsnotify/fsnotify"
2424
"github.com/manifoldco/promptui"
25-
"github.com/minio/minio-go/v7"
26-
"github.com/minio/minio-go/v7/pkg/credentials"
2725
"github.com/spf13/cobra"
2826
"github.com/weaveworks/weave-gitops/cmd/gitops/cmderrors"
2927
"github.com/weaveworks/weave-gitops/cmd/gitops/config"
@@ -35,6 +33,7 @@ import (
3533
"github.com/weaveworks/weave-gitops/pkg/run/bootstrap"
3634
"github.com/weaveworks/weave-gitops/pkg/run/install"
3735
"github.com/weaveworks/weave-gitops/pkg/run/watch"
36+
"github.com/weaveworks/weave-gitops/pkg/s3"
3837
"github.com/weaveworks/weave-gitops/pkg/validate"
3938
"github.com/weaveworks/weave-gitops/pkg/version"
4039
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -615,21 +614,22 @@ func runCommandWithoutSession(cmd *cobra.Command, args []string) error {
615614

616615
// ====================== Dev-bucket ======================
617616
// Install dev-bucket server before everything, so that we can also forward logs to it
618-
unusedPorts, err := run.GetUnusedPorts(1)
617+
unusedPorts, err := run.GetUnusedPorts(2)
619618
if err != nil {
620619
cancel()
621620
return err
622621
}
623622

624-
devBucketPort := unusedPorts[0]
625-
cancelDevBucketPortForwarding, err := watch.InstallDevBucketServer(ctx, log0, kubeClient, cfg, devBucketPort)
623+
devBucketHTTPPort := unusedPorts[0]
624+
devBucketHTTPSPort := unusedPorts[1]
626625

626+
cancelDevBucketPortForwarding, cert, err := watch.InstallDevBucketServer(ctx, log0, kubeClient, cfg, devBucketHTTPPort, devBucketHTTPSPort)
627627
if err != nil {
628628
cancel()
629-
return err
629+
return fmt.Errorf("unable to install S3 bucket server: %w", err)
630630
}
631631

632-
log, err := logger.NewS3LogWriter(sessionName, fmt.Sprintf("localhost:%d", devBucketPort), log0)
632+
log, err := logger.NewS3LogWriter(sessionName, fmt.Sprintf("localhost:%d", devBucketHTTPSPort), cert, log0)
633633
if err != nil {
634634
cancel()
635635
return err
@@ -666,7 +666,7 @@ func runCommandWithoutSession(cmd *cobra.Command, args []string) error {
666666
Namespace: flags.Namespace,
667667
Path: paths.TargetDir,
668668
Timeout: flags.Timeout,
669-
DevBucketPort: devBucketPort,
669+
DevBucketPort: devBucketHTTPPort,
670670
SessionName: sessionName,
671671
Username: username,
672672
}
@@ -683,16 +683,7 @@ func runCommandWithoutSession(cmd *cobra.Command, args []string) error {
683683
}
684684
}
685685

686-
ignorer := watch.CreateIgnorer(paths.RootDir)
687-
minioClient, err := minio.New(
688-
"localhost:"+strconv.Itoa(int(devBucketPort)),
689-
&minio.Options{
690-
Creds: credentials.NewStaticV4("user", "doesn't matter", ""),
691-
Secure: false,
692-
BucketLookup: minio.BucketLookupPath,
693-
},
694-
)
695-
686+
minioClient, err := s3.NewMinioClient("localhost:"+strconv.Itoa(int(devBucketHTTPSPort)), cert)
696687
if err != nil {
697688
cancel()
698689
return err
@@ -705,6 +696,8 @@ func runCommandWithoutSession(cmd *cobra.Command, args []string) error {
705696
return err
706697
}
707698

699+
ignorer := watch.CreateIgnorer(paths.RootDir)
700+
708701
err = filepath.Walk(paths.RootDir, watch.WatchDirsForFileWalker(watcher, ignorer))
709702
if err != nil {
710703
cancel()
@@ -1043,12 +1036,10 @@ func runBootstrap(ctx context.Context, log logger.Logger, paths *run.Paths, mani
10431036

10441037
workloadKustomizationContentStr := string(workloadKustomizationContent)
10451038

1046-
commitFiles := []gitprovider.CommitFile{
1047-
gitprovider.CommitFile{
1048-
Path: &workloadKustomizationPath,
1049-
Content: &workloadKustomizationContentStr,
1050-
},
1051-
}
1039+
commitFiles := []gitprovider.CommitFile{{
1040+
Path: &workloadKustomizationPath,
1041+
Content: &workloadKustomizationContentStr,
1042+
}}
10521043

10531044
if len(manifests) > 0 {
10541045
strManifests := string(manifests)

‎pkg/logger/s3_log_writer.go‎

Lines changed: 3 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88

99
"github.com/go-logr/logr"
1010
"github.com/minio/minio-go/v7"
11-
"github.com/minio/minio-go/v7/pkg/credentials"
11+
"github.com/weaveworks/weave-gitops/pkg/s3"
1212
)
1313

1414
type S3LogWriter struct {
@@ -23,16 +23,8 @@ func (l *S3LogWriter) L() logr.Logger {
2323
return l.log0.L()
2424
}
2525

26-
func NewS3LogWriter(id string, endpoint string, log0 Logger) (Logger, error) {
27-
minioClient, err := minio.New(
28-
endpoint,
29-
&minio.Options{
30-
Creds: credentials.NewStaticV4("user", "doesn't matter", ""),
31-
Secure: false,
32-
BucketLookup: minio.BucketLookupPath,
33-
},
34-
)
35-
26+
func NewS3LogWriter(id, endpoint string, caCert []byte, log0 Logger) (Logger, error) {
27+
minioClient, err := s3.NewMinioClient(endpoint, caCert)
3628
if err != nil {
3729
return nil, err
3830
}

‎pkg/run/watch/install_dev_bucket_server.go‎

Lines changed: 67 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88

99
"github.com/weaveworks/weave-gitops/pkg/logger"
1010
"github.com/weaveworks/weave-gitops/pkg/run"
11+
"github.com/weaveworks/weave-gitops/pkg/tls"
1112
appsv1 "k8s.io/api/apps/v1"
1213
corev1 "k8s.io/api/core/v1"
1314
apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -29,11 +30,11 @@ var (
2930
// The variables below are to be set by flags passed to `go build`.
3031
// Examples: -X run.DevBucketContainerImage=xxxxx
3132

32-
DevBucketContainerImage = "ghcr.io/weaveworks/gitops-bucket-server@sha256:8fbb7534e772e14ea598d287a4b54a3f556416cac6621095ce45f78346fda78a"
33+
DevBucketContainerImage = "ghcr.io/weaveworks/gitops-bucket-server:1670322194"
3334
)
3435

3536
// InstallDevBucketServer installs the dev bucket server, open port forwarding, and returns a function that can be used to the port forwarding.
36-
func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient client.Client, config *rest.Config, devBucketPort int32) (func(), error) {
37+
func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient client.Client, config *rest.Config, httpPort, httpsPort int32) (func(), []byte, error) {
3738
var (
3839
err error
3940
devBucketAppLabels = map[string]string{
@@ -57,7 +58,7 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
5758
if err != nil && apierrors.IsNotFound(err) {
5859
if err := kubeClient.Create(ctx, &devBucketNamespace); err != nil {
5960
log.Failuref("Error creating namespace %s: %v", GitOpsRunNamespace, err.Error())
60-
return nil, err
61+
return nil, nil, err
6162
} else {
6263
log.Successf("Created namespace %s", GitOpsRunNamespace)
6364
}
@@ -76,8 +77,12 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
7677
Type: corev1.ServiceTypeClusterIP,
7778
Ports: []corev1.ServicePort{
7879
{
79-
Name: RunDevBucketName,
80-
Port: devBucketPort,
80+
Name: fmt.Sprintf("%s-http", RunDevBucketName),
81+
Port: httpPort,
82+
},
83+
{
84+
Name: fmt.Sprintf("%s-https", RunDevBucketName),
85+
Port: httpsPort,
8186
},
8287
},
8388
Selector: devBucketAppLabels,
@@ -93,14 +98,38 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
9398
if err != nil && apierrors.IsNotFound(err) {
9499
if err := kubeClient.Create(ctx, &devBucketService); err != nil {
95100
log.Failuref("Error creating service %s/%s: %v", GitOpsRunNamespace, RunDevBucketName, err.Error())
96-
return nil, err
101+
return nil, nil, err
97102
} else {
98103
log.Successf("Created service %s/%s", GitOpsRunNamespace, RunDevBucketName)
99104
}
100105
} else if err == nil {
101106
log.Successf("Service %s/%s already existed", GitOpsRunNamespace, RunDevBucketName)
102107
}
103108

109+
cert, err := tls.GenerateSelfSignedCertificate("localhost", fmt.Sprintf("%s.%s.svc.cluster.local", devBucketService.Name, devBucketService.Namespace))
110+
if err != nil {
111+
err = fmt.Errorf("failed generating self-signed certificate for dev bucket server: %w", err)
112+
log.Failuref(err.Error())
113+
114+
return nil, nil, err
115+
}
116+
117+
certsSecret := &corev1.Secret{
118+
ObjectMeta: metav1.ObjectMeta{
119+
Name: "dev-bucket-server-certs",
120+
Namespace: GitOpsRunNamespace,
121+
Labels: devBucketAppLabels,
122+
},
123+
Data: map[string][]byte{
124+
"cert.pem": cert.Cert,
125+
"cert.key": cert.Key,
126+
},
127+
}
128+
if err := kubeClient.Create(ctx, certsSecret); err != nil {
129+
log.Failuref("Error creating Secret %s/%s: %v", certsSecret.Namespace, certsSecret.Name, err.Error())
130+
return nil, nil, err
131+
}
132+
104133
// create deployment
105134
replicas := int32(1)
106135
devBucketDeployment := appsv1.Deployment{
@@ -119,21 +148,43 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
119148
Labels: devBucketAppLabels,
120149
},
121150
Spec: corev1.PodSpec{
151+
Volumes: []corev1.Volume{{
152+
Name: "certs",
153+
VolumeSource: corev1.VolumeSource{
154+
Secret: &corev1.SecretVolumeSource{
155+
SecretName: "dev-bucket-server-certs",
156+
},
157+
},
158+
}},
122159
Containers: []corev1.Container{
123160
{
124-
Name: RunDevBucketName,
125-
Image: DevBucketContainerImage,
161+
Name: RunDevBucketName,
162+
Image: DevBucketContainerImage,
163+
ImagePullPolicy: corev1.PullIfNotPresent,
126164
Env: []corev1.EnvVar{
127165
{Name: "MINIO_ROOT_USER", Value: "user"},
128166
{Name: "MINIO_ROOT_PASSWORD", Value: "doesn't matter"},
129167
},
130168
Ports: []corev1.ContainerPort{
131169
{
132-
ContainerPort: devBucketPort,
133-
HostPort: devBucketPort,
170+
ContainerPort: httpPort,
171+
HostPort: httpPort,
172+
},
173+
{
174+
ContainerPort: httpsPort,
175+
HostPort: httpsPort,
134176
},
135177
},
136-
Args: []string{strconv.Itoa(int(devBucketPort))},
178+
Args: []string{
179+
fmt.Sprintf("--http-port=%d", httpPort),
180+
fmt.Sprintf("--https-port=%d", httpsPort),
181+
"--cert-file=/tmp/certs/cert.pem",
182+
"--key-file=/tmp/certs/cert.key",
183+
},
184+
VolumeMounts: []corev1.VolumeMount{{
185+
Name: "certs",
186+
MountPath: "/tmp/certs",
187+
}},
137188
},
138189
},
139190
RestartPolicy: corev1.RestartPolicyAlways,
@@ -151,7 +202,7 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
151202
if err != nil && apierrors.IsNotFound(err) {
152203
if err := kubeClient.Create(ctx, &devBucketDeployment); err != nil {
153204
log.Failuref("Error creating deployment %s/%s: %v", GitOpsRunNamespace, RunDevBucketName, err.Error())
154-
return nil, err
205+
return nil, nil, err
155206
} else {
156207
log.Successf("Created deployment %s/%s", GitOpsRunNamespace, RunDevBucketName)
157208
}
@@ -189,8 +240,8 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
189240
Name: RunDevBucketName,
190241
Namespace: GitOpsRunNamespace,
191242
Kind: "service",
192-
HostPort: strconv.Itoa(int(devBucketPort)),
193-
ContainerPort: strconv.Itoa(int(devBucketPort)),
243+
HostPort: strconv.Itoa(int(httpsPort)),
244+
ContainerPort: strconv.Itoa(int(httpsPort)),
194245
}
195246
// get pod from specMap
196247
namespacedName := types.NamespacedName{Namespace: specMap.Namespace, Name: specMap.Name}
@@ -218,10 +269,10 @@ func InstallDevBucketServer(ctx context.Context, log logger.Logger, kubeClient c
218269

219270
log.Successf("Port forwarding for %s is ready.", RunDevBucketName)
220271

221-
return cancelPortFwd, nil
272+
return cancelPortFwd, cert.Cert, nil
222273
}
223274

224-
return nil, fmt.Errorf("pod not found")
275+
return nil, nil, fmt.Errorf("pod not found")
225276
}
226277

227278
// UninstallDevBucketServer deletes the dev-bucket namespace.

‎pkg/s3/minio.go‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
package s3
2+
3+
import (
4+
"crypto/tls"
5+
"crypto/x509"
6+
"fmt"
7+
"net/http"
8+
9+
"github.com/minio/minio-go/v7"
10+
"github.com/minio/minio-go/v7/pkg/credentials"
11+
)
12+
13+
func NewMinioClient(endpoint string, caCert []byte) (*minio.Client, error) {
14+
tr, err := NewTLSRoundTripper(caCert)
15+
if err != nil {
16+
return nil, fmt.Errorf("failed creating transport: %w", err)
17+
}
18+
19+
return minio.New(
20+
endpoint,
21+
&minio.Options{
22+
Creds: credentials.NewStaticV4("user", "doesn't matter", ""),
23+
Secure: true,
24+
BucketLookup: minio.BucketLookupPath,
25+
Transport: tr,
26+
},
27+
)
28+
}
29+
30+
func NewTLSRoundTripper(caCert []byte) (http.RoundTripper, error) {
31+
tr, err := minio.DefaultTransport(true)
32+
if err != nil {
33+
return nil, fmt.Errorf("failed creating default transport: %w", err)
34+
}
35+
36+
tr.TLSClientConfig = &tls.Config{
37+
// Can't use SSLv3 because of POODLE and BEAST
38+
// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
39+
// Can't use TLSv1.1 because of RC4 cipher usage
40+
MinVersion: tls.VersionTLS12,
41+
}
42+
rootCAs := mustGetSystemCertPool()
43+
44+
rootCAs.AppendCertsFromPEM(caCert)
45+
tr.TLSClientConfig.RootCAs = rootCAs
46+
47+
return tr, nil
48+
}
49+
50+
// mustGetSystemCertPool - return system CAs or empty pool in case of error (or windows)
51+
func mustGetSystemCertPool() *x509.CertPool {
52+
pool, err := x509.SystemCertPool()
53+
if err != nil {
54+
return x509.NewCertPool()
55+
}
56+
57+
return pool
58+
}

0 commit comments

Comments
 (0)