Skip to content

Commit 9ef1821

Browse files
AlanLonguetAlancorrierilucaLucasMrqes
committed
refactor(provider): create dedicated credentials package and rework repository provider (#528)
* refactor(provider): create dedicated credentials package and rework repository provider * test(credentials): cover most non transient paths * fix(creds): ttl condition on updating credentials * fix(creds): add some comments in credential store * fix(credentials): make mock provider work * feat(credentials): controllers use credentials store * feat(webhook): webhook server use credentials store * feat(manifests): remove secretName from tfrepo * refactor(config): remove deprecated structs for credentials * fix(lint): check err values * fix(test): rework test NormalizeURL * fix(lint): check err values * feat(config): add config key for credentials TTL * feat(repo-controller): update bundle with latest changes * fix(credentials): add secret indexer on field type * fix(providers): use the correct repo URL --------- Co-authored-by: Alan <[email protected]> Co-authored-by: Luca Corrieri <[email protected]> Co-authored-by: Lucas Marques <[email protected]>
1 parent e79bda0 commit 9ef1821

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+1662
-1794
lines changed

‎api/v1alpha1/terraformrepository_types.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,8 +41,7 @@ type TerraformRepositorySpec struct {
4141
SyncWindows []SyncWindow `json:"syncWindows,omitempty"`
4242
}
4343
type TerraformRepositoryRepository struct {
44-
Url string `json:"url,omitempty"`
45-
SecretName string `json:"secretName,omitempty"`
44+
Url string `json:"url,omitempty"`
4645
}
4746

4847
// TerraformRepositoryStatus defines the observed state of TerraformRepository

‎cmd/controllers/start.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,13 @@ func buildControllersStartCmd(app *burrito.App) *cobra.Command {
2828
defaultWaitActionTimer, _ := time.ParseDuration("5s")
2929
defaultFailureGracePeriod, _ := time.ParseDuration("15s")
3030
defaultRepositorySyncTimer, _ := time.ParseDuration("5m")
31+
defaultCredentialsTTL, _ := time.ParseDuration("2m")
3132

3233
cmd.Flags().StringSliceVar(&app.Config.Controller.Namespaces, "namespaces", []string{"burrito-system"}, "list of namespaces to watch")
3334
cmd.Flags().StringArrayVar(&app.Config.Controller.Types, "types", []string{"layer", "run", "pullrequest"}, "list of controllers to start")
3435
cmd.Flags().DurationVar(&app.Config.Controller.Timers.DriftDetection, "drift-detection-period", defaultDriftDetectionTimer, "period between two plans. Must end with s, m or h.")
3536
cmd.Flags().DurationVar(&app.Config.Controller.Timers.RepositorySync, "repository-sync-period", defaultRepositorySyncTimer, "period between two repository sync. Must end with s, m or h.")
37+
cmd.Flags().DurationVar(&app.Config.Controller.Timers.CredentialsTTL, "credentials-ttl", defaultCredentialsTTL, "default TTL for git providers credentials in controller's memory. Must end with s, m or h.")
3638
cmd.Flags().DurationVar(&app.Config.Controller.Timers.OnError, "on-error-period", defaultOnErrorTimer, "period between two runners launch when an error occurred in the controllers. Must end with s, m or h.")
3739
cmd.Flags().DurationVar(&app.Config.Controller.Timers.WaitAction, "wait-action-period", defaultWaitActionTimer, "period between two runners when a layer is locked. Must end with s, m or h.")
3840
cmd.Flags().DurationVar(&app.Config.Controller.Timers.FailureGracePeriod, "failure-grace-period", defaultFailureGracePeriod, "initial time before retry, goes exponential function of number failure. Must end with s, m or h.")

‎internal/annotations/annotations.go

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,8 @@ const (
2727
ForceApply string = "notifications.terraform.padok.cloud/force-apply"
2828
AdditionnalTriggerPaths string = "config.terraform.padok.cloud/additionnal-trigger-paths"
2929

30-
SyncNow string = "api.terraform.padok.cloud/sync-now"
30+
SyncNow string = "api.terraform.padok.cloud/sync-now"
31+
AllowedTenants string = "credentials.terraform.padok.cloud/allowed-tenants"
3132
)
3233

3334
func ComputeKeyForSyncBranchNow(branch string) string {

‎internal/burrito/config/config.go

Lines changed: 12 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -50,19 +50,6 @@ type AzureConfig struct {
5050
Container string `mapstructure:"container"`
5151
}
5252

53-
type WebhookConfig struct {
54-
Github WebhookGithubConfig `mapstructure:"github"`
55-
Gitlab WebhookGitlabConfig `mapstructure:"gitlab"`
56-
}
57-
58-
type WebhookGithubConfig struct {
59-
Secret string `mapstructure:"secret"`
60-
}
61-
62-
type WebhookGitlabConfig struct {
63-
Secret string `mapstructure:"secret"`
64-
}
65-
6653
type ControllerConfig struct {
6754
MainNamespace string `mapstructure:"mainNamespace"`
6855
Namespaces []string `mapstructure:"namespaces"`
@@ -74,25 +61,11 @@ type ControllerConfig struct {
7461
MetricsBindAddress string `mapstructure:"metricsBindAddress"`
7562
HealthProbeBindAddress string `mapstructure:"healthProbeBindAddress"`
7663
KubernetesWebhookPort int `mapstructure:"kubernetesWebhookPort"`
77-
GithubConfig GithubConfig `mapstructure:"githubConfig"`
78-
GitlabConfig GitlabConfig `mapstructure:"gitlabConfig"`
7964
RunParallelism int `mapstructure:"runParallelism"`
8065
MaxConcurrentReconciles int `mapstructure:"maxConcurrentReconciles"`
8166
MaxConcurrentRunnerPods int `mapstructure:"maxConcurrentRunnerPods"`
8267
}
8368

84-
type GithubConfig struct {
85-
AppId int64 `mapstructure:"appId"`
86-
InstallationId int64 `mapstructure:"installationId"`
87-
PrivateKey string `mapstructure:"privateKey"`
88-
APIToken string `mapstructure:"apiToken"`
89-
}
90-
91-
type GitlabConfig struct {
92-
APIToken string `mapstructure:"apiToken"`
93-
URL string `mapstructure:"url"`
94-
}
95-
9669
type LeaderElectionConfig struct {
9770
Enabled bool `mapstructure:"enabled"`
9871
ID string `mapstructure:"id"`
@@ -104,30 +77,19 @@ type ControllerTimers struct {
10477
WaitAction time.Duration `mapstructure:"waitAction"`
10578
FailureGracePeriod time.Duration `mapstructure:"failureGracePeriod"`
10679
RepositorySync time.Duration `mapstructure:"repositorySync"`
107-
}
108-
109-
type RepositoryConfig struct {
110-
SSHPrivateKey string `mapstructure:"sshPrivateKey"`
111-
Username string `mapstructure:"username"`
112-
Password string `mapstructure:"password"`
113-
GithubAppId int64 `mapstructure:"githubAppId"`
114-
GithubAppInstallationId int64 `mapstructure:"githubAppInstallationId"`
115-
GithubAppPrivateKey string `mapstructure:"githubAppPrivateKey"`
116-
GithubToken string `mapstructure:"githubToken"`
117-
GitlabToken string `mapstructure:"gitlabToken"`
80+
CredentialsTTL time.Duration `mapstructure:"credentialsTTL"`
11881
}
11982

12083
type RunnerConfig struct {
121-
Action string `mapstructure:"action"`
122-
Layer Layer `mapstructure:"layer"`
123-
Run string `mapstructure:"run"`
124-
Repository RepositoryConfig `mapstructure:"repository"`
125-
SSHKnownHostsConfigMapName string `mapstructure:"sshKnownHostsConfigMapName"`
126-
Image ImageConfig `mapstructure:"image"`
127-
RunnerBinaryPath string `mapstructure:"runnerBinaryPath"`
128-
RepositoryPath string `mapstructure:"repositoryPath"`
129-
Args []string `mapstructure:"args"`
130-
Command []string `mapstructure:"command"`
84+
Action string `mapstructure:"action"`
85+
Layer Layer `mapstructure:"layer"`
86+
Run string `mapstructure:"run"`
87+
SSHKnownHostsConfigMapName string `mapstructure:"sshKnownHostsConfigMapName"`
88+
Image ImageConfig `mapstructure:"image"`
89+
RunnerBinaryPath string `mapstructure:"runnerBinaryPath"`
90+
RepositoryPath string `mapstructure:"repositoryPath"`
91+
Args []string `mapstructure:"args"`
92+
Command []string `mapstructure:"command"`
13193
}
13294

13395
type ImageConfig struct {
@@ -148,8 +110,7 @@ type HermitcrabConfig struct {
148110
}
149111

150112
type ServerConfig struct {
151-
Addr string `mapstructure:"addr"`
152-
Webhook WebhookConfig `mapstructure:"webhook"`
113+
Addr string `mapstructure:"addr"`
153114
}
154115

155116
func (c *Config) Load(flags *pflag.FlagSet) error {
@@ -241,6 +202,7 @@ func TestConfig() *Config {
241202
FailureGracePeriod: 15 * time.Second,
242203
OnError: 1 * time.Minute,
243204
RepositorySync: 5 * time.Minute,
205+
CredentialsTTL: 5 * time.Second,
244206
},
245207
},
246208
Runner: RunnerConfig{

‎internal/burrito/config/config_test.go

Lines changed: 3 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,6 @@ func TestConfig_FromYamlFile(t *testing.T) {
5050
Tag: "test-tag",
5151
PullPolicy: "Always",
5252
},
53-
Repository: config.RepositoryConfig{
54-
SSHPrivateKey: "private-key",
55-
Username: "test",
56-
Password: "password",
57-
},
5853
SSHKnownHostsConfigMapName: "burrito-ssh-known-hosts",
5954
},
6055
Controller: config.ControllerConfig{
@@ -64,6 +59,7 @@ func TestConfig_FromYamlFile(t *testing.T) {
6459
OnError: 1 * time.Minute,
6560
WaitAction: 1 * time.Minute,
6661
FailureGracePeriod: 15 * time.Second,
62+
CredentialsTTL: 1 * time.Hour,
6763
},
6864
DefaultSyncWindows: []configv1alpha1.SyncWindow{
6965
{
@@ -85,27 +81,9 @@ func TestConfig_FromYamlFile(t *testing.T) {
8581
MetricsBindAddress: ":8080",
8682
HealthProbeBindAddress: ":8081",
8783
KubernetesWebhookPort: 9443,
88-
GithubConfig: config.GithubConfig{
89-
AppId: 123456,
90-
InstallationId: 12345678,
91-
PrivateKey: "private-key",
92-
APIToken: "github-token",
93-
},
94-
GitlabConfig: config.GitlabConfig{
95-
APIToken: "gitlab-token",
96-
URL: "https://gitlab.example.com",
97-
},
9884
},
9985
Server: config.ServerConfig{
10086
Addr: ":9090",
101-
Webhook: config.WebhookConfig{
102-
Github: config.WebhookGithubConfig{
103-
Secret: "github-secret",
104-
},
105-
Gitlab: config.WebhookGitlabConfig{
106-
Secret: "gitlab-secret",
107-
},
108-
},
10987
},
11088
}
11189

@@ -143,9 +121,6 @@ func TestConfig_EnvVarOverrides(t *testing.T) {
143121
setEnvVar(t, "BURRITO_RUNNER_ACTION", "plan", &envVarList)
144122
setEnvVar(t, "BURRITO_RUNNER_LAYER_NAME", "other-layer", &envVarList)
145123
setEnvVar(t, "BURRITO_RUNNER_LAYER_NAMESPACE", "other-namespace", &envVarList)
146-
setEnvVar(t, "BURRITO_RUNNER_REPOSITORY_USERNAME", "other-username", &envVarList)
147-
setEnvVar(t, "BURRITO_RUNNER_REPOSITORY_PASSWORD", "other-password", &envVarList)
148-
setEnvVar(t, "BURRITO_RUNNER_REPOSITORY_SSHPRIVATEKEY", "other-private-key", &envVarList)
149124
setEnvVar(t, "BURRITO_RUNNER_IMAGE_REPOSITORY", "other-repository", &envVarList)
150125
setEnvVar(t, "BURRITO_RUNNER_IMAGE_TAG", "other-tag", &envVarList)
151126
setEnvVar(t, "BURRITO_RUNNER_IMAGE_PULLPOLICY", "Always", &envVarList)
@@ -156,20 +131,13 @@ func TestConfig_EnvVarOverrides(t *testing.T) {
156131
setEnvVar(t, "BURRITO_CONTROLLER_TIMERS_ONERROR", "30s", &envVarList)
157132
setEnvVar(t, "BURRITO_CONTROLLER_TIMERS_WAITACTION", "30s", &envVarList)
158133
setEnvVar(t, "BURRITO_CONTROLLER_TIMERS_FAILUREGRACEPERIOD", "1m", &envVarList)
134+
setEnvVar(t, "BURRITO_CONTROLLER_TIMERS_CREDENTIALSTTL", "2h", &envVarList)
159135
setEnvVar(t, "BURRITO_CONTROLLER_MAXCONCURRENTRECONCILES", "3", &envVarList)
160136
setEnvVar(t, "BURRITO_CONTROLLER_MAXCONCURRENTRUNNERPODS", "10", &envVarList)
161137
setEnvVar(t, "BURRITO_CONTROLLER_TERRAFORMMAXRETRIES", "32", &envVarList)
162138
setEnvVar(t, "BURRITO_CONTROLLER_LEADERELECTION_ID", "other-leader-id", &envVarList)
163-
setEnvVar(t, "BURRITO_CONTROLLER_GITHUBCONFIG_APPID", "123456", &envVarList)
164-
setEnvVar(t, "BURRITO_CONTROLLER_GITHUBCONFIG_INSTALLATIONID", "12345678", &envVarList)
165-
setEnvVar(t, "BURRITO_CONTROLLER_GITHUBCONFIG_PRIVATEKEY", "private-key", &envVarList)
166-
setEnvVar(t, "BURRITO_CONTROLLER_GITHUBCONFIG_APITOKEN", "pr-github-token", &envVarList)
167-
setEnvVar(t, "BURRITO_CONTROLLER_GITLABCONFIG_APITOKEN", "mr-gitlab-token", &envVarList)
168-
setEnvVar(t, "BURRITO_CONTROLLER_GITLABCONFIG_URL", "https://gitlab.com", &envVarList)
169139
// Server
170140
setEnvVar(t, "BURRITO_SERVER_ADDR", ":8090", &envVarList)
171-
setEnvVar(t, "BURRITO_SERVER_WEBHOOK_GITHUB_SECRET", "other-github-secret", &envVarList)
172-
setEnvVar(t, "BURRITO_SERVER_WEBHOOK_GITLAB_SECRET", "other-gitlab-secret", &envVarList)
173141

174142
// Create a test config instance
175143
cfg := &config.Config{}
@@ -201,11 +169,6 @@ func TestConfig_EnvVarOverrides(t *testing.T) {
201169
Tag: "other-tag",
202170
PullPolicy: "Always",
203171
},
204-
Repository: config.RepositoryConfig{
205-
SSHPrivateKey: "other-private-key",
206-
Username: "other-username",
207-
Password: "other-password",
208-
},
209172
SSHKnownHostsConfigMapName: "burrito-ssh-known-hosts",
210173
},
211174
Controller: config.ControllerConfig{
@@ -215,6 +178,7 @@ func TestConfig_EnvVarOverrides(t *testing.T) {
215178
OnError: 30 * time.Second,
216179
WaitAction: 30 * time.Second,
217180
FailureGracePeriod: 1 * time.Minute,
181+
CredentialsTTL: 2 * time.Hour,
218182
},
219183
DefaultSyncWindows: []configv1alpha1.SyncWindow{
220184
{
@@ -236,27 +200,9 @@ func TestConfig_EnvVarOverrides(t *testing.T) {
236200
MetricsBindAddress: ":8080",
237201
HealthProbeBindAddress: ":8081",
238202
KubernetesWebhookPort: 9443,
239-
GithubConfig: config.GithubConfig{
240-
AppId: 123456,
241-
InstallationId: 12345678,
242-
PrivateKey: "private-key",
243-
APIToken: "pr-github-token",
244-
},
245-
GitlabConfig: config.GitlabConfig{
246-
APIToken: "mr-gitlab-token",
247-
URL: "https://gitlab.com",
248-
},
249203
},
250204
Server: config.ServerConfig{
251205
Addr: ":8090",
252-
Webhook: config.WebhookConfig{
253-
Github: config.WebhookGithubConfig{
254-
Secret: "other-github-secret",
255-
},
256-
Gitlab: config.WebhookGitlabConfig{
257-
Secret: "other-gitlab-secret",
258-
},
259-
},
260206
},
261207
}
262208

‎internal/burrito/config/testdata/test-config-1.yaml

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,6 @@ runner:
77
repository: test-repository
88
tag: test-tag
99
pullPolicy: Always
10-
repository:
11-
sshPrivateKey: "private-key"
12-
username: "test"
13-
password: "password"
1410
sshKnownHostsConfigMapName: burrito-ssh-known-hosts
1511

1612
controller:
@@ -22,6 +18,7 @@ controller:
2218
onError: 1m
2319
waitAction: 1m
2420
failureGracePeriod: 15s
21+
credentialsTTL: 1h
2522
defaultSyncWindows:
2623
- kind: "deny"
2724
schedule: "0 0 * * *"
@@ -38,19 +35,6 @@ controller:
3835
metricsBindAddress: ":8080"
3936
healthProbeBindAddress: ":8081"
4037
kubernetesWebhookPort: 9443
41-
githubConfig:
42-
appId: 123456
43-
installationId: 12345678
44-
privateKey: "private-key"
45-
apiToken: "github-token"
46-
gitlabConfig:
47-
apiToken: "gitlab-token"
48-
url: "https://gitlab.example.com"
4938

5039
server:
5140
addr: ":9090"
52-
webhook:
53-
github:
54-
secret: "github-secret"
55-
gitlab:
56-
secret: "gitlab-secret"

‎internal/controllers/manager.go

Lines changed: 25 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,10 @@ limitations under the License.
1717
package controllers
1818

1919
import (
20+
"context"
2021
"os"
2122

23+
corev1 "k8s.io/api/core/v1"
2224
logClient "k8s.io/client-go/kubernetes"
2325
_ "k8s.io/client-go/plugin/pkg/client/auth"
2426
"k8s.io/client-go/rest"
@@ -28,6 +30,7 @@ import (
2830
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
2931
ctrl "sigs.k8s.io/controller-runtime"
3032
"sigs.k8s.io/controller-runtime/pkg/cache"
33+
"sigs.k8s.io/controller-runtime/pkg/client"
3134
"sigs.k8s.io/controller-runtime/pkg/healthz"
3235
"sigs.k8s.io/controller-runtime/pkg/manager"
3336
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -38,6 +41,7 @@ import (
3841
"github.com/padok-team/burrito/internal/controllers/terraformrepository"
3942
"github.com/padok-team/burrito/internal/controllers/terraformrun"
4043
datastore "github.com/padok-team/burrito/internal/datastore/client"
44+
"github.com/padok-team/burrito/internal/repository/credentials"
4145
"github.com/sirupsen/logrus"
4246
log "github.com/sirupsen/logrus"
4347

@@ -99,6 +103,7 @@ func (c *Controllers) Exec() {
99103
log.Fatalf("unable to start manager: %s", err)
100104
}
101105
datastoreClient := datastore.NewDefaultClient(c.config.Datastore)
106+
credentialStore := credentials.NewCredentialStore(mgr.GetClient(), c.config.Controller.Timers.CredentialsTTL)
102107
config, err := rest.InClusterConfig()
103108
if err != nil {
104109
panic(err.Error())
@@ -108,6 +113,14 @@ func (c *Controllers) Exec() {
108113
panic(err.Error())
109114
}
110115

116+
// Setup field indexer for Secret type (used for credentials)
117+
if err := mgr.GetFieldIndexer().IndexField(context.Background(), &corev1.Secret{}, "type", func(rawObj client.Object) []string {
118+
secret := rawObj.(*corev1.Secret)
119+
return []string{string(secret.Type)}
120+
}); err != nil {
121+
panic(err.Error())
122+
}
123+
111124
log.Infof("starting these controllers: %v", c.config.Controller.Types)
112125

113126
for _, ctrlType := range c.config.Controller.Types {
@@ -125,11 +138,12 @@ func (c *Controllers) Exec() {
125138
log.Infof("layer controller started successfully")
126139
case "repository":
127140
if err = (&terraformrepository.Reconciler{
128-
Client: mgr.GetClient(),
129-
Scheme: mgr.GetScheme(),
130-
Recorder: mgr.GetEventRecorderFor("Burrito"),
131-
Config: c.config,
132-
Datastore: datastoreClient,
141+
Client: mgr.GetClient(),
142+
Scheme: mgr.GetScheme(),
143+
Recorder: mgr.GetEventRecorderFor("Burrito"),
144+
Config: c.config,
145+
Datastore: datastoreClient,
146+
Credentials: credentialStore,
133147
}).SetupWithManager(mgr); err != nil {
134148
log.Fatalf("unable to create repository controller: %s", err)
135149
}
@@ -148,11 +162,12 @@ func (c *Controllers) Exec() {
148162
log.Infof("run controller started successfully")
149163
case "pullrequest":
150164
if err = (&terraformpullrequest.Reconciler{
151-
Client: mgr.GetClient(),
152-
Scheme: mgr.GetScheme(),
153-
Recorder: mgr.GetEventRecorderFor("Burrito"),
154-
Config: c.config,
155-
Datastore: datastoreClient,
165+
Client: mgr.GetClient(),
166+
Scheme: mgr.GetScheme(),
167+
Recorder: mgr.GetEventRecorderFor("Burrito"),
168+
Config: c.config,
169+
Datastore: datastoreClient,
170+
Credentials: credentialStore,
156171
}).SetupWithManager(mgr); err != nil {
157172
log.Fatalf("unable to create pullrequest controller: %s", err)
158173
}

0 commit comments

Comments
 (0)