Skip to content

feat(controller): add parameter to fine-tune max concurrent reconciles #288

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jun 4, 2024
Merged

feat(controller): add parameter to fine-tune max concurrent reconciles #288

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/controllers/start.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ func buildControllersStartCmd(app *burrito.App) *cobra.Command {
cmd.Flags().DurationVar(&app.Config.Controller.Timers.WaitAction, "wait-action-period", defaultWaitActionTimer, "period between two runners when a layer is locked. Must end with s, m or h.")
cmd.Flags().DurationVar(&app.Config.Controller.Timers.FailureGracePeriod, "failure-grace-period", defaultFailureGracePeriod, "initial time before retry, goes exponential function of number failure. Must end with s, m or h.")
cmd.Flags().IntVar(&app.Config.Controller.TerraformMaxRetries, "terraform-max-retries", 5, "default number of retries for terraform actions (can be overriden in CRDs)")
cmd.Flags().IntVar(&app.Config.Controller.MaxConcurrentReconciles, "max-concurrent-reconciles", 1, "maximum number of concurrent reconciles")
cmd.Flags().BoolVar(&app.Config.Controller.LeaderElection.Enabled, "leader-election", true, "whether leader election is enabled or not, default to true")
cmd.Flags().StringVar(&app.Config.Controller.LeaderElection.ID, "leader-election-id", "6d185457.terraform.padok.cloud", "lease id used for leader election")
cmd.Flags().StringVar(&app.Config.Controller.HealthProbeBindAddress, "health-probe-bind-address", ":8081", "address to bind the metrics server embedded in the controllers")
Expand Down
3 changes: 2 additions & 1 deletion deploy/charts/burrito/values.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ config:
# -- By default, the controller will only watch the tenants namespaces
namespaces: []
timers: {}
maxConcurrentReconciles: 1
terraformMaxRetries: 3
types: ["layer", "repository", "run", "pullrequest"]
leaderElection:
Expand Down Expand Up @@ -45,7 +46,7 @@ config:
storage:
gcs: {}
azure: {}
aws: {}
s3: {}
addr: ":8080"

# Burrito server configuration
Expand Down
28 changes: 15 additions & 13 deletions internal/burrito/config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,18 +58,19 @@ type WebhookGitlabConfig struct {
}

type ControllerConfig struct {
MainNamespace string `mapstructure:"mainNamespace"`
Namespaces []string `mapstructure:"namespaces"`
Timers ControllerTimers `mapstructure:"timers"`
TerraformMaxRetries int `mapstructure:"terraformMaxRetries"`
Types []string `mapstructure:"types"`
LeaderElection LeaderElectionConfig `mapstructure:"leaderElection"`
MetricsBindAddress string `mapstructure:"metricsBindAddress"`
HealthProbeBindAddress string `mapstructure:"healthProbeBindAddress"`
KubernetesWebhookPort int `mapstructure:"kubernetesWebhookPort"`
GithubConfig GithubConfig `mapstructure:"githubConfig"`
GitlabConfig GitlabConfig `mapstructure:"gitlabConfig"`
RunParallelism int `mapstructure:"runParallelism"`
MainNamespace string `mapstructure:"mainNamespace"`
Namespaces []string `mapstructure:"namespaces"`
Timers ControllerTimers `mapstructure:"timers"`
TerraformMaxRetries int `mapstructure:"terraformMaxRetries"`
Types []string `mapstructure:"types"`
LeaderElection LeaderElectionConfig `mapstructure:"leaderElection"`
MetricsBindAddress string `mapstructure:"metricsBindAddress"`
HealthProbeBindAddress string `mapstructure:"healthProbeBindAddress"`
KubernetesWebhookPort int `mapstructure:"kubernetesWebhookPort"`
GithubConfig GithubConfig `mapstructure:"githubConfig"`
GitlabConfig GitlabConfig `mapstructure:"gitlabConfig"`
RunParallelism int `mapstructure:"runParallelism"`
MaxConcurrentReconciles int `mapstructure:"maxConcurrentReconciles"`
}

type GithubConfig struct {
Expand Down Expand Up @@ -206,7 +207,8 @@ func bindEnvironmentVariables(v *viper.Viper, iface interface{}, parts ...string
func TestConfig() *Config {
return &Config{
Controller: ControllerConfig{
TerraformMaxRetries: 5,
TerraformMaxRetries: 5,
MaxConcurrentReconciles: 1,
Timers: ControllerTimers{
DriftDetection: 20 * time.Minute,
WaitAction: 5 * time.Minute,
Expand Down
11 changes: 7 additions & 4 deletions internal/burrito/config/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,9 @@ func TestConfig_FromYamlFile(t *testing.T) {
WaitAction: 1 * time.Minute,
FailureGracePeriod: 15 * time.Second,
},
TerraformMaxRetries: 5,
Types: []string{"layer", "repository", "run", "pullrequest"},
TerraformMaxRetries: 5,
MaxConcurrentReconciles: 1,
Types: []string{"layer", "repository", "run", "pullrequest"},
LeaderElection: config.LeaderElectionConfig{
Enabled: true,
ID: "6d185457.terraform.padok.cloud",
Expand Down Expand Up @@ -136,6 +137,7 @@ func TestConfig_EnvVarOverrides(t *testing.T) {
setEnvVar(t, "BURRITO_CONTROLLER_TIMERS_ONERROR", "30s", &envVarList)
setEnvVar(t, "BURRITO_CONTROLLER_TIMERS_WAITACTION", "30s", &envVarList)
setEnvVar(t, "BURRITO_CONTROLLER_TIMERS_FAILUREGRACEPERIOD", "1m", &envVarList)
setEnvVar(t, "BURRITO_CONTROLLER_MAXCONCURRENTRECONCILES", "3", &envVarList)
setEnvVar(t, "BURRITO_CONTROLLER_TERRAFORMMAXRETRIES", "32", &envVarList)
setEnvVar(t, "BURRITO_CONTROLLER_LEADERELECTION_ID", "other-leader-id", &envVarList)
setEnvVar(t, "BURRITO_CONTROLLER_GITHUBCONFIG_APPID", "123456", &envVarList)
Expand Down Expand Up @@ -189,8 +191,9 @@ func TestConfig_EnvVarOverrides(t *testing.T) {
WaitAction: 30 * time.Second,
FailureGracePeriod: 1 * time.Minute,
},
TerraformMaxRetries: 32,
Types: []string{"layer", "repository"},
MaxConcurrentReconciles: 3,
TerraformMaxRetries: 32,
Types: []string{"layer", "repository"},
LeaderElection: config.LeaderElectionConfig{
Enabled: true,
ID: "other-leader-id",
Expand Down
1 change: 1 addition & 0 deletions internal/burrito/config/testdata/test-config-1.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ controller:
waitAction: 1m
failureGracePeriod: 15s
terraformMaxRetries: 5
maxConcurrentReconciles: 1
types: ["layer", "repository", "run", "pullrequest"]
leaderElection:
enabled: true
Expand Down
1 change: 1 addition & 0 deletions internal/controllers/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,7 @@ func (c *Controllers) Exec() {
Client: mgr.GetClient(),
Scheme: mgr.GetScheme(),
Recorder: mgr.GetEventRecorderFor("Burrito"),
Config: c.config,
}).SetupWithManager(mgr); err != nil {
log.Fatalf("unable to create repository controller: %s", err)
}
Expand Down
2 changes: 2 additions & 0 deletions internal/controllers/terraformlayer/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/predicate"

Expand Down Expand Up @@ -202,6 +203,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
r.Clock = RealClock{}
return ctrl.NewControllerManagedBy(mgr).
For(&configv1alpha1.TerraformLayer{}).
WithOptions(controller.Options{MaxConcurrentReconciles: r.Config.Controller.MaxConcurrentReconciles}).
WithEventFilter(ignorePredicate()).
Complete(r)
}
Expand Down
2 changes: 2 additions & 0 deletions internal/controllers/terraformpullrequest/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/predicate"

Expand Down Expand Up @@ -111,6 +112,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
r.Providers = providers
return ctrl.NewControllerManagedBy(mgr).
For(&configv1alpha1.TerraformPullRequest{}).
WithOptions(controller.Options{MaxConcurrentReconciles: r.Config.Controller.MaxConcurrentReconciles}).
WithEventFilter(ignorePredicate()).
Complete(r)
}
Expand Down
4 changes: 4 additions & 0 deletions internal/controllers/terraformrepository/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,18 @@ import (
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"

configv1alpha1 "github.com/padok-team/burrito/api/v1alpha1"
"github.com/padok-team/burrito/internal/burrito/config"
)

// RepositoryReconciler reconciles a TerraformRepository object
type Reconciler struct {
client.Client
Scheme *runtime.Scheme
Recorder record.EventRecorder
Config *config.Config
}

//+kubebuilder:rbac:groups=config.terraform.padok.cloud,resources=terraformrepositories,verbs=get;list;watch;create;update;patch;delete
Expand Down Expand Up @@ -60,5 +63,6 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Resu
func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&configv1alpha1.TerraformRepository{}).
WithOptions(controller.Options{MaxConcurrentReconciles: r.Config.Controller.MaxConcurrentReconciles}).
Complete(r)
}
2 changes: 2 additions & 0 deletions internal/controllers/terraformrun/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ import (
"k8s.io/client-go/tools/record"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/controller"
"sigs.k8s.io/controller-runtime/pkg/event"
"sigs.k8s.io/controller-runtime/pkg/predicate"

Expand Down Expand Up @@ -186,6 +187,7 @@ func (r *Reconciler) SetupWithManager(mgr ctrl.Manager) error {
r.Clock = RealClock{}
return ctrl.NewControllerManagedBy(mgr).
For(&configv1alpha1.TerraformRun{}).
WithOptions(controller.Options{MaxConcurrentReconciles: r.Config.Controller.MaxConcurrentReconciles}).
WithEventFilter(ignorePredicate()).
Complete(r)
}
Expand Down