implement exponential backoff

The createTaskRun and createCustomRun now uses wait.ExponentialBackoff to retry
the creation of a taskRun or customRun when certain errors occur, specifically
webhook timeouts.

The function isWebhookTimeout checks if an error is a mutating adminssion
webhook timeout, by looking for HTTP 500 and the phrase "timeout" in the error
message.

If a webhook timeout is detected, the backoff loop will retry the creation
up to a configured number of steps, with increasing delay between attempts.

if the error is not a webhook timeout, the function will not retry and will
return the error immediately.

Errors that not webhook timeouts, e.g. HTTP 400 bad request, validation errors,
etc. are not retried and will cause the taskRun creation to fail as expected.

By default, the exponential backoff strategy is disabled. To enable this
feature, set the `enable-wait-exponential-backoff` to `true` in
feature-flags config map.

When enabled, the controller will use an exponential backoff strategy to retry
taskRun and customRun creation if it encounters transient errors such as
admission webhook timeouts.

This improves robustness against temporary webhook issues. If the feature flag
is set to false, the controller will not retry and will fail immediately on
such errors.

Configuration for the backoff parameters (duration, factor, steps, etc) can be
set in the wait-exponential-backoff config map.

Signed-off-by: Priti Desai <[email protected]>

Author: pritidesai

config/300-crds/300-pipelinerun.yaml

@@ -2200,6 +2200,8 @@ spec:
                         enableStepActions:
                           description: EnableStepActions is a no-op flag since StepActions are stable
                           type: boolean
+                        enableWaitExponentialBackoff:
+                          type: boolean
                         enforceNonfalsifiability:
                           type: string
                         maxResultSize:
@@ -2613,6 +2615,8 @@ spec:
                                   enableStepActions:
                                     description: EnableStepActions is a no-op flag since StepActions are stable
                                     type: boolean
+                                  enableWaitExponentialBackoff:
+                                    type: boolean
                                   enforceNonfalsifiability:
                                     type: string
                                   maxResultSize:
@@ -2900,6 +2904,8 @@ spec:
                                         enableStepActions:
                                           description: EnableStepActions is a no-op flag since StepActions are stable
                                           type: boolean
+                                        enableWaitExponentialBackoff:
+                                          type: boolean
                                         enforceNonfalsifiability:
                                           type: string
                                         maxResultSize:
@@ -5120,6 +5126,8 @@ spec:
                         enableStepActions:
                           description: EnableStepActions is a no-op flag since StepActions are stable
                           type: boolean
+                        enableWaitExponentialBackoff:
+                          type: boolean
                         enforceNonfalsifiability:
                           type: string
                         maxResultSize:

config/300-crds/300-taskrun.yaml

@@ -1776,6 +1776,8 @@ spec:
                         enableStepActions:
                           description: EnableStepActions is a no-op flag since StepActions are stable
                           type: boolean
+                        enableWaitExponentialBackoff:
+                          type: boolean
                         enforceNonfalsifiability:
                           type: string
                         maxResultSize:
@@ -2063,6 +2065,8 @@ spec:
                               enableStepActions:
                                 description: EnableStepActions is a no-op flag since StepActions are stable
                                 type: boolean
+                              enableWaitExponentialBackoff:
+                                type: boolean
                               enforceNonfalsifiability:
                                 type: string
                               maxResultSize:
@@ -3767,6 +3771,8 @@ spec:
                         enableStepActions:
                           description: EnableStepActions is a no-op flag since StepActions are stable
                           type: boolean
+                        enableWaitExponentialBackoff:
+                          type: boolean
                         enforceNonfalsifiability:
                           type: string
                         maxResultSize:
@@ -4020,6 +4026,8 @@ spec:
                               enableStepActions:
                                 description: EnableStepActions is a no-op flag since StepActions are stable
                                 type: boolean
+                              enableWaitExponentialBackoff:
+                                type: boolean
                               enforceNonfalsifiability:
                                 type: string
                               maxResultSize:

config/config-feature-flags.yaml

@@ -130,3 +130,9 @@ data:
   enable-kubernetes-sidecar: "false"
   # Setting this flag to "false" will have no effect since StepActions are a stable feature
   enable-step-actions: "true"
+  # Controls whether exponential backoff is enabled when creating TaskRuns or CustomRuns.
+  # If set to "true", the controller will use exponential backoff when retrying failed create operations,
+  # which can help mitigate issues caused by temporary API server or webhook unavailability.
+  # If set to "false", exponential backoff will be disabled.
+  # For advanced tuning of backoff parameters, update the 'wait-exponential-backoff' ConfigMap.
+  enable-wait-exponential-backoff: "false"

config/config-wait-exponential-backoff.yaml

@@ -0,0 +1,33 @@
+# Copyright 2025 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This ConfigMap allows cluster operators to configure the exponential backoff
+# parameters used by Tekton Pipelines when retrying Kubernetes API operations,
+# such as creating TaskRuns or CustomRuns. Adjusting these values can help
+# tune retry behavior in response to webhook timeouts or transient errors.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: wait-exponential-backoff
+  namespace: tekton-pipelines
+  labels:
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/part-of: tekton-pipelines
+data:
+  duration: "10s"         # The initial duration before the first retry (Go duration string, e.g. "1s").
+  factor: "2.0"          # The factor by which the duration increases after each retry (should not be negative).
+  jitter: "0.0"          # Jitter factor (0.0 = no jitter, 0.2 = up to 20% random additional wait).
+  steps: "5"            # The number of times the duration may change (number of backoff steps).
+  cap: "60s"             # The maximum duration between retries (Go duration string, e.g. "30s").

pkg/apis/config/feature_flags.go

@@ -107,6 +107,11 @@ const (
 	EnableKubernetesSidecar = "enable-kubernetes-sidecar"
 	// DefaultEnableKubernetesSidecar is the default value for EnableKubernetesSidecar
 	DefaultEnableKubernetesSidecar = false
+	// EnableWaitExponentialBackoff is the flag to enable exponential backoff strategy
+	EnableWaitExponentialBackoff = "enable-wait-exponential-backoff"
+	// DefaultEnableWaitExponentialBackoff is the default value for EnableWaitExponentialBackoff
+	DefaultEnableWaitExponentialBackoff = false
+
 	// EnableStepActions is the flag to enable step actions (no-op since it's stable)
 	EnableStepActions = "enable-step-actions"
 
@@ -198,12 +203,13 @@ type FeatureFlags struct {
 	Coschedule                               string `json:"coschedule,omitempty"`
 	EnableCELInWhenExpression                bool   `json:"enableCELInWhenExpression,omitempty"`
 	// EnableStepActions is a no-op flag since StepActions are stable
-	EnableStepActions           bool   `json:"enableStepActions,omitempty"`
-	EnableParamEnum             bool   `json:"enableParamEnum,omitempty"`
-	EnableArtifacts             bool   `json:"enableArtifacts,omitempty"`
-	DisableInlineSpec           string `json:"disableInlineSpec,omitempty"`
-	EnableConciseResolverSyntax bool   `json:"enableConciseResolverSyntax,omitempty"`
-	EnableKubernetesSidecar     bool   `json:"enableKubernetesSidecar,omitempty"`
+	EnableStepActions            bool   `json:"enableStepActions,omitempty"`
+	EnableParamEnum              bool   `json:"enableParamEnum,omitempty"`
+	EnableArtifacts              bool   `json:"enableArtifacts,omitempty"`
+	DisableInlineSpec            string `json:"disableInlineSpec,omitempty"`
+	EnableConciseResolverSyntax  bool   `json:"enableConciseResolverSyntax,omitempty"`
+	EnableKubernetesSidecar      bool   `json:"enableKubernetesSidecar,omitempty"`
+	EnableWaitExponentialBackoff bool   `json:"enableWaitExponentialBackoff,omitempty"`
 }
 
 // GetFeatureFlagsConfigName returns the name of the configmap containing all
@@ -308,6 +314,9 @@ func NewFeatureFlagsFromMap(cfgMap map[string]string) (*FeatureFlags, error) {
 	if err := setFeature(EnableKubernetesSidecar, DefaultEnableKubernetesSidecar, &tc.EnableKubernetesSidecar); err != nil {
 		return nil, err
 	}
+	if err := setFeature(EnableWaitExponentialBackoff, DefaultEnableWaitExponentialBackoff, &tc.EnableWaitExponentialBackoff); err != nil {
+		return nil, err
+	}
 
 	return &tc, nil
 }

pkg/apis/config/store.go

@@ -30,12 +30,13 @@ type cfgKey struct{}
 // Config holds the collection of configurations that we attach to contexts.
 // +k8s:deepcopy-gen=false
 type Config struct {
-	Defaults     *Defaults
-	FeatureFlags *FeatureFlags
-	Metrics      *Metrics
-	SpireConfig  *sc.SpireConfig
-	Events       *Events
-	Tracing      *Tracing
+	Defaults               *Defaults
+	FeatureFlags           *FeatureFlags
+	Metrics                *Metrics
+	SpireConfig            *sc.SpireConfig
+	Events                 *Events
+	Tracing                *Tracing
+	WaitExponentialBackoff *WaitExponentialBackoff
 }
 
 // FromContext extracts a Config from the provided context.
@@ -55,12 +56,13 @@ func FromContextOrDefaults(ctx context.Context) *Config {
 	}
 
 	return &Config{
-		Defaults:     DefaultConfig.DeepCopy(),
-		FeatureFlags: DefaultFeatureFlags.DeepCopy(),
-		Metrics:      DefaultMetrics.DeepCopy(),
-		SpireConfig:  DefaultSpire.DeepCopy(),
-		Events:       DefaultEvents.DeepCopy(),
-		Tracing:      DefaultTracing.DeepCopy(),
+		Defaults:               DefaultConfig.DeepCopy(),
+		FeatureFlags:           DefaultFeatureFlags.DeepCopy(),
+		Metrics:                DefaultMetrics.DeepCopy(),
+		SpireConfig:            DefaultSpire.DeepCopy(),
+		Events:                 DefaultEvents.DeepCopy(),
+		Tracing:                DefaultTracing.DeepCopy(),
+		WaitExponentialBackoff: DefaultWaitExponentialBackoff.DeepCopy(),
 	}
 }
 
@@ -83,12 +85,13 @@ func NewStore(logger configmap.Logger, onAfterStore ...func(name string, value i
 			"defaults/features/artifacts",
 			logger,
 			configmap.Constructors{
-				GetDefaultsConfigName():     NewDefaultsFromConfigMap,
-				GetFeatureFlagsConfigName(): NewFeatureFlagsFromConfigMap,
-				GetMetricsConfigName():      NewMetricsFromConfigMap,
-				GetSpireConfigName():        NewSpireConfigFromConfigMap,
-				GetEventsConfigName():       NewEventsFromConfigMap,
-				GetTracingConfigName():      NewTracingFromConfigMap,
+				GetDefaultsConfigName():               NewDefaultsFromConfigMap,
+				GetFeatureFlagsConfigName():           NewFeatureFlagsFromConfigMap,
+				GetMetricsConfigName():                NewMetricsFromConfigMap,
+				GetSpireConfigName():                  NewSpireConfigFromConfigMap,
+				GetEventsConfigName():                 NewEventsFromConfigMap,
+				GetTracingConfigName():                NewTracingFromConfigMap,
+				GetWaitExponentialBackoffConfigName(): NewWaitExponentialBackoffFromConfigMap,
 			},
 			onAfterStore...,
 		),
@@ -129,13 +132,18 @@ func (s *Store) Load() *Config {
 	if events == nil {
 		events = DefaultEvents.DeepCopy()
 	}
+	waitExponentialBackoff := s.UntypedLoad(GetWaitExponentialBackoffConfigName())
+	if waitExponentialBackoff == nil {
+		waitExponentialBackoff = DefaultWaitExponentialBackoff.DeepCopy()
+	}
 
 	return &Config{
-		Defaults:     defaults.(*Defaults).DeepCopy(),
-		FeatureFlags: featureFlags.(*FeatureFlags).DeepCopy(),
-		Metrics:      metrics.(*Metrics).DeepCopy(),
-		Tracing:      tracing.(*Tracing).DeepCopy(),
-		SpireConfig:  spireconfig.(*sc.SpireConfig).DeepCopy(),
-		Events:       events.(*Events).DeepCopy(),
+		Defaults:               defaults.(*Defaults).DeepCopy(),
+		FeatureFlags:           featureFlags.(*FeatureFlags).DeepCopy(),
+		Metrics:                metrics.(*Metrics).DeepCopy(),
+		Tracing:                tracing.(*Tracing).DeepCopy(),
+		SpireConfig:            spireconfig.(*sc.SpireConfig).DeepCopy(),
+		Events:                 events.(*Events).DeepCopy(),
+		WaitExponentialBackoff: waitExponentialBackoff.(*WaitExponentialBackoff).DeepCopy(),
 	}
 }

pkg/apis/config/store_test.go

@@ -33,21 +33,24 @@ func TestStoreLoadWithContext(t *testing.T) {
 	spireConfig := test.ConfigMapFromTestFile(t, "config-spire")
 	eventsConfig := test.ConfigMapFromTestFile(t, "config-events")
 	tracingConfig := test.ConfigMapFromTestFile(t, "config-tracing")
+	waitExponentialBackoffConfig := test.ConfigMapFromTestFile(t, "config-wait-exponential-backoff")
 
 	expectedDefaults, _ := config.NewDefaultsFromConfigMap(defaultConfig)
 	expectedFeatures, _ := config.NewFeatureFlagsFromConfigMap(featuresConfig)
 	metrics, _ := config.NewMetricsFromConfigMap(metricsConfig)
 	expectedSpireConfig, _ := config.NewSpireConfigFromConfigMap(spireConfig)
 	expectedEventsConfig, _ := config.NewEventsFromConfigMap(eventsConfig)
 	expectedTracingConfig, _ := config.NewTracingFromConfigMap(tracingConfig)
+	expectedWaitExponentialBackoffConfig, _ := config.NewWaitExponentialBackoffFromConfigMap(waitExponentialBackoffConfig)
 
 	expected := &config.Config{
-		Defaults:     expectedDefaults,
-		FeatureFlags: expectedFeatures,
-		Metrics:      metrics,
-		SpireConfig:  expectedSpireConfig,
-		Events:       expectedEventsConfig,
-		Tracing:      expectedTracingConfig,
+		Defaults:               expectedDefaults,
+		FeatureFlags:           expectedFeatures,
+		Metrics:                metrics,
+		SpireConfig:            expectedSpireConfig,
+		Events:                 expectedEventsConfig,
+		Tracing:                expectedTracingConfig,
+		WaitExponentialBackoff: expectedWaitExponentialBackoffConfig,
 	}
 
 	store := config.NewStore(logtesting.TestLogger(t))
@@ -57,6 +60,7 @@ func TestStoreLoadWithContext(t *testing.T) {
 	store.OnConfigChanged(spireConfig)
 	store.OnConfigChanged(eventsConfig)
 	store.OnConfigChanged(tracingConfig)
+	store.OnConfigChanged(waitExponentialBackoffConfig)
 
 	cfg := config.FromContext(store.ToContext(t.Context()))
 
@@ -67,12 +71,13 @@ func TestStoreLoadWithContext(t *testing.T) {
 
 func TestStoreLoadWithContext_Empty(t *testing.T) {
 	want := &config.Config{
-		Defaults:     config.DefaultConfig.DeepCopy(),
-		FeatureFlags: config.DefaultFeatureFlags.DeepCopy(),
-		Metrics:      config.DefaultMetrics.DeepCopy(),
-		SpireConfig:  config.DefaultSpire.DeepCopy(),
-		Events:       config.DefaultEvents.DeepCopy(),
-		Tracing:      config.DefaultTracing.DeepCopy(),
+		Defaults:               config.DefaultConfig.DeepCopy(),
+		FeatureFlags:           config.DefaultFeatureFlags.DeepCopy(),
+		Metrics:                config.DefaultMetrics.DeepCopy(),
+		SpireConfig:            config.DefaultSpire.DeepCopy(),
+		Events:                 config.DefaultEvents.DeepCopy(),
+		Tracing:                config.DefaultTracing.DeepCopy(),
+		WaitExponentialBackoff: config.DefaultWaitExponentialBackoff.DeepCopy(),
 	}
 
 	store := config.NewStore(logtesting.TestLogger(t))

pkg/apis/config/testdata/config-wait-exponential-backoff-custom.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-wait-exponential-backoff-custom
+data:
+  duration: "5s"
+  factor: "3.5"
+  jitter: "0.2"
+  steps: "7"
+  cap: "120s"

pkg/apis/config/testdata/config-wait-exponential-backoff-empty.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-wait-exponential-backoff-empty
+data: {}

pkg/apis/config/testdata/config-wait-exponential-backoff.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-wait-exponential-backoff
+data:
+  duration: "1s"
+  factor: "2.0"
+  jitter: "0.0"
+  steps: "10"
+  cap: "30s"