implement exponential backoff

The createTaskRun and createCustomRun now uses wait.ExponentialBackoff to retry
the creation of a taskRun or customRun when certain errors occur, specifically
webhook timeouts.

The function isWebhookTimeout checks if an error is a mutating adminssion
webhook timeout, by looking for HTTP 500 and the phrase "timeout" in the error
message.

If a webhook timeout is detected, the backoff loop will retry the creation
up to a configured number of steps, with increasing delay between attempts.

if the error is not a webhook timeout, the function will not retry and will
return the error immediately.

Errors that not webhook timeouts, e.g. HTTP 400 bad request, validation errors,
etc. are not retried and will cause the taskRun creation to fail as expected.

By default, the exponential backoff strategy is disabled. To enable this
feature, set the `enable-wait-exponential-backoff` to `true` in
feature-flags config map.

When enabled, the controller will use an exponential backoff strategy to retry
taskRun and customRun creation if it encounters transient errors such as
admission webhook timeouts.

This improves robustness against temporary webhook issues. If the feature flag
is set to false, the controller will not retry and will fail immediately on
such errors.

Configuration for the backoff parameters (duration, factor, steps, etc) can be
set in the wait-exponential-backoff config map.

Signed-off-by: Priti Desai <[email protected]>

Author: pritidesai

config/300-crds/300-pipelinerun.yaml

@@ -2200,6 +2200,8 @@ spec:
                         enableStepActions:
                           description: EnableStepActions is a no-op flag since StepActions are stable
                           type: boolean
+                        enableWaitExponentialBackoff:
+                          type: boolean
                         enforceNonfalsifiability:
                           type: string
                         maxResultSize:
@@ -2613,6 +2615,8 @@ spec:
                                   enableStepActions:
                                     description: EnableStepActions is a no-op flag since StepActions are stable
                                     type: boolean
+                                  enableWaitExponentialBackoff:
+                                    type: boolean
                                   enforceNonfalsifiability:
                                     type: string
                                   maxResultSize:
@@ -2900,6 +2904,8 @@ spec:
                                         enableStepActions:
                                           description: EnableStepActions is a no-op flag since StepActions are stable
                                           type: boolean
+                                        enableWaitExponentialBackoff:
+                                          type: boolean
                                         enforceNonfalsifiability:
                                           type: string
                                         maxResultSize:
@@ -5120,6 +5126,8 @@ spec:
                         enableStepActions:
                           description: EnableStepActions is a no-op flag since StepActions are stable
                           type: boolean
+                        enableWaitExponentialBackoff:
+                          type: boolean
                         enforceNonfalsifiability:
                           type: string
                         maxResultSize:

config/300-crds/300-taskrun.yaml

@@ -1776,6 +1776,8 @@ spec:
                         enableStepActions:
                           description: EnableStepActions is a no-op flag since StepActions are stable
                           type: boolean
+                        enableWaitExponentialBackoff:
+                          type: boolean
                         enforceNonfalsifiability:
                           type: string
                         maxResultSize:
@@ -2063,6 +2065,8 @@ spec:
                               enableStepActions:
                                 description: EnableStepActions is a no-op flag since StepActions are stable
                                 type: boolean
+                              enableWaitExponentialBackoff:
+                                type: boolean
                               enforceNonfalsifiability:
                                 type: string
                               maxResultSize:
@@ -3767,6 +3771,8 @@ spec:
                         enableStepActions:
                           description: EnableStepActions is a no-op flag since StepActions are stable
                           type: boolean
+                        enableWaitExponentialBackoff:
+                          type: boolean
                         enforceNonfalsifiability:
                           type: string
                         maxResultSize:
@@ -4020,6 +4026,8 @@ spec:
                               enableStepActions:
                                 description: EnableStepActions is a no-op flag since StepActions are stable
                                 type: boolean
+                              enableWaitExponentialBackoff:
+                                type: boolean
                               enforceNonfalsifiability:
                                 type: string
                               maxResultSize:

config/config-feature-flags.yaml

@@ -130,3 +130,9 @@ data:
   enable-kubernetes-sidecar: "false"
   # Setting this flag to "false" will have no effect since StepActions are a stable feature
   enable-step-actions: "true"
+  # Controls whether exponential backoff is enabled when creating TaskRuns or CustomRuns.
+  # If set to "true", the controller will use exponential backoff when retrying failed create operations,
+  # which can help mitigate issues caused by temporary API server or webhook unavailability.
+  # If set to "false", exponential backoff will be disabled.
+  # For advanced tuning of backoff parameters, update the 'wait-exponential-backoff' ConfigMap.
+  enable-wait-exponential-backoff: "false"

config/config-wait-exponential-backoff.yaml

@@ -0,0 +1,33 @@
+# Copyright 2025 The Tekton Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+# This ConfigMap allows cluster operators to configure the exponential backoff
+# parameters used by Tekton Pipelines when retrying Kubernetes API operations,
+# such as creating TaskRuns or CustomRuns. Adjusting these values can help
+# tune retry behavior in response to webhook timeouts or transient errors.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: wait-exponential-backoff
+  namespace: tekton-pipelines
+  labels:
+    app.kubernetes.io/instance: default
+    app.kubernetes.io/part-of: tekton-pipelines
+data:
+  duration: "10s"         # The initial duration before the first retry (Go duration string, e.g. "1s").
+  factor: "2.0"          # The factor by which the duration increases after each retry (should not be negative).
+  jitter: "0.0"          # Jitter factor (0.0 = no jitter, 0.2 = up to 20% random additional wait).
+  steps: "5"            # The number of times the duration may change (number of backoff steps).
+  cap: "60s"             # The maximum duration between retries (Go duration string, e.g. "30s").

docs/additional-configs.md

@@ -33,6 +33,7 @@ installation.
   - [Pipelinerun with Affinity Assistant](#pipelineruns-with-affinity-assistant)
   - [TaskRuns with `imagePullBackOff` Timeout](#taskruns-with-imagepullbackoff-timeout)
   - [Disabling Inline Spec in TaskRun and PipelineRun](#disabling-inline-spec-in-taskrun-and-pipelinerun)
+  - [Exponential Backoff for TaskRun and CustomRun Creation](#exponential-backoff-for-taskrun-and-customrun-creation)
   - [Next steps](#next-steps)
 
 
@@ -711,6 +712,59 @@ Inline specifications can be disabled for specific resources only. To achieve th
 
 The default value of disable-inline-spec is "", which means inline specification is enabled in all cases.
 
+## Exponential Backoff for TaskRun and CustomRun Creation
+
+By default, when Tekton Pipelines attempts to create a TaskRun or CustomRun resource and encounters an error, the controller will requeue the PipelineRun for another reconciliation attempt after a short delay. However, this may not be sufficient in a heavily loaded or busy cluster, where transient errors such as webhook timeouts or network issues are more likely to occur.
+
+To improve robustness in environments where webhook timeouts or network errors are possible, you can enable an **exponential backoff retry strategy** for TaskRun and CustomRun creation by setting the `enable-wait-exponential-backoff` feature flag to `"true"` in the `feature-flags` ConfigMap:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: feature-flags
+  namespace: tekton-pipelines
+data:
+  enable-wait-exponential-backoff: "true"
+```
+
+When this flag is enabled, the controller will retry TaskRun and CustomRun creation using an exponential backoff strategy if it encounters admission webhook timeouts (e.g., `"timeout"`).
+
+This helps mitigate failures due to temporary unavailability of webhooks or network disruptions.
+
+### Backoff Configuration
+
+You can further customize the backoff parameters (such as initial duration, factor, steps, and cap) using the `wait-exponential-backoff` ConfigMap:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: wait-exponential-backoff
+  namespace: tekton-pipelines
+data:
+  duration: "1s"
+  factor: "2.0"
+  jitter: "0.0"
+  steps: "10"
+  cap: "30s"
+```
+
+- **duration**: Initial wait time before the first retry.
+- **factor**: Multiplier for each subsequent retry interval.
+- **steps**: Maximum number of retry attempts.
+- **cap**: Maximum wait time between retries.
+
+### Default Behavior
+
+If `enable-wait-exponential-backoff` is not set or is set to `"false"`, the controller will rely on its standard reconcilation loop to retry TaskRun or CustomRun creation after a short delay when failures occur due to webhook or network errors.
+
+---
+
+**Note:** This feature is especially useful in clusters where webhook services (such as Kyverno, OPA, or custom admission controllers) may be temporarily unavailable or slow to respond.
+
+---
+
 ## Next steps
 
 To get started with Tekton check the [Introductory tutorials][quickstarts],

pkg/apis/config/feature_flags.go

@@ -107,6 +107,11 @@ const (
 	EnableKubernetesSidecar = "enable-kubernetes-sidecar"
 	// DefaultEnableKubernetesSidecar is the default value for EnableKubernetesSidecar
 	DefaultEnableKubernetesSidecar = false
+	// EnableWaitExponentialBackoff is the flag to enable exponential backoff strategy
+	EnableWaitExponentialBackoff = "enable-wait-exponential-backoff"
+	// DefaultEnableWaitExponentialBackoff is the default value for EnableWaitExponentialBackoff
+	DefaultEnableWaitExponentialBackoff = false
+
 	// EnableStepActions is the flag to enable step actions (no-op since it's stable)
 	EnableStepActions = "enable-step-actions"
 
@@ -198,12 +203,13 @@ type FeatureFlags struct {
 	Coschedule                               string `json:"coschedule,omitempty"`
 	EnableCELInWhenExpression                bool   `json:"enableCELInWhenExpression,omitempty"`
 	// EnableStepActions is a no-op flag since StepActions are stable
-	EnableStepActions           bool   `json:"enableStepActions,omitempty"`
-	EnableParamEnum             bool   `json:"enableParamEnum,omitempty"`
-	EnableArtifacts             bool   `json:"enableArtifacts,omitempty"`
-	DisableInlineSpec           string `json:"disableInlineSpec,omitempty"`
-	EnableConciseResolverSyntax bool   `json:"enableConciseResolverSyntax,omitempty"`
-	EnableKubernetesSidecar     bool   `json:"enableKubernetesSidecar,omitempty"`
+	EnableStepActions            bool   `json:"enableStepActions,omitempty"`
+	EnableParamEnum              bool   `json:"enableParamEnum,omitempty"`
+	EnableArtifacts              bool   `json:"enableArtifacts,omitempty"`
+	DisableInlineSpec            string `json:"disableInlineSpec,omitempty"`
+	EnableConciseResolverSyntax  bool   `json:"enableConciseResolverSyntax,omitempty"`
+	EnableKubernetesSidecar      bool   `json:"enableKubernetesSidecar,omitempty"`
+	EnableWaitExponentialBackoff bool   `json:"enableWaitExponentialBackoff,omitempty"`
 }
 
 // GetFeatureFlagsConfigName returns the name of the configmap containing all
@@ -308,6 +314,9 @@ func NewFeatureFlagsFromMap(cfgMap map[string]string) (*FeatureFlags, error) {
 	if err := setFeature(EnableKubernetesSidecar, DefaultEnableKubernetesSidecar, &tc.EnableKubernetesSidecar); err != nil {
 		return nil, err
 	}
+	if err := setFeature(EnableWaitExponentialBackoff, DefaultEnableWaitExponentialBackoff, &tc.EnableWaitExponentialBackoff); err != nil {
+		return nil, err
+	}
 
 	return &tc, nil
 }

pkg/apis/config/store.go

@@ -30,12 +30,13 @@ type cfgKey struct{}
 // Config holds the collection of configurations that we attach to contexts.
 // +k8s:deepcopy-gen=false
 type Config struct {
-	Defaults     *Defaults
-	FeatureFlags *FeatureFlags
-	Metrics      *Metrics
-	SpireConfig  *sc.SpireConfig
-	Events       *Events
-	Tracing      *Tracing
+	Defaults               *Defaults
+	FeatureFlags           *FeatureFlags
+	Metrics                *Metrics
+	SpireConfig            *sc.SpireConfig
+	Events                 *Events
+	Tracing                *Tracing
+	WaitExponentialBackoff *WaitExponentialBackoff
 }
 
 // FromContext extracts a Config from the provided context.
@@ -55,12 +56,13 @@ func FromContextOrDefaults(ctx context.Context) *Config {
 	}
 
 	return &Config{
-		Defaults:     DefaultConfig.DeepCopy(),
-		FeatureFlags: DefaultFeatureFlags.DeepCopy(),
-		Metrics:      DefaultMetrics.DeepCopy(),
-		SpireConfig:  DefaultSpire.DeepCopy(),
-		Events:       DefaultEvents.DeepCopy(),
-		Tracing:      DefaultTracing.DeepCopy(),
+		Defaults:               DefaultConfig.DeepCopy(),
+		FeatureFlags:           DefaultFeatureFlags.DeepCopy(),
+		Metrics:                DefaultMetrics.DeepCopy(),
+		SpireConfig:            DefaultSpire.DeepCopy(),
+		Events:                 DefaultEvents.DeepCopy(),
+		Tracing:                DefaultTracing.DeepCopy(),
+		WaitExponentialBackoff: DefaultWaitExponentialBackoff.DeepCopy(),
 	}
 }
 
@@ -83,12 +85,13 @@ func NewStore(logger configmap.Logger, onAfterStore ...func(name string, value i
 			"defaults/features/artifacts",
 			logger,
 			configmap.Constructors{
-				GetDefaultsConfigName():     NewDefaultsFromConfigMap,
-				GetFeatureFlagsConfigName(): NewFeatureFlagsFromConfigMap,
-				GetMetricsConfigName():      NewMetricsFromConfigMap,
-				GetSpireConfigName():        NewSpireConfigFromConfigMap,
-				GetEventsConfigName():       NewEventsFromConfigMap,
-				GetTracingConfigName():      NewTracingFromConfigMap,
+				GetDefaultsConfigName():               NewDefaultsFromConfigMap,
+				GetFeatureFlagsConfigName():           NewFeatureFlagsFromConfigMap,
+				GetMetricsConfigName():                NewMetricsFromConfigMap,
+				GetSpireConfigName():                  NewSpireConfigFromConfigMap,
+				GetEventsConfigName():                 NewEventsFromConfigMap,
+				GetTracingConfigName():                NewTracingFromConfigMap,
+				GetWaitExponentialBackoffConfigName(): NewWaitExponentialBackoffFromConfigMap,
 			},
 			onAfterStore...,
 		),
@@ -129,13 +132,18 @@ func (s *Store) Load() *Config {
 	if events == nil {
 		events = DefaultEvents.DeepCopy()
 	}
+	waitExponentialBackoff := s.UntypedLoad(GetWaitExponentialBackoffConfigName())
+	if waitExponentialBackoff == nil {
+		waitExponentialBackoff = DefaultWaitExponentialBackoff.DeepCopy()
+	}
 
 	return &Config{
-		Defaults:     defaults.(*Defaults).DeepCopy(),
-		FeatureFlags: featureFlags.(*FeatureFlags).DeepCopy(),
-		Metrics:      metrics.(*Metrics).DeepCopy(),
-		Tracing:      tracing.(*Tracing).DeepCopy(),
-		SpireConfig:  spireconfig.(*sc.SpireConfig).DeepCopy(),
-		Events:       events.(*Events).DeepCopy(),
+		Defaults:               defaults.(*Defaults).DeepCopy(),
+		FeatureFlags:           featureFlags.(*FeatureFlags).DeepCopy(),
+		Metrics:                metrics.(*Metrics).DeepCopy(),
+		Tracing:                tracing.(*Tracing).DeepCopy(),
+		SpireConfig:            spireconfig.(*sc.SpireConfig).DeepCopy(),
+		Events:                 events.(*Events).DeepCopy(),
+		WaitExponentialBackoff: waitExponentialBackoff.(*WaitExponentialBackoff).DeepCopy(),
 	}
 }

pkg/apis/config/store_test.go

@@ -33,21 +33,24 @@ func TestStoreLoadWithContext(t *testing.T) {
 	spireConfig := test.ConfigMapFromTestFile(t, "config-spire")
 	eventsConfig := test.ConfigMapFromTestFile(t, "config-events")
 	tracingConfig := test.ConfigMapFromTestFile(t, "config-tracing")
+	waitExponentialBackoffConfig := test.ConfigMapFromTestFile(t, "config-wait-exponential-backoff")
 
 	expectedDefaults, _ := config.NewDefaultsFromConfigMap(defaultConfig)
 	expectedFeatures, _ := config.NewFeatureFlagsFromConfigMap(featuresConfig)
 	metrics, _ := config.NewMetricsFromConfigMap(metricsConfig)
 	expectedSpireConfig, _ := config.NewSpireConfigFromConfigMap(spireConfig)
 	expectedEventsConfig, _ := config.NewEventsFromConfigMap(eventsConfig)
 	expectedTracingConfig, _ := config.NewTracingFromConfigMap(tracingConfig)
+	expectedWaitExponentialBackoffConfig, _ := config.NewWaitExponentialBackoffFromConfigMap(waitExponentialBackoffConfig)
 
 	expected := &config.Config{
-		Defaults:     expectedDefaults,
-		FeatureFlags: expectedFeatures,
-		Metrics:      metrics,
-		SpireConfig:  expectedSpireConfig,
-		Events:       expectedEventsConfig,
-		Tracing:      expectedTracingConfig,
+		Defaults:               expectedDefaults,
+		FeatureFlags:           expectedFeatures,
+		Metrics:                metrics,
+		SpireConfig:            expectedSpireConfig,
+		Events:                 expectedEventsConfig,
+		Tracing:                expectedTracingConfig,
+		WaitExponentialBackoff: expectedWaitExponentialBackoffConfig,
 	}
 
 	store := config.NewStore(logtesting.TestLogger(t))
@@ -57,6 +60,7 @@ func TestStoreLoadWithContext(t *testing.T) {
 	store.OnConfigChanged(spireConfig)
 	store.OnConfigChanged(eventsConfig)
 	store.OnConfigChanged(tracingConfig)
+	store.OnConfigChanged(waitExponentialBackoffConfig)
 
 	cfg := config.FromContext(store.ToContext(t.Context()))
 
@@ -67,12 +71,13 @@ func TestStoreLoadWithContext(t *testing.T) {
 
 func TestStoreLoadWithContext_Empty(t *testing.T) {
 	want := &config.Config{
-		Defaults:     config.DefaultConfig.DeepCopy(),
-		FeatureFlags: config.DefaultFeatureFlags.DeepCopy(),
-		Metrics:      config.DefaultMetrics.DeepCopy(),
-		SpireConfig:  config.DefaultSpire.DeepCopy(),
-		Events:       config.DefaultEvents.DeepCopy(),
-		Tracing:      config.DefaultTracing.DeepCopy(),
+		Defaults:               config.DefaultConfig.DeepCopy(),
+		FeatureFlags:           config.DefaultFeatureFlags.DeepCopy(),
+		Metrics:                config.DefaultMetrics.DeepCopy(),
+		SpireConfig:            config.DefaultSpire.DeepCopy(),
+		Events:                 config.DefaultEvents.DeepCopy(),
+		Tracing:                config.DefaultTracing.DeepCopy(),
+		WaitExponentialBackoff: config.DefaultWaitExponentialBackoff.DeepCopy(),
 	}
 
 	store := config.NewStore(logtesting.TestLogger(t))

pkg/apis/config/testdata/config-wait-exponential-backoff-custom.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-wait-exponential-backoff-custom
+data:
+  duration: "5s"
+  factor: "3.5"
+  jitter: "0.2"
+  steps: "7"
+  cap: "120s"

pkg/apis/config/testdata/config-wait-exponential-backoff-empty.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: config-wait-exponential-backoff-empty
+data: {}