Merge pull request #2275 from beyonnex-io/add-span-metrics-reporting-configuration

thjaeckle · web-flow · commit 153b8f2900c6 · 2025-11-20T15:30:37.000+01:00
add feature toggle for configuring trace span metric reporting
diff --git a/base/model/src/main/java/org/eclipse/ditto/base/model/signals/FeatureToggle.java b/base/model/src/main/java/org/eclipse/ditto/base/model/signals/FeatureToggle.java
@@ -54,6 +54,15 @@ public final class FeatureToggle {
      */
     public static final String JSON_KEY_VALIDATION_ENABLED = "ditto.devops.feature.json-key-validation-enabled";
 
+    /**
+     * System property name of the property defining whether when tracing is activated span metrics are also recorded
+     * and exposed as Prometheus metrics. This is a built-in default of Kamon, Ditto's used tracing library, but can
+     * be deactivated to reduce the amount of recorded metrics.
+     *
+     * @since 3.8.7
+     */
+    public static final String TRACING_SPAN_METRICS_ENABLED = "ditto.devops.feature.tracing-span-metrics-enabled";
+
     /**
      * Resolves the system property {@value MERGE_THINGS_ENABLED}.
      */
@@ -79,6 +88,11 @@ public final class FeatureToggle {
      */
     private static final boolean IS_JSON_KEY_VALIDATION_ENABLED = resolveProperty(JSON_KEY_VALIDATION_ENABLED);
 
+    /**
+     * Resolves the system property {@value TRACING_SPAN_METRICS_ENABLED}.
+     */
+    private static final boolean IS_TRACING_SPAN_METRICS_ENABLED = resolveProperty(TRACING_SPAN_METRICS_ENABLED);
+
     private static boolean resolveProperty(final String propertyName) {
         final String propertyValue = System.getProperty(propertyName, Boolean.TRUE.toString());
         return !Boolean.FALSE.toString().equalsIgnoreCase(propertyValue);
@@ -183,10 +197,21 @@ public static boolean isPreserveKnownMqttHeadersFeatureEnabled() {
     /**
      * Returns whether JSON key validation is enabled based on the system property {@value JSON_KEY_VALIDATION_ENABLED}.
      *
-     * @return whether JSON key validation is enabled based on the system property
+     * @return whether JSON key validation is enabled.
      * @since 3.7.5
      */
     public static boolean isJsonKeyValidationEnabled() {
         return IS_JSON_KEY_VALIDATION_ENABLED;
     }
+
+    /**
+     * Returns whether tracing span metric reporting is enabled based on the system property
+     * {@value TRACING_SPAN_METRICS_ENABLED}.
+     *
+     * @return whether tracing span metric reporting is enabled.
+     * @since 3.8.7
+     */
+    public static boolean isTracingSpanMetricsEnabled() {
+        return IS_TRACING_SPAN_METRICS_ENABLED;
+    }
 }
diff --git a/base/service/src/main/java/org/eclipse/ditto/base/service/DittoService.java b/base/service/src/main/java/org/eclipse/ditto/base/service/DittoService.java
@@ -421,6 +421,8 @@ private void injectSystemPropertiesLimits(final C serviceSpecificConfig) {
                 Boolean.toString(rawConfig.getBoolean(FeatureToggle.PRESERVE_KNOWN_MQTT_HEADERS_ENABLED)));
         System.setProperty(FeatureToggle.JSON_KEY_VALIDATION_ENABLED,
                 Boolean.toString(rawConfig.getBoolean(FeatureToggle.JSON_KEY_VALIDATION_ENABLED)));
+        System.setProperty(FeatureToggle.TRACING_SPAN_METRICS_ENABLED,
+                Boolean.toString(rawConfig.getBoolean(FeatureToggle.TRACING_SPAN_METRICS_ENABLED)));
         System.setProperty(DittoSystemProperties.DITTO_LIMITS_POLICY_IMPORTS_LIMIT,
                 Integer.toString(limitsConfig.getPolicyImportsLimit()));
         final MetricsConfig metricsConfig = serviceSpecificConfig.getMetricsConfig();
diff --git a/deployment/helm/ditto/Chart.yaml b/deployment/helm/ditto/Chart.yaml
@@ -16,7 +16,7 @@ description: |
   A digital twin is a virtual, cloud based, representation of his real world counterpart
   (real world “Things”, e.g. devices like sensors, smart heating, connected cars, smart grids, EV charging stations etc).
 type: application
-version: 3.8.7  # chart version is effectively set by release-job
+version: 3.8.8  # chart version is effectively set by release-job
 appVersion: 3.8.6
 keywords:
   - iot-chart
diff --git a/deployment/helm/ditto/templates/connectivity-deployment.yaml b/deployment/helm/ditto/templates/connectivity-deployment.yaml
@@ -216,6 +216,8 @@ spec:
               value: "{{ .Values.global.featureFlags.preserveKnownMqttHeadersEnabled }}"
             - name: DITTO_DEVOPS_FEATURE_JSON_KEY_VALIDATION_ENABLED
               value: "{{ .Values.global.featureFlags.jsonKeyValidationEnabled }}"
+            - name: DITTO_DEVOPS_FEATURE_TRACING_SPAN_METRICS_ENABLED
+              value: "{{ .Values.global.featureFlags.tracingSpanMetricsEnabled }}"
             - name: PEKKO_COORDINATED_SHUTDOWN_PHASES_CLUSTER_SHARDING_SHUTDOWN_REGION
               value: "{{ .Values.connectivity.config.cluster.coordinatedShutdown.phases.clusterShardingShutdownRegion.timeout }}"
             - name: PEKKO_COORDINATED_SHUTDOWN_PHASES_CLUSTER_EXITING_TIMEOUT
diff --git a/deployment/helm/ditto/templates/gateway-deployment.yaml b/deployment/helm/ditto/templates/gateway-deployment.yaml
@@ -201,6 +201,8 @@ spec:
               value: "{{ .Values.global.featureFlags.historicalApisEnabled }}"
             - name: DITTO_DEVOPS_FEATURE_JSON_KEY_VALIDATION_ENABLED
               value: "{{ .Values.global.featureFlags.jsonKeyValidationEnabled }}"
+            - name: DITTO_DEVOPS_FEATURE_TRACING_SPAN_METRICS_ENABLED
+              value: "{{ .Values.global.featureFlags.tracingSpanMetricsEnabled }}"
             - name: PEKKO_COORDINATED_SHUTDOWN_PHASES_CLUSTER_SHARDING_SHUTDOWN_REGION
               value: "{{ .Values.gateway.config.cluster.coordinatedShutdown.phases.clusterShardingShutdownRegion.timeout }}"
             - name: PEKKO_COORDINATED_SHUTDOWN_PHASES_CLUSTER_EXITING_TIMEOUT
diff --git a/deployment/helm/ditto/templates/policies-deployment.yaml b/deployment/helm/ditto/templates/policies-deployment.yaml
@@ -210,6 +210,8 @@ spec:
               value: "{{ .Values.global.featureFlags.historicalApisEnabled }}"
             - name: DITTO_DEVOPS_FEATURE_JSON_KEY_VALIDATION_ENABLED
               value: "{{ .Values.global.featureFlags.jsonKeyValidationEnabled }}"
+            - name: DITTO_DEVOPS_FEATURE_TRACING_SPAN_METRICS_ENABLED
+              value: "{{ .Values.global.featureFlags.tracingSpanMetricsEnabled }}"
             - name: PEKKO_COORDINATED_SHUTDOWN_PHASES_CLUSTER_SHARDING_SHUTDOWN_REGION
               value: "{{ .Values.policies.config.cluster.coordinatedShutdown.phases.clusterShardingShutdownRegion.timeout }}"
             - name: PEKKO_COORDINATED_SHUTDOWN_PHASES_CLUSTER_EXITING_TIMEOUT
diff --git a/deployment/helm/ditto/templates/things-deployment.yaml b/deployment/helm/ditto/templates/things-deployment.yaml
@@ -214,6 +214,8 @@ spec:
               value: "{{ .Values.global.featureFlags.historicalApisEnabled }}"
             - name: DITTO_DEVOPS_FEATURE_JSON_KEY_VALIDATION_ENABLED
               value: "{{ .Values.global.featureFlags.jsonKeyValidationEnabled }}"
+            - name: DITTO_DEVOPS_FEATURE_TRACING_SPAN_METRICS_ENABLED
+              value: "{{ .Values.global.featureFlags.tracingSpanMetricsEnabled }}"
             - name: PEKKO_COORDINATED_SHUTDOWN_PHASES_CLUSTER_SHARDING_SHUTDOWN_REGION
               value: "{{ .Values.things.config.cluster.coordinatedShutdown.phases.clusterShardingShutdownRegion.timeout }}"
             - name: PEKKO_COORDINATED_SHUTDOWN_PHASES_CLUSTER_EXITING_TIMEOUT
diff --git a/deployment/helm/ditto/templates/thingssearch-deployment.yaml b/deployment/helm/ditto/templates/thingssearch-deployment.yaml
@@ -210,6 +210,8 @@ spec:
               value: "{{ .Values.global.limits.maxAuthSubjectsCount }}"
             - name: DITTO_DEVOPS_FEATURE_JSON_KEY_VALIDATION_ENABLED
               value: "{{ .Values.global.featureFlags.jsonKeyValidationEnabled }}"
+            - name: DITTO_DEVOPS_FEATURE_TRACING_SPAN_METRICS_ENABLED
+              value: "{{ .Values.global.featureFlags.tracingSpanMetricsEnabled }}"
             - name: PEKKO_COORDINATED_SHUTDOWN_PHASES_CLUSTER_SHARDING_SHUTDOWN_REGION
               value: "{{ .Values.thingsSearch.config.cluster.coordinatedShutdown.phases.clusterShardingShutdownRegion.timeout }}"
             - name: PEKKO_COORDINATED_SHUTDOWN_PHASES_CLUSTER_EXITING_TIMEOUT
diff --git a/deployment/helm/ditto/values.yaml b/deployment/helm/ditto/values.yaml
@@ -155,6 +155,10 @@ global:
     # jsonKeyValidationEnabled controls whether for each json object, each JSON key should be validated against a regex pattern
     #  excluding control characters to be used as JSON keys. This can have quite a performance impact.
     jsonKeyValidationEnabled: true
+    # tracingSpanMetricsEnabled controls whether when tracing is enabled, spans should automatically create metrics for span durations
+    #  reported as Prometheus histogram "span_processing_time_seconds".
+    #  As Ditto tracks a lot of spans, enabling those metrics can cause a lot of additional metrics and overhead.
+    tracingSpanMetricsEnabled: true
   # logging the logging configuration for Ditto
   logging:
     # sysout holds the logging to SYSOUT config
diff --git a/internal/utils/config/src/main/resources/ditto-devops.conf b/internal/utils/config/src/main/resources/ditto-devops.conf
@@ -28,5 +28,12 @@ ditto.devops {
     // enables/disables the validation of json keys in JsonKeyValidator
     json-key-validation-enabled = true
     json-key-validation-enabled = ${?DITTO_DEVOPS_FEATURE_JSON_KEY_VALIDATION_ENABLED}
+
+    // enables/disables the reporting of span metrics by Kamon tracing
+    //  When tracing is activated, span metrics are also recorded and exposed as Prometheus metrics.
+    //  This is a built-in default of Kamon, Ditto's used tracing library, but can
+    //  be deactivated to reduce the amount of recorded metrics.
+    tracing-span-metrics-enabled = true
+    tracing-span-metrics-enabled = ${?DITTO_DEVOPS_FEATURE_TRACING_SPAN_METRICS_ENABLED}
   }
 }
diff --git a/internal/utils/tracing/src/main/java/org/eclipse/ditto/internal/utils/tracing/span/PreparedKamonSpan.java b/internal/utils/tracing/src/main/java/org/eclipse/ditto/internal/utils/tracing/span/PreparedKamonSpan.java
@@ -19,6 +19,7 @@
 import javax.annotation.concurrent.NotThreadSafe;
 
 import org.eclipse.ditto.base.model.headers.DittoHeaderDefinition;
+import org.eclipse.ditto.base.model.signals.FeatureToggle;
 import org.eclipse.ditto.internal.utils.metrics.instruments.tag.KamonTagSetConverter;
 import org.eclipse.ditto.internal.utils.metrics.instruments.tag.Tag;
 import org.eclipse.ditto.internal.utils.metrics.instruments.tag.TagSet;
@@ -45,6 +46,11 @@ private PreparedKamonSpan(
     ) {
         final var context = httpContextPropagation.getContextFromHeaders(headers);
         spanBuilder = Kamon.spanBuilder(operationName.toString()).asChildOf(context.get(Span.Key()));
+        if (FeatureToggle.isTracingSpanMetricsEnabled()) {
+            spanBuilder.trackMetrics();
+        } else {
+            spanBuilder.doNotTrackMetrics();
+        }
         this.httpContextPropagation = httpContextPropagation;
     }
 

Original file line number	Diff line number	Diff line change
`@@ -28,5 +28,12 @@ ditto.devops {`
`28`	`28`	`// enables/disables the validation of json keys in JsonKeyValidator`
`29`	`29`	`json-key-validation-enabled = true`
`30`	`30`	`json-key-validation-enabled = ${?DITTO_DEVOPS_FEATURE_JSON_KEY_VALIDATION_ENABLED}`
	`31`	`+`
	`32`	`+ // enables/disables the reporting of span metrics by Kamon tracing`
	`33`	`+ // When tracing is activated, span metrics are also recorded and exposed as Prometheus metrics.`
	`34`	`+ // This is a built-in default of Kamon, Ditto's used tracing library, but can`
	`35`	`+ // be deactivated to reduce the amount of recorded metrics.`
	`36`	`+ tracing-span-metrics-enabled = true`
	`37`	`+ tracing-span-metrics-enabled = ${?DITTO_DEVOPS_FEATURE_TRACING_SPAN_METRICS_ENABLED}`
`31`	`38`	`}`
`32`	`39`	`}`