Allow changing the severity of resilience events (#2072)

Author: martintmk

docs/advanced/telemetry.md

@@ -255,3 +255,31 @@ Each resilience strategy can generate telemetry data through the [`ResilienceStr
 To leverage this telemetry data, users should assign a `TelemetryListener` instance to `ResiliencePipelineBuilder.TelemetryListener` and then consume the `TelemetryEventArguments`.
 
 For common scenarios, it is expected that users would make use of `Polly.Extensions`. This extension enables telemetry configuration through the `ResiliencePipelineBuilder.ConfigureTelemetry(...)` method, which processes `TelemetryEventArguments` to generate logs and metrics.
+
+## Customizing the severity of telemetry events
+
+To customize the severity of telemetry events, set the [`SeverityProvider`](xref:Polly.Telemetry.TelemetryOptions.SeverityProvider) delegate that allows changing the default severity of resilience events:
+
+<!-- snippet: telemetry-severity-override -->
+```cs
+services.AddResiliencePipeline("my-strategy", (builder, context) =>
+{
+    // Create a new instance of telemetry options by using copy-constructor of the global ones.
+    // This ensures that common configuration is preserved.
+    var telemetryOptions = new TelemetryOptions(context.GetOptions<TelemetryOptions>());
+
+    telemetryOptions.SeverityProvider = args => args.Event.EventName switch
+    {
+        // Decrease severity of specific events
+        "OnRetry" => ResilienceEventSeverity.Debug,
+        "ExecutionAttempt" => ResilienceEventSeverity.Debug,
+        _ => args.Event.Severity
+    };
+
+    builder.AddRetry(new RetryStrategyOptions());
+
+    // Override the telemetry configuration for this pipeline.
+    builder.ConfigureTelemetry(telemetryOptions);
+});
+```
+<!-- endSnippet -->

src/Polly.Extensions/PublicAPI.Unshipped.txt

@@ -1,4 +1,13 @@
 #nullable enable
+Polly.Telemetry.SeverityProviderArguments
+Polly.Telemetry.SeverityProviderArguments.Context.get -> Polly.ResilienceContext!
+Polly.Telemetry.SeverityProviderArguments.Event.get -> Polly.Telemetry.ResilienceEvent
+Polly.Telemetry.SeverityProviderArguments.SeverityProviderArguments() -> void
+Polly.Telemetry.SeverityProviderArguments.SeverityProviderArguments(Polly.Telemetry.ResilienceTelemetrySource! source, Polly.Telemetry.ResilienceEvent resilienceEvent, Polly.ResilienceContext! context) -> void
+Polly.Telemetry.SeverityProviderArguments.Source.get -> Polly.Telemetry.ResilienceTelemetrySource!
+Polly.Telemetry.TelemetryOptions.SeverityProvider.get -> System.Func<Polly.Telemetry.SeverityProviderArguments, Polly.Telemetry.ResilienceEventSeverity>?
+Polly.Telemetry.TelemetryOptions.SeverityProvider.set -> void
+Polly.Telemetry.TelemetryOptions.TelemetryOptions(Polly.Telemetry.TelemetryOptions! other) -> void
 static Polly.PollyServiceCollectionExtensions.AddResiliencePipelines<TKey>(this Microsoft.Extensions.DependencyInjection.IServiceCollection! services, System.Action<Polly.DependencyInjection.AddResiliencePipelinesContext<TKey>!>! configure) -> Microsoft.Extensions.DependencyInjection.IServiceCollection!
 Polly.DependencyInjection.AddResiliencePipelinesContext<TKey>
 Polly.DependencyInjection.AddResiliencePipelinesContext<TKey>.AddResiliencePipeline(TKey key, System.Action<Polly.ResiliencePipelineBuilder!, Polly.DependencyInjection.AddResiliencePipelineContext<TKey>!>! configure) -> void

src/Polly.Extensions/Telemetry/SeverityProviderArguments.cs

@@ -0,0 +1,37 @@
+namespace Polly.Telemetry;
+
+#pragma warning disable CA1815 // Override equals and operator equals on value types
+
+/// <summary>
+/// Arguments used by <see cref="TelemetryOptions.SeverityProvider"/>.
+/// </summary>
+public readonly struct SeverityProviderArguments
+{
+    /// <summary>
+    /// Initializes a new instance of the <see cref="SeverityProviderArguments"/> struct.
+    /// </summary>
+    /// <param name="source">The source that produced the resilience event.</param>
+    /// <param name="resilienceEvent">The resilience event.</param>
+    /// <param name="context">The resilience context.</param>
+    public SeverityProviderArguments(ResilienceTelemetrySource source, ResilienceEvent resilienceEvent, ResilienceContext context)
+    {
+        Source = source;
+        Event = resilienceEvent;
+        Context = context;
+    }
+
+    /// <summary>
+    /// Gets the source that produced the resilience event.
+    /// </summary>
+    public ResilienceTelemetrySource Source { get; }
+
+    /// <summary>
+    /// Gets the resilience event.
+    /// </summary>
+    public ResilienceEvent Event { get; }
+
+    /// <summary>
+    /// Gets the resilience context.
+    /// </summary>
+    public ResilienceContext Context { get; }
+}

src/Polly.Extensions/Telemetry/TelemetryListenerImpl.cs

@@ -11,6 +11,7 @@ internal sealed class TelemetryListenerImpl : TelemetryListener
     private readonly ILogger _logger;
     private readonly Func<ResilienceContext, object?, object?> _resultFormatter;
     private readonly List<TelemetryListener> _listeners;
+    private readonly Func<SeverityProviderArguments, ResilienceEventSeverity>? _severityProvider;
     private readonly List<MeteringEnricher> _enrichers;
 
     public TelemetryListenerImpl(TelemetryOptions options)
@@ -19,6 +20,7 @@ public TelemetryListenerImpl(TelemetryOptions options)
         _logger = options.LoggerFactory.CreateLogger(TelemetryUtil.PollyDiagnosticSource);
         _resultFormatter = options.ResultFormatter;
         _listeners = options.TelemetryListeners.ToList();
+        _severityProvider = options.SeverityProvider;
 
         Counter = Meter.CreateCounter<int>(
             "resilience.polly.strategy.events",
@@ -52,8 +54,15 @@ public override void Write<TResult, TArgs>(in TelemetryEventArguments<TResult, T
             }
         }
 
-        LogEvent(in args);
-        MeterEvent(in args);
+        var severity = args.Event.Severity;
+
+        if (_severityProvider is { } provider)
+        {
+            severity = provider(new SeverityProviderArguments(args.Source, args.Event, args.Context));
+        }
+
+        LogEvent(in args, severity);
+        MeterEvent(in args, severity);
     }
 
     private static bool GetArgs<T, TArgs>(T inArgs, out TArgs outArgs)
@@ -68,13 +77,13 @@ private static bool GetArgs<T, TArgs>(T inArgs, out TArgs outArgs)
         return false;
     }
 
-    private static void AddCommonTags<TResult, TArgs>(in EnrichmentContext<TResult, TArgs> context)
+    private static void AddCommonTags<TResult, TArgs>(in EnrichmentContext<TResult, TArgs> context, ResilienceEventSeverity severity)
     {
         var source = context.TelemetryEvent.Source;
         var ev = context.TelemetryEvent.Event;
 
         context.Tags.Add(new(ResilienceTelemetryTags.EventName, context.TelemetryEvent.Event.EventName));
-        context.Tags.Add(new(ResilienceTelemetryTags.EventSeverity, context.TelemetryEvent.Event.Severity.AsString()));
+        context.Tags.Add(new(ResilienceTelemetryTags.EventSeverity, severity.AsString()));
 
         if (source.PipelineName is not null)
         {
@@ -102,7 +111,7 @@ private static void AddCommonTags<TResult, TArgs>(in EnrichmentContext<TResult,
         }
     }
 
-    private void MeterEvent<TResult, TArgs>(in TelemetryEventArguments<TResult, TArgs> args)
+    private void MeterEvent<TResult, TArgs>(in TelemetryEventArguments<TResult, TArgs> args, ResilienceEventSeverity severity)
     {
         var arguments = args.Arguments;
 
@@ -115,7 +124,7 @@ private void MeterEvent<TResult, TArgs>(in TelemetryEventArguments<TResult, TArg
 
             var tags = TagsList.Get();
             var context = new EnrichmentContext<TResult, TArgs>(in args, tags.Tags);
-            UpdateEnrichmentContext(in context);
+            UpdateEnrichmentContext(in context, severity);
             ExecutionDuration.Record(executionFinished.Duration.TotalMilliseconds, tags.TagsSpan);
             TagsList.Return(tags);
         }
@@ -128,7 +137,7 @@ private void MeterEvent<TResult, TArgs>(in TelemetryEventArguments<TResult, TArg
 
             var tags = TagsList.Get();
             var context = new EnrichmentContext<TResult, TArgs>(in args, tags.Tags);
-            UpdateEnrichmentContext(in context);
+            UpdateEnrichmentContext(in context, severity);
             context.Tags.Add(new(ResilienceTelemetryTags.AttemptNumber, executionAttempt.AttemptNumber.AsBoxedInt()));
             context.Tags.Add(new(ResilienceTelemetryTags.AttemptHandled, executionAttempt.Handled.AsBoxedBool()));
             AttemptDuration.Record(executionAttempt.Duration.TotalMilliseconds, tags.TagsSpan);
@@ -138,15 +147,15 @@ private void MeterEvent<TResult, TArgs>(in TelemetryEventArguments<TResult, TArg
         {
             var tags = TagsList.Get();
             var context = new EnrichmentContext<TResult, TArgs>(in args, tags.Tags);
-            UpdateEnrichmentContext(in context);
+            UpdateEnrichmentContext(in context, severity);
             Counter.Add(1, tags.TagsSpan);
             TagsList.Return(tags);
         }
     }
 
-    private void UpdateEnrichmentContext<TResult, TArgs>(in EnrichmentContext<TResult, TArgs> context)
+    private void UpdateEnrichmentContext<TResult, TArgs>(in EnrichmentContext<TResult, TArgs> context, ResilienceEventSeverity severity)
     {
-        AddCommonTags(in context);
+        AddCommonTags(in context, severity);
 
         if (_enrichers.Count != 0)
         {
@@ -157,10 +166,10 @@ private void UpdateEnrichmentContext<TResult, TArgs>(in EnrichmentContext<TResul
         }
     }
 
-    private void LogEvent<TResult, TArgs>(in TelemetryEventArguments<TResult, TArgs> args)
+    private void LogEvent<TResult, TArgs>(in TelemetryEventArguments<TResult, TArgs> args, ResilienceEventSeverity severity)
     {
         var result = GetResult(args.Context, args.Outcome);
-        var level = args.Event.Severity.AsLogLevel();
+        var level = severity.AsLogLevel();
 
         if (GetArgs<TArgs, PipelineExecutingArguments>(args.Arguments, out _))
         {

src/Polly.Extensions/Telemetry/TelemetryOptions.cs

@@ -2,6 +2,7 @@
 using System.Net.Http;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
+using Polly.Utils;
 
 namespace Polly.Telemetry;
 
@@ -10,6 +11,28 @@ namespace Polly.Telemetry;
 /// </summary>
 public class TelemetryOptions
 {
+    /// <summary>
+    /// Initializes a new instance of the <see cref="TelemetryOptions"/> class.
+    /// </summary>
+    public TelemetryOptions()
+    {
+    }
+
+    /// <summary>
+    /// Initializes a new instance of the <see cref="TelemetryOptions"/> class.
+    /// </summary>
+    /// <param name="other">The telemetry options instance to copy the data from.</param>
+    public TelemetryOptions(TelemetryOptions other)
+    {
+        Guard.NotNull(other);
+
+        TelemetryListeners = other.TelemetryListeners.ToList();
+        LoggerFactory = other.LoggerFactory;
+        MeteringEnrichers = other.MeteringEnrichers.ToList();
+        ResultFormatter = other.ResultFormatter;
+        SeverityProvider = other.SeverityProvider;
+    }
+
     /// <summary>
     /// Gets the collection of telemetry listeners.
     /// </summary>
@@ -48,4 +71,12 @@ public class TelemetryOptions
         HttpResponseMessage response => (int)response.StatusCode,
         _ => result,
     };
+
+    /// <summary>
+    /// Gets or sets the resilience event severity provider that allows customizing the severity of resilience events.
+    /// </summary>
+    /// <value>
+    /// The default value is <see langword="null"/>.
+    /// </value>
+    public Func<SeverityProviderArguments, ResilienceEventSeverity>? SeverityProvider { get; set; }
 }

src/Snippets/Docs/Telemetry.cs

@@ -134,4 +134,33 @@ public override void Enrich<TResult, TArgs>(in EnrichmentContext<TResult, TArgs>
     }
 
     #endregion
+
+    public static void SeverityOverrides()
+    {
+        var services = new ServiceCollection();
+
+        #region telemetry-severity-override
+
+        services.AddResiliencePipeline("my-strategy", (builder, context) =>
+        {
+            // Create a new instance of telemetry options by using copy-constructor of the global ones.
+            // This ensures that common configuration is preserved.
+            var telemetryOptions = new TelemetryOptions(context.GetOptions<TelemetryOptions>());
+
+            telemetryOptions.SeverityProvider = args => args.Event.EventName switch
+            {
+                // Decrease severity of specific events
+                "OnRetry" => ResilienceEventSeverity.Debug,
+                "ExecutionAttempt" => ResilienceEventSeverity.Debug,
+                _ => args.Event.Severity
+            };
+
+            builder.AddRetry(new RetryStrategyOptions());
+
+            // Override the telemetry configuration for this pipeline.
+            builder.ConfigureTelemetry(telemetryOptions);
+        });
+
+        #endregion
+    }
 }

test/Polly.Extensions.Tests/Telemetry/TelemetryListenerImplTests.cs

@@ -453,9 +453,45 @@ public void PipelineExecuted_ShouldBeSkipped()
         events.Should().HaveCount(0);
     }
 
+    [Fact]
+    public void SeverityProvider_NotSet_SeverityRespected()
+    {
+        var telemetry = Create();
+
+        ReportEvent(telemetry, null, severity: ResilienceEventSeverity.Critical);
+
+        var records = _logger.GetRecords();
+
+        records.Single().LogLevel.Should().Be(LogLevel.Critical);
+    }
+
+    [Fact]
+    public void SeverityProvider_EnsureRespected()
+    {
+        var called = false;
+        var telemetry = Create(configure: options =>
+        {
+            options.SeverityProvider = args =>
+            {
+                args.Event.Severity.Should().Be(ResilienceEventSeverity.Warning);
+                args.Source.Should().NotBeNull();
+                args.Context.Should().NotBeNull();
+                called = true;
+                return ResilienceEventSeverity.Critical;
+            };
+        });
+
+        ReportEvent(telemetry, null, severity: ResilienceEventSeverity.Warning);
+
+        var records = _logger.GetRecords();
+
+        records.Single().LogLevel.Should().Be(LogLevel.Critical);
+        called.Should().BeTrue();
+    }
+
     private List<Dictionary<string, object?>> GetEvents(string eventName) => _events.Where(e => e.Name == eventName).Select(v => v.Tags).ToList();
 
-    private TelemetryListenerImpl Create(IEnumerable<MeteringEnricher>? enrichers = null)
+    private TelemetryListenerImpl Create(IEnumerable<MeteringEnricher>? enrichers = null, Action<TelemetryOptions>? configure = null)
     {
         var options = new TelemetryOptions
         {
@@ -475,6 +511,8 @@ private TelemetryListenerImpl Create(IEnumerable<MeteringEnricher>? enrichers =
             }
         }
 
+        configure?.Invoke(options);
+
         return new(options);
     }
 

test/Polly.Extensions.Tests/Telemetry/TelemetryOptionsTests.cs

@@ -1,6 +1,8 @@
 using System.Net;
 using System.Net.Http;
+using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Logging.Abstractions;
+using NSubstitute;
 using Polly.Telemetry;
 
 namespace Polly.Extensions.Tests.Telemetry;
@@ -21,4 +23,35 @@ public void Ctor_EnsureDefaults()
         using var response = new HttpResponseMessage(HttpStatusCode.OK);
         options.ResultFormatter(resilienceContext, response).Should().Be(200);
     }
+
+    [Fact]
+    public void CopyCtor_Ok()
+    {
+        var options = new TelemetryOptions
+        {
+            LoggerFactory = Substitute.For<ILoggerFactory>(),
+            SeverityProvider = _ => ResilienceEventSeverity.Error,
+            ResultFormatter = (_, _) => "x",
+        };
+
+        options.MeteringEnrichers.Add(Substitute.For<MeteringEnricher>());
+        options.TelemetryListeners.Add(Substitute.For<TelemetryListener>());
+
+        var other = new TelemetryOptions(options);
+
+        other.ResultFormatter.Should().Be(options.ResultFormatter);
+        other.LoggerFactory.Should().Be(options.LoggerFactory);
+        other.SeverityProvider.Should().Be(options.SeverityProvider);
+        other.MeteringEnrichers.Should().BeEquivalentTo(options.MeteringEnrichers);
+        other.TelemetryListeners.Should().BeEquivalentTo(options.TelemetryListeners);
+        other.TelemetryListeners.Should().NotBeSameAs(options.TelemetryListeners);
+        other.MeteringEnrichers.Should().NotBeSameAs(options.MeteringEnrichers);
+
+        typeof(TelemetryOptions).GetRuntimeProperties().Should().HaveCount(5);
+    }
+
+    [Fact]
+    public void CopyCtor_Reminder()
+        => typeof(TelemetryOptions).GetRuntimeProperties().Should()
+        .HaveCount(5, "Make sure that when you increase this number, you also update the copy constructor to assign the new property.");
 }