Skip to content

Commit 6e61568

Browse files
stephentoubstephentoub
authored
Update genai otel implementation with recent additions (dotnet#6829)
* Add gen_ai.embeddings.dimension.count tag * Add gen_ai.tool.call.arguments/result tags * Add gen_ai.tool.definitions tag
1 parent 53ef115 commit 6e61568

File tree

10 files changed

+158
-44
lines changed

10 files changed

+158
-44
lines changed

‎src/Libraries/Microsoft.Extensions.AI.Abstractions/Functions/AIFunctionDeclaration.cs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,10 @@ protected AIFunctionDeclaration()
3030
/// </para>
3131
/// <code>
3232
/// {
33-
/// "title" : "addNumbers",
34-
/// "description": "A simple function that adds two numbers together.",
3533
/// "type": "object",
3634
/// "properties": {
3735
/// "a" : { "type": "number" },
38-
/// "b" : { "type": "number", "default": 1 }
36+
/// "b" : { "type": ["number","null"], "default": 1 }
3937
/// },
4038
/// "required" : ["a"]
4139
/// }

‎src/Libraries/Microsoft.Extensions.AI/ChatCompletion/FunctionInvokingChatClient.cs‎

Lines changed: 35 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1113,31 +1113,43 @@ FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult resul
11131113
_ = Throw.IfNull(context);
11141114

11151115
using Activity? activity = _activitySource?.StartActivity(
1116-
$"{OpenTelemetryConsts.GenAI.ExecuteTool} {context.Function.Name}",
1116+
$"{OpenTelemetryConsts.GenAI.ExecuteToolName} {context.Function.Name}",
11171117
ActivityKind.Internal,
11181118
default(ActivityContext),
11191119
[
1120-
new(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.ExecuteTool),
1120+
new(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.ExecuteToolName),
11211121
new(OpenTelemetryConsts.GenAI.Tool.Type, OpenTelemetryConsts.ToolTypeFunction),
11221122
new(OpenTelemetryConsts.GenAI.Tool.Call.Id, context.CallContent.CallId),
11231123
new(OpenTelemetryConsts.GenAI.Tool.Name, context.Function.Name),
11241124
new(OpenTelemetryConsts.GenAI.Tool.Description, context.Function.Description),
11251125
]);
11261126

1127-
long startingTimestamp = 0;
1128-
if (_logger.IsEnabled(LogLevel.Debug))
1127+
long startingTimestamp = Stopwatch.GetTimestamp();
1128+
1129+
bool enableSensitiveData = activity is { IsAllDataRequested: true } && InnerClient.GetService<OpenTelemetryChatClient>()?.EnableSensitiveData is true;
1130+
bool traceLoggingEnabled = _logger.IsEnabled(LogLevel.Trace);
1131+
bool loggedInvoke = false;
1132+
if (enableSensitiveData || traceLoggingEnabled)
11291133
{
1130-
startingTimestamp = Stopwatch.GetTimestamp();
1131-
if (_logger.IsEnabled(LogLevel.Trace))
1134+
string functionArguments = TelemetryHelpers.AsJson(context.Arguments, context.Function.JsonSerializerOptions);
1135+
1136+
if (enableSensitiveData)
11321137
{
1133-
LogInvokingSensitive(context.Function.Name, TelemetryHelpers.AsJson(context.Arguments, context.Function.JsonSerializerOptions));
1138+
_ = activity?.SetTag(OpenTelemetryConsts.GenAI.Tool.Call.Arguments, functionArguments);
11341139
}
1135-
else
1140+
1141+
if (traceLoggingEnabled)
11361142
{
1137-
LogInvoking(context.Function.Name);
1143+
LogInvokingSensitive(context.Function.Name, functionArguments);
1144+
loggedInvoke = true;
11381145
}
11391146
}
11401147

1148+
if (!loggedInvoke && _logger.IsEnabled(LogLevel.Debug))
1149+
{
1150+
LogInvoking(context.Function.Name);
1151+
}
1152+
11411153
object? result = null;
11421154
try
11431155
{
@@ -1165,19 +1177,27 @@ FunctionResultContent CreateFunctionResultContent(FunctionInvocationResult resul
11651177
}
11661178
finally
11671179
{
1168-
if (_logger.IsEnabled(LogLevel.Debug))
1180+
bool loggedResult = false;
1181+
if (enableSensitiveData || traceLoggingEnabled)
11691182
{
1170-
TimeSpan elapsed = GetElapsedTime(startingTimestamp);
1183+
string functionResult = TelemetryHelpers.AsJson(result, context.Function.JsonSerializerOptions);
11711184

1172-
if (result is not null && _logger.IsEnabled(LogLevel.Trace))
1185+
if (enableSensitiveData)
11731186
{
1174-
LogInvocationCompletedSensitive(context.Function.Name, elapsed, TelemetryHelpers.AsJson(result, context.Function.JsonSerializerOptions));
1187+
_ = activity?.SetTag(OpenTelemetryConsts.GenAI.Tool.Call.Result, functionResult);
11751188
}
1176-
else
1189+
1190+
if (traceLoggingEnabled)
11771191
{
1178-
LogInvocationCompleted(context.Function.Name, elapsed);
1192+
LogInvocationCompletedSensitive(context.Function.Name, GetElapsedTime(startingTimestamp), functionResult);
1193+
loggedResult = true;
11791194
}
11801195
}
1196+
1197+
if (!loggedResult && _logger.IsEnabled(LogLevel.Debug))
1198+
{
1199+
LogInvocationCompleted(context.Function.Name, GetElapsedTime(startingTimestamp));
1200+
}
11811201
}
11821202

11831203
return result;

‎src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryChatClient.cs‎

Lines changed: 32 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -318,13 +318,13 @@ internal static string SerializeChatMessages(IEnumerable<ChatMessage> messages,
318318
string? modelId = options?.ModelId ?? _defaultModelId;
319319

320320
activity = _activitySource.StartActivity(
321-
string.IsNullOrWhiteSpace(modelId) ? OpenTelemetryConsts.GenAI.Chat : $"{OpenTelemetryConsts.GenAI.Chat} {modelId}",
321+
string.IsNullOrWhiteSpace(modelId) ? OpenTelemetryConsts.GenAI.ChatName : $"{OpenTelemetryConsts.GenAI.ChatName} {modelId}",
322322
ActivityKind.Client);
323323

324324
if (activity is { IsAllDataRequested: true })
325325
{
326326
_ = activity
327-
.AddTag(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.Chat)
327+
.AddTag(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.ChatName)
328328
.AddTag(OpenTelemetryConsts.GenAI.Request.Model, modelId)
329329
.AddTag(OpenTelemetryConsts.GenAI.Provider.Name, _providerName);
330330

@@ -395,13 +395,28 @@ internal static string SerializeChatMessages(IEnumerable<ChatMessage> messages,
395395
}
396396
}
397397

398-
// Log all additional request options as raw values on the span.
399-
// Since AdditionalProperties has undefined meaning, we treat it as potentially sensitive data.
400-
if (EnableSensitiveData && options.AdditionalProperties is { } props)
398+
if (EnableSensitiveData)
401399
{
402-
foreach (KeyValuePair<string, object?> prop in props)
400+
if (options.Tools?.Any(t => t is AIFunctionDeclaration) is true)
403401
{
404-
_ = activity.AddTag(prop.Key, prop.Value);
402+
_ = activity.AddTag(
403+
OpenTelemetryConsts.GenAI.Tool.Definitions,
404+
JsonSerializer.Serialize(options.Tools.OfType<AIFunctionDeclaration>().Select(t => new OtelFunction
405+
{
406+
Name = t.Name,
407+
Description = t.Description,
408+
Parameters = t.JsonSchema,
409+
}), OtelContext.Default.IEnumerableOtelFunction));
410+
}
411+
412+
// Log all additional request options as raw values on the span.
413+
// Since AdditionalProperties has undefined meaning, we treat it as potentially sensitive data.
414+
if (options.AdditionalProperties is { } props)
415+
{
416+
foreach (KeyValuePair<string, object?> prop in props)
417+
{
418+
_ = activity.AddTag(prop.Key, prop.Value);
419+
}
405420
}
406421
}
407422
}
@@ -505,7 +520,7 @@ private void TraceResponse(
505520

506521
void AddMetricTags(ref TagList tags, string? requestModelId, ChatResponse? response)
507522
{
508-
tags.Add(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.Chat);
523+
tags.Add(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.ChatName);
509524

510525
if (requestModelId is not null)
511526
{
@@ -582,6 +597,14 @@ private sealed class OtelToolCallResponsePart
582597
public object? Response { get; set; }
583598
}
584599

600+
private sealed class OtelFunction
601+
{
602+
public string Type { get; set; } = "function";
603+
public string? Name { get; set; }
604+
public string? Description { get; set; }
605+
public JsonElement Parameters { get; set; }
606+
}
607+
585608
private static readonly JsonSerializerOptions _defaultOptions = CreateDefaultOptions();
586609

587610
private static JsonSerializerOptions CreateDefaultOptions()
@@ -606,5 +629,6 @@ private static JsonSerializerOptions CreateDefaultOptions()
606629
[JsonSerializable(typeof(OtelGenericPart))]
607630
[JsonSerializable(typeof(OtelToolCallRequestPart))]
608631
[JsonSerializable(typeof(OtelToolCallResponsePart))]
632+
[JsonSerializable(typeof(IEnumerable<OtelFunction>))]
609633
private sealed partial class OtelContext : JsonSerializerContext;
610634
}

‎src/Libraries/Microsoft.Extensions.AI/ChatCompletion/OpenTelemetryImageGenerator.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -151,13 +151,13 @@ public async override Task<ImageGenerationResponse> GenerateAsync(
151151
string? modelId = options?.ModelId ?? _defaultModelId;
152152

153153
activity = _activitySource.StartActivity(
154-
string.IsNullOrWhiteSpace(modelId) ? OpenTelemetryConsts.GenAI.GenerateContent : $"{OpenTelemetryConsts.GenAI.GenerateContent} {modelId}",
154+
string.IsNullOrWhiteSpace(modelId) ? OpenTelemetryConsts.GenAI.GenerateContentName : $"{OpenTelemetryConsts.GenAI.GenerateContentName} {modelId}",
155155
ActivityKind.Client);
156156

157157
if (activity is { IsAllDataRequested: true })
158158
{
159159
_ = activity
160-
.AddTag(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.GenerateContent)
160+
.AddTag(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.GenerateContentName)
161161
.AddTag(OpenTelemetryConsts.GenAI.Output.Type, OpenTelemetryConsts.TypeImage)
162162
.AddTag(OpenTelemetryConsts.GenAI.Request.Model, modelId)
163163
.AddTag(OpenTelemetryConsts.GenAI.Provider.Name, _providerName);
@@ -294,7 +294,7 @@ private void TraceResponse(
294294

295295
void AddMetricTags(ref TagList tags, string? requestModelId)
296296
{
297-
tags.Add(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.GenerateContent);
297+
tags.Add(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.GenerateContentName);
298298

299299
if (requestModelId is not null)
300300
{

‎src/Libraries/Microsoft.Extensions.AI/Embeddings/OpenTelemetryEmbeddingGenerator.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -154,11 +154,11 @@ protected override void Dispose(bool disposing)
154154
string? modelId = options?.ModelId ?? _defaultModelId;
155155

156156
activity = _activitySource.StartActivity(
157-
string.IsNullOrWhiteSpace(modelId) ? OpenTelemetryConsts.GenAI.Embeddings : $"{OpenTelemetryConsts.GenAI.Embeddings} {modelId}",
157+
string.IsNullOrWhiteSpace(modelId) ? OpenTelemetryConsts.GenAI.EmbeddingsName : $"{OpenTelemetryConsts.GenAI.EmbeddingsName} {modelId}",
158158
ActivityKind.Client,
159159
default(ActivityContext),
160160
[
161-
new(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.Embeddings),
161+
new(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.EmbeddingsName),
162162
new(OpenTelemetryConsts.GenAI.Request.Model, modelId),
163163
new(OpenTelemetryConsts.GenAI.Provider.Name, _providerName),
164164
]);
@@ -174,7 +174,7 @@ protected override void Dispose(bool disposing)
174174

175175
if ((options?.Dimensions ?? _defaultModelDimensions) is int dimensionsValue)
176176
{
177-
_ = activity.AddTag(OpenTelemetryConsts.GenAI.Request.EmbeddingDimensions, dimensionsValue);
177+
_ = activity.AddTag(OpenTelemetryConsts.GenAI.Embeddings.Dimension.Count, dimensionsValue);
178178
}
179179

180180
// Log all additional request options as raw values on the span.
@@ -265,7 +265,7 @@ private void TraceResponse(
265265

266266
private void AddMetricTags(ref TagList tags, string? requestModelId, string? responseModelId)
267267
{
268-
tags.Add(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.Embeddings);
268+
tags.Add(OpenTelemetryConsts.GenAI.Operation.Name, OpenTelemetryConsts.GenAI.EmbeddingsName);
269269

270270
if (requestModelId is not null)
271271
{

‎src/Libraries/Microsoft.Extensions.AI/OpenTelemetryConsts.cs‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,10 +33,10 @@ public static class Error
3333

3434
public static class GenAI
3535
{
36-
public const string Chat = "chat";
37-
public const string Embeddings = "embeddings";
38-
public const string ExecuteTool = "execute_tool";
39-
public const string GenerateContent = "generate_content";
36+
public const string ChatName = "chat";
37+
public const string EmbeddingsName = "embeddings";
38+
public const string ExecuteToolName = "execute_tool";
39+
public const string GenerateContentName = "generate_content";
4040

4141
public const string SystemInstructions = "gen_ai.system_instructions";
4242

@@ -62,6 +62,14 @@ public static class Conversation
6262
public const string Id = "gen_ai.conversation.id";
6363
}
6464

65+
public static class Embeddings
66+
{
67+
public static class Dimension
68+
{
69+
public const string Count = "gen_ai.embeddings.dimension.count";
70+
}
71+
}
72+
6573
public static class Input
6674
{
6775
public const string Messages = "gen_ai.input.messages";
@@ -86,7 +94,6 @@ public static class Provider
8694
public static class Request
8795
{
8896
public const string ChoiceCount = "gen_ai.request.choice.count";
89-
public const string EmbeddingDimensions = "gen_ai.request.embedding.dimensions";
9097
public const string FrequencyPenalty = "gen_ai.request.frequency_penalty";
9198
public const string Model = "gen_ai.request.model";
9299
public const string MaxTokens = "gen_ai.request.max_tokens";
@@ -116,10 +123,13 @@ public static class Tool
116123
public const string Description = "gen_ai.tool.description";
117124
public const string Message = "gen_ai.tool.message";
118125
public const string Type = "gen_ai.tool.type";
126+
public const string Definitions = "gen_ai.tool.definitions";
119127

120128
public static class Call
121129
{
122130
public const string Id = "gen_ai.tool.call.id";
131+
public const string Arguments = "gen_ai.tool.call.arguments";
132+
public const string Result = "gen_ai.tool.call.result";
123133
}
124134
}
125135

‎test/Libraries/Microsoft.Extensions.AI.Integration.Tests/EmbeddingGeneratorIntegrationTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -124,7 +124,7 @@ public virtual async Task OpenTelemetry_CanEmitTracesAndMetrics()
124124
Assert.Single(activities);
125125
var activity = activities.Single();
126126
Assert.StartsWith("embed", activity.DisplayName);
127-
Assert.StartsWith("http", (string)activity.GetTagItem("server.address")!);
127+
Assert.Contains(".", (string)activity.GetTagItem("server.address")!);
128128
Assert.Equal(embeddingGenerator.GetService<EmbeddingGeneratorMetadata>()?.ProviderUri?.Port, (int)activity.GetTagItem("server.port")!);
129129
Assert.NotNull(activity.Id);
130130
Assert.NotEmpty(activity.Id);

‎test/Libraries/Microsoft.Extensions.AI.Tests/ChatCompletion/FunctionInvokingChatClientTests.cs‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -655,9 +655,10 @@ async Task InvokeAsync(Func<IServiceProvider, Task> work)
655655
}
656656

657657
[Theory]
658-
[InlineData(false)]
659-
[InlineData(true)]
660-
public async Task FunctionInvocationTrackedWithActivity(bool enableTelemetry)
658+
[InlineData(false, false)]
659+
[InlineData(true, false)]
660+
[InlineData(true, true)]
661+
public async Task FunctionInvocationTrackedWithActivity(bool enableTelemetry, bool enableSensitiveData)
661662
{
662663
string sourceName = Guid.NewGuid().ToString();
663664

@@ -675,7 +676,7 @@ public async Task FunctionInvocationTrackedWithActivity(bool enableTelemetry)
675676
};
676677

677678
Func<ChatClientBuilder, ChatClientBuilder> configure = b => b.Use(c =>
678-
new FunctionInvokingChatClient(new OpenTelemetryChatClient(c, sourceName: sourceName)));
679+
new FunctionInvokingChatClient(new OpenTelemetryChatClient(c, sourceName: sourceName) { EnableSensitiveData = enableSensitiveData }));
679680

680681
await InvokeAsync(() => InvokeAndAssertAsync(options, plan, configurePipeline: configure), streaming: false);
681682

@@ -701,6 +702,23 @@ async Task InvokeAsync(Func<Task> work, bool streaming)
701702
activity => Assert.Equal("chat", activity.DisplayName),
702703
activity => Assert.Equal(streaming ? "FunctionInvokingChatClient.GetStreamingResponseAsync" : "FunctionInvokingChatClient.GetResponseAsync", activity.DisplayName));
703704

705+
var executeTool = activities[1];
706+
if (enableSensitiveData)
707+
{
708+
var args = Assert.Single(executeTool.Tags, t => t.Key == "gen_ai.tool.call.arguments");
709+
Assert.Equal(
710+
JsonSerializer.Serialize(new Dictionary<string, object?> { ["arg1"] = "value1" }, AIJsonUtilities.DefaultOptions),
711+
args.Value);
712+
713+
var result = Assert.Single(executeTool.Tags, t => t.Key == "gen_ai.tool.call.result");
714+
Assert.Equal("Result 1", JsonSerializer.Deserialize<string>(result.Value!, AIJsonUtilities.DefaultOptions));
715+
}
716+
else
717+
{
718+
Assert.DoesNotContain(executeTool.Tags, t => t.Key == "gen_ai.tool.call.arguments");
719+
Assert.DoesNotContain(executeTool.Tags, t => t.Key == "gen_ai.tool.call.result");
720+
}
721+
704722
for (int i = 0; i < activities.Count - 1; i++)
705723
{
706724
// Activities are exported in the order of completion, so all except the last are children of the last (i.e., outer)

0 commit comments

Comments
 (0)