Fix exception occurring when Azure OpenAI service is throttling requests

.github/_typos.toml

@@ -15,7 +15,8 @@ extend-exclude = [
     "encoder.json",
     "appsettings.development.json",
     "appsettings.Development.json",
-    "AzureAISearchFilteringTest.cs"
+    "AzureAISearchFilteringTest.cs",
+    "KernelMemory.sln.DotSettings"
 ]
 
 [default.extend-words]

KernelMemory.sln

@@ -187,7 +187,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureQueues", "extensions\A
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureAIDocIntel", "extensions\AzureAIDocIntel\AzureAIDocIntel.csproj", "{CFE7C192-2561-40CC-8592-136293451EC1}"
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureOpenAI", "extensions\AzureOpenAI\AzureOpenAI.csproj", "{93FA6DD6-D0B2-4751-8680-3F959E1F7AF2}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureOpenAI", "extensions\AzureOpenAI\AzureOpenAI\AzureOpenAI.csproj", "{93FA6DD6-D0B2-4751-8680-3F959E1F7AF2}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AzureAISearch.TestApplication", "extensions\AzureAISearch\AzureAISearch.TestApplication\AzureAISearch.TestApplication.csproj", "{11445C36-1B94-4AFB-AC23-976C94924603}"
 EndProject
@@ -331,6 +331,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureAIContentSafety", "ext
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "KernelMemory", "extensions\KM\KernelMemory\KernelMemory.csproj", "{AB097B62-5A0B-4D74-9F8B-A41FE8241447}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AzureOpenAI.FunctionalTests", "extensions\AzureOpenAI\AzureOpenAI.FunctionalTests\AzureOpenAI.FunctionalTests.csproj", "{8E907766-4A7D-46E2-B5E3-EB2994B1AA54}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -613,6 +615,9 @@ Global
 		{AB097B62-5A0B-4D74-9F8B-A41FE8241447}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{AB097B62-5A0B-4D74-9F8B-A41FE8241447}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{AB097B62-5A0B-4D74-9F8B-A41FE8241447}.Release|Any CPU.Build.0 = Release|Any CPU
+		{8E907766-4A7D-46E2-B5E3-EB2994B1AA54}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{8E907766-4A7D-46E2-B5E3-EB2994B1AA54}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{8E907766-4A7D-46E2-B5E3-EB2994B1AA54}.Release|Any CPU.ActiveCfg = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -711,6 +716,7 @@ Global
 		{82670921-FDCD-4672-84BD-4353F5AC24A0} = {3C17F42B-CFC8-4900-8CFB-88936311E919}
 		{58E65B3F-EFF0-401A-AC76-A49835AE0220} = {155DA079-E267-49AF-973A-D1D44681970F}
 		{AB097B62-5A0B-4D74-9F8B-A41FE8241447} = {155DA079-E267-49AF-973A-D1D44681970F}
+		{8E907766-4A7D-46E2-B5E3-EB2994B1AA54} = {3C17F42B-CFC8-4900-8CFB-88936311E919}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {CC136C62-115C-41D1-B414-F9473EFF6EA8}

applications/tests/Evaluation.Tests/Evaluation.FunctionalTests.csproj

@@ -33,7 +33,7 @@
     </ItemGroup>
 
     <ItemGroup>
-      <ProjectReference Include="..\..\..\extensions\AzureOpenAI\AzureOpenAI.csproj" />
+      <ProjectReference Include="..\..\..\extensions\AzureOpenAI\AzureOpenAI\AzureOpenAI.csproj" />
       <ProjectReference Include="..\..\evaluation\Evaluation.csproj" />
     </ItemGroup>
 

extensions/AzureOpenAI/AzureOpenAI.FunctionalTests/AzureOpenAI.FunctionalTests.csproj

@@ -0,0 +1,32 @@
+<Project Sdk="Microsoft.NET.Sdk.Web">
+
+    <PropertyGroup>
+        <AssemblyName>Microsoft.AzureOpenAI.FunctionalTests</AssemblyName>
+        <RootNamespace>Microsoft.AzureOpenAI.FunctionalTests</RootNamespace>
+        <TargetFramework>net8.0</TargetFramework>
+        <RollForward>LatestMajor</RollForward>
+        <IsTestProject>true</IsTestProject>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+        <IsPackable>false</IsPackable>
+        <NoWarn>KMEXP01;</NoWarn>
+    </PropertyGroup>
+
+    <ItemGroup>
+        <ProjectReference Include="..\..\..\service\tests\TestHelpers\TestHelpers.csproj" />
+    </ItemGroup>
+
+    <ItemGroup>
+        <PackageReference Include="Microsoft.Extensions.DependencyInjection" />
+        <PackageReference Include="Microsoft.NET.Test.Sdk" />
+        <PackageReference Include="xunit" />
+        <PackageReference Include="xunit.abstractions" />
+        <PackageReference Include="Xunit.DependencyInjection" />
+        <PackageReference Include="Xunit.DependencyInjection.Logging" />
+        <PackageReference Include="xunit.runner.visualstudio">
+            <PrivateAssets>all</PrivateAssets>
+            <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+        </PackageReference>
+    </ItemGroup>
+
+</Project>

extensions/AzureOpenAI/AzureOpenAI.FunctionalTests/Issue855Test.cs

@@ -0,0 +1,87 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using Microsoft.KernelMemory.AI.AzureOpenAI;
+using Microsoft.KM.TestHelpers;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.AzureOpenAI.FunctionalTests;
+
+/// <summary>
+/// References:
+/// - https://github.com/Azure/azure-sdk-for-net/issues/46109
+/// - https://github.com/microsoft/semantic-kernel/issues/8929
+/// - https://github.com/microsoft/kernel-memory/issues/855
+/// </summary>
+public class Issue855Test : BaseFunctionalTestCase
+{
+    private readonly AzureOpenAITextEmbeddingGenerator _target;
+
+    public Issue855Test(IConfiguration cfg, ITestOutputHelper output) : base(cfg, output)
+    {
+        this._target = new AzureOpenAITextEmbeddingGenerator(this.AzureOpenAIEmbeddingConfiguration);
+    }
+
+    [Fact(Skip = "Enable and run manually")]
+    [Trait("Category", "Manual")]
+    [Trait("Category", "BugFix")]
+    public async Task ItDoesntWhenThrottling()
+    {
+        for (int i = 0; i < 50; i++)
+        {
+            Console.WriteLine($"## {i}");
+            await this._target.GenerateEmbeddingBatchAsync(
+                [RndStr(), RndStr(), RndStr(), RndStr(), RndStr(), RndStr(), RndStr(), RndStr(), RndStr(), RndStr()]);
+        }
+    }
+
+#pragma warning disable CA5394
+    private static string RndStr()
+    {
+        var random = new Random();
+        return new(Enumerable.Repeat(" ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ", 8000)
+            .Select(s => s[random.Next(s.Length)]).ToArray());
+    }
+}
+
+#pragma warning disable IDE0055
+/* When the test fails: after pausing and trying to restart, an exception occurs.
+
+Microsoft.SemanticKernel.HttpOperationException: Service request failed.
+
+Microsoft.SemanticKernel.HttpOperationException
+Service request failed.
+Status: 401 (Unauthorized) <===== ******* Caused by https://github.com/Azure/azure-sdk-for-net/issues/46109
+
+  at Microsoft.SemanticKernel.Connectors.OpenAI.ClientCore.RunRequestAsync[T](Func`1 request)
+  at Microsoft.SemanticKernel.Connectors.OpenAI.ClientCore.GetEmbeddingsAsync(String targetModel, IList`1 data, Kernel kernel, Nullable`1 dimensions, CancellationToken cancellationToken)
+  at Microsoft.KernelMemory.AI.AzureOpenAI.AzureOpenAITextEmbeddingGenerator.GenerateEmbeddingBatchAsync(IEnumerable`1 textList, CancellationToken cancellationToken) in extensions/AzureOpenAI/AzureOpenAI/AzureOpenAITextEmbeddingGenerator.cs:line 132
+  at Microsoft.AzureOpenAI.FunctionalTests.Issue855Test.ItDoesntFailWith401() in extensions/AzureOpenAI/AzureOpenAI.FunctionalTests/Bug46109Test.cs:line 43
+  at Xunit.DependencyInjection.DependencyInjectionTestInvoker.AsyncStack(Task task, Activity activity) in S:\GitHub\Xunit.DependencyInjection\src\Xunit.DependencyInjection\DependencyInjectionTestInvoker.cs:line 174
+
+System.ClientModel.ClientResultException
+Service request failed.
+Status: 401 (Unauthorized)
+
+  at Azure.AI.OpenAI.ClientPipelineExtensions.ProcessMessageAsync(ClientPipeline pipeline, PipelineMessage message, RequestOptions options)
+  at Azure.AI.OpenAI.Embeddings.AzureEmbeddingClient.GenerateEmbeddingsAsync(BinaryContent content, RequestOptions options)
+  at OpenAI.Embeddings.EmbeddingClient.GenerateEmbeddingsAsync(IEnumerable`1 inputs, EmbeddingGenerationOptions options, CancellationToken cancellationToken)
+  at Microsoft.SemanticKernel.Connectors.OpenAI.ClientCore.RunRequestAsync[T](Func`1 request)
+
+
+
+warn: Microsoft.KernelMemory.AI.AzureOpenAI.AzureOpenAITextEmbeddingGenerator[0]
+     Tokenizer not specified, will use GPT4oTokenizer. The token count might be incorrect, causing unexpected errors
+
+## 0
+## 1
+## 2
+## 3
+...
+...
+warn: Microsoft.KernelMemory.AI.AzureOpenAI.Internals.ClientSequentialRetryPolicy[0]
+     Header Retry-After found, value 21
+
+warn: Microsoft.KernelMemory.AI.AzureOpenAI.Internals.ClientSequentialRetryPolicy[0]
+     Delay extracted from HTTP response: 21000 msecs
+*/

extensions/AzureOpenAI/AzureOpenAI.FunctionalTests/Startup.cs

@@ -0,0 +1,22 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+/* IMPORTANT: the Startup class must be at the root of the namespace and
+ * the namespace must match exactly (required by Xunit.DependencyInjection) */
+
+namespace Microsoft.AzureOpenAI.FunctionalTests;
+
+public class Startup
+{
+    public void ConfigureHost(IHostBuilder hostBuilder)
+    {
+        var config = new ConfigurationBuilder()
+            .AddJsonFile("appsettings.json")
+            .AddJsonFile("appsettings.development.json", optional: true)
+            .AddJsonFile("appsettings.Development.json", optional: true)
+            .AddUserSecrets<Startup>()
+            .AddEnvironmentVariables()
+            .Build();
+
+        hostBuilder.ConfigureHostConfiguration(builder => builder.AddConfiguration(config));
+    }
+}

extensions/AzureOpenAI/AzureOpenAI.FunctionalTests/appsettings.json

@@ -0,0 +1,12 @@
+{
+  "KernelMemory": {
+    "Services": {
+      "AzureOpenAIEmbedding": {
+        "Auth": "AzureIdentity", // "ApiKey" or "AzureIdentity"
+        "Endpoint": "https://<...>.openai.azure.com/",
+        "APIKey": "",
+        "Deployment": ""
+      }
+    }
+  }
+}

extensions/AzureOpenAI/AzureOpenAI.csproj → extensions/AzureOpenAI/AzureOpenAI/AzureOpenAI.csproj

@@ -9,8 +9,8 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <ProjectReference Include="..\..\service\Abstractions\Abstractions.csproj" />
-        <ProjectReference Include="..\OpenAI\OpenAI\OpenAI.csproj" />
+        <ProjectReference Include="..\..\..\service\Abstractions\Abstractions.csproj" />
+        <ProjectReference Include="..\..\OpenAI\OpenAI\OpenAI.csproj" />
     </ItemGroup>
 
     <ItemGroup>
@@ -28,7 +28,7 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <None Include="README.md" Link="README.md" Pack="true" PackagePath="." Visible="false" />
+        <None Include="..\README.md" Link="README.md" Pack="true" PackagePath="." Visible="false" />
     </ItemGroup>
 
 </Project>

extensions/AzureOpenAI/Internals/AzureOpenAIClientBuilder.cs → extensions/AzureOpenAI/AzureOpenAI/Internals/AzureOpenAIClientBuilder.cs

@@ -29,6 +29,9 @@ internal static AzureOpenAIClient Build(
             UserAgentApplicationId = Telemetry.HttpUserAgent,
         };
 
+        // See https://github.com/Azure/azure-sdk-for-net/issues/46109
+        options.AddPolicy(new SingleAuthorizationHeaderPolicy(), PipelinePosition.PerTry);
+
         if (httpClient is not null)
         {
             options.Transport = new HttpClientPipelineTransport(httpClient);

extensions/AzureOpenAI/AzureOpenAI/Internals/SingleAuthorizationHeaderPolicy.cs

@@ -0,0 +1,46 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.ClientModel.Primitives;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+namespace Microsoft.KernelMemory.AI.AzureOpenAI.Internals;
+
+/// <summary>
+/// Bug fix: Remove duplicate Authorization headers from the request.
+/// See https://github.com/Azure/azure-sdk-for-net/issues/46109
+/// </summary>
+internal sealed class SingleAuthorizationHeaderPolicy : PipelinePolicy
+{
+    public override void Process(PipelineMessage message, IReadOnlyList<PipelinePolicy> pipeline, int currentIndex)
+    {
+        RemoveDuplicateHeader(message.Request.Headers);
+        ProcessNext(message, pipeline, currentIndex);
+    }
+
+    public override async ValueTask ProcessAsync(PipelineMessage message, IReadOnlyList<PipelinePolicy> pipeline, int currentIndex)
+    {
+        RemoveDuplicateHeader(message.Request.Headers);
+        await ProcessNextAsync(message, pipeline, currentIndex).ConfigureAwait(false);
+    }
+
+    private static void RemoveDuplicateHeader(PipelineRequestHeaders headers)
+    {
+        if (!headers.TryGetValues("Authorization", out var headerValues) || headerValues == null)
+        {
+            return;
+        }
+
+        using var enumerator = headerValues.GetEnumerator();
+
+        if (!enumerator.MoveNext()) { return; }
+
+        var firstValue = enumerator.Current;
+
+        // Check if there’s more than one value
+        if (enumerator.MoveNext())
+        {
+            headers.Set("Authorization", firstValue);
+        }
+    }
+}

extensions/KM/KernelMemory/KernelMemory.csproj

@@ -19,7 +19,7 @@
         <ProjectReference Include="..\..\AzureAIDocIntel\AzureAIDocIntel.csproj" />
         <ProjectReference Include="..\..\AzureAISearch\AzureAISearch\AzureAISearch.csproj" />
         <ProjectReference Include="..\..\AzureBlobs\AzureBlobs.csproj" />
-        <ProjectReference Include="..\..\AzureOpenAI\AzureOpenAI.csproj" />
+        <ProjectReference Include="..\..\AzureOpenAI\AzureOpenAI\AzureOpenAI.csproj" />
         <ProjectReference Include="..\..\AzureQueues\AzureQueues.csproj" />
         <ProjectReference Include="..\..\Elasticsearch\Elasticsearch\Elasticsearch.csproj" />
         <ProjectReference Include="..\..\LlamaSharp\LlamaSharp\LlamaSharp.csproj" />