-
Notifications
You must be signed in to change notification settings - Fork 320
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fix exception occurring when Azure OpenAI service is throttling reque…
…sts (#876) ## Motivation and Context (Why the change? What's the scenario?) When retrying requests after throttling, the Azure OpenAI SDK is sending malformed requests, with multiple `Authorization` headers, which are rejected by the Azure OpenAI service with a `401 (Unauthorized)` error code, leading to an exception in the SDK. See - Azure/azure-sdk-for-net#46109 - microsoft/semantic-kernel#8929 - #855 ## High level description (Approach, Design) Inject a policy to fix malformed HTTP headers. Functional test included, to verify the fix.
- Loading branch information
Showing
17 changed files
with
216 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
32 changes: 32 additions & 0 deletions
32
extensions/AzureOpenAI/AzureOpenAI.FunctionalTests/AzureOpenAI.FunctionalTests.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,32 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk.Web"> | ||
|
|
||
| <PropertyGroup> | ||
| <AssemblyName>Microsoft.AzureOpenAI.FunctionalTests</AssemblyName> | ||
| <RootNamespace>Microsoft.AzureOpenAI.FunctionalTests</RootNamespace> | ||
| <TargetFramework>net8.0</TargetFramework> | ||
| <RollForward>LatestMajor</RollForward> | ||
| <IsTestProject>true</IsTestProject> | ||
| <ImplicitUsings>enable</ImplicitUsings> | ||
| <Nullable>enable</Nullable> | ||
| <IsPackable>false</IsPackable> | ||
| <NoWarn>KMEXP01;</NoWarn> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\..\..\service\tests\TestHelpers\TestHelpers.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Microsoft.Extensions.DependencyInjection" /> | ||
| <PackageReference Include="Microsoft.NET.Test.Sdk" /> | ||
| <PackageReference Include="xunit" /> | ||
| <PackageReference Include="xunit.abstractions" /> | ||
| <PackageReference Include="Xunit.DependencyInjection" /> | ||
| <PackageReference Include="Xunit.DependencyInjection.Logging" /> | ||
| <PackageReference Include="xunit.runner.visualstudio"> | ||
| <PrivateAssets>all</PrivateAssets> | ||
| <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> | ||
| </PackageReference> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
87 changes: 87 additions & 0 deletions
87
extensions/AzureOpenAI/AzureOpenAI.FunctionalTests/Issue855Test.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,87 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using Microsoft.KernelMemory.AI.AzureOpenAI; | ||
| using Microsoft.KM.TestHelpers; | ||
| using Xunit; | ||
| using Xunit.Abstractions; | ||
|
|
||
| namespace Microsoft.AzureOpenAI.FunctionalTests; | ||
|
|
||
| /// <summary> | ||
| /// References: | ||
| /// - https://github.com/Azure/azure-sdk-for-net/issues/46109 | ||
| /// - https://github.com/microsoft/semantic-kernel/issues/8929 | ||
| /// - https://github.com/microsoft/kernel-memory/issues/855 | ||
| /// </summary> | ||
| public class Issue855Test : BaseFunctionalTestCase | ||
| { | ||
| private readonly AzureOpenAITextEmbeddingGenerator _target; | ||
|
|
||
| public Issue855Test(IConfiguration cfg, ITestOutputHelper output) : base(cfg, output) | ||
| { | ||
| this._target = new AzureOpenAITextEmbeddingGenerator(this.AzureOpenAIEmbeddingConfiguration); | ||
| } | ||
|
|
||
| [Fact(Skip = "Enable and run manually")] | ||
| [Trait("Category", "Manual")] | ||
| [Trait("Category", "BugFix")] | ||
| public async Task ItDoesntWhenThrottling() | ||
| { | ||
| for (int i = 0; i < 50; i++) | ||
| { | ||
| Console.WriteLine($"## {i}"); | ||
| await this._target.GenerateEmbeddingBatchAsync( | ||
| [RndStr(), RndStr(), RndStr(), RndStr(), RndStr(), RndStr(), RndStr(), RndStr(), RndStr(), RndStr()]); | ||
| } | ||
| } | ||
|
|
||
| #pragma warning disable CA5394 | ||
| private static string RndStr() | ||
| { | ||
| var random = new Random(); | ||
| return new(Enumerable.Repeat(" ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz 0123456789 ", 8000) | ||
| .Select(s => s[random.Next(s.Length)]).ToArray()); | ||
| } | ||
| } | ||
|
|
||
| #pragma warning disable IDE0055 | ||
| /* When the test fails: after pausing and trying to restart, an exception occurs. | ||
| Microsoft.SemanticKernel.HttpOperationException: Service request failed. | ||
| Microsoft.SemanticKernel.HttpOperationException | ||
| Service request failed. | ||
| Status: 401 (Unauthorized) <===== ******* Caused by https://github.com/Azure/azure-sdk-for-net/issues/46109 | ||
| at Microsoft.SemanticKernel.Connectors.OpenAI.ClientCore.RunRequestAsync[T](Func`1 request) | ||
| at Microsoft.SemanticKernel.Connectors.OpenAI.ClientCore.GetEmbeddingsAsync(String targetModel, IList`1 data, Kernel kernel, Nullable`1 dimensions, CancellationToken cancellationToken) | ||
| at Microsoft.KernelMemory.AI.AzureOpenAI.AzureOpenAITextEmbeddingGenerator.GenerateEmbeddingBatchAsync(IEnumerable`1 textList, CancellationToken cancellationToken) in extensions/AzureOpenAI/AzureOpenAI/AzureOpenAITextEmbeddingGenerator.cs:line 132 | ||
| at Microsoft.AzureOpenAI.FunctionalTests.Issue855Test.ItDoesntFailWith401() in extensions/AzureOpenAI/AzureOpenAI.FunctionalTests/Bug46109Test.cs:line 43 | ||
| at Xunit.DependencyInjection.DependencyInjectionTestInvoker.AsyncStack(Task task, Activity activity) in S:\GitHub\Xunit.DependencyInjection\src\Xunit.DependencyInjection\DependencyInjectionTestInvoker.cs:line 174 | ||
| System.ClientModel.ClientResultException | ||
| Service request failed. | ||
| Status: 401 (Unauthorized) | ||
| at Azure.AI.OpenAI.ClientPipelineExtensions.ProcessMessageAsync(ClientPipeline pipeline, PipelineMessage message, RequestOptions options) | ||
| at Azure.AI.OpenAI.Embeddings.AzureEmbeddingClient.GenerateEmbeddingsAsync(BinaryContent content, RequestOptions options) | ||
| at OpenAI.Embeddings.EmbeddingClient.GenerateEmbeddingsAsync(IEnumerable`1 inputs, EmbeddingGenerationOptions options, CancellationToken cancellationToken) | ||
| at Microsoft.SemanticKernel.Connectors.OpenAI.ClientCore.RunRequestAsync[T](Func`1 request) | ||
| warn: Microsoft.KernelMemory.AI.AzureOpenAI.AzureOpenAITextEmbeddingGenerator[0] | ||
| Tokenizer not specified, will use GPT4oTokenizer. The token count might be incorrect, causing unexpected errors | ||
| ## 0 | ||
| ## 1 | ||
| ## 2 | ||
| ## 3 | ||
| ... | ||
| ... | ||
| warn: Microsoft.KernelMemory.AI.AzureOpenAI.Internals.ClientSequentialRetryPolicy[0] | ||
| Header Retry-After found, value 21 | ||
| warn: Microsoft.KernelMemory.AI.AzureOpenAI.Internals.ClientSequentialRetryPolicy[0] | ||
| Delay extracted from HTTP response: 21000 msecs | ||
| */ |
22 changes: 22 additions & 0 deletions
22
extensions/AzureOpenAI/AzureOpenAI.FunctionalTests/Startup.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| /* IMPORTANT: the Startup class must be at the root of the namespace and | ||
| * the namespace must match exactly (required by Xunit.DependencyInjection) */ | ||
|
|
||
| namespace Microsoft.AzureOpenAI.FunctionalTests; | ||
|
|
||
| public class Startup | ||
| { | ||
| public void ConfigureHost(IHostBuilder hostBuilder) | ||
| { | ||
| var config = new ConfigurationBuilder() | ||
| .AddJsonFile("appsettings.json") | ||
| .AddJsonFile("appsettings.development.json", optional: true) | ||
| .AddJsonFile("appsettings.Development.json", optional: true) | ||
| .AddUserSecrets<Startup>() | ||
| .AddEnvironmentVariables() | ||
| .Build(); | ||
|
|
||
| hostBuilder.ConfigureHostConfiguration(builder => builder.AddConfiguration(config)); | ||
| } | ||
| } |
12 changes: 12 additions & 0 deletions
12
extensions/AzureOpenAI/AzureOpenAI.FunctionalTests/appsettings.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| { | ||
| "KernelMemory": { | ||
| "Services": { | ||
| "AzureOpenAIEmbedding": { | ||
| "Auth": "AzureIdentity", // "ApiKey" or "AzureIdentity" | ||
| "Endpoint": "https://<...>.openai.azure.com/", | ||
| "APIKey": "", | ||
| "Deployment": "" | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
File renamed without changes.
File renamed without changes.
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
File renamed without changes.
46 changes: 46 additions & 0 deletions
46
extensions/AzureOpenAI/AzureOpenAI/Internals/SingleAuthorizationHeaderPolicy.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| // Copyright (c) Microsoft. All rights reserved. | ||
|
|
||
| using System.ClientModel.Primitives; | ||
| using System.Collections.Generic; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace Microsoft.KernelMemory.AI.AzureOpenAI.Internals; | ||
|
|
||
| /// <summary> | ||
| /// Bug fix: Remove duplicate Authorization headers from the request. | ||
| /// See https://github.com/Azure/azure-sdk-for-net/issues/46109 | ||
| /// </summary> | ||
| internal sealed class SingleAuthorizationHeaderPolicy : PipelinePolicy | ||
| { | ||
| public override void Process(PipelineMessage message, IReadOnlyList<PipelinePolicy> pipeline, int currentIndex) | ||
| { | ||
| RemoveDuplicateHeader(message.Request.Headers); | ||
| ProcessNext(message, pipeline, currentIndex); | ||
| } | ||
|
|
||
| public override async ValueTask ProcessAsync(PipelineMessage message, IReadOnlyList<PipelinePolicy> pipeline, int currentIndex) | ||
| { | ||
| RemoveDuplicateHeader(message.Request.Headers); | ||
| await ProcessNextAsync(message, pipeline, currentIndex).ConfigureAwait(false); | ||
| } | ||
|
|
||
| private static void RemoveDuplicateHeader(PipelineRequestHeaders headers) | ||
| { | ||
| if (!headers.TryGetValues("Authorization", out var headerValues) || headerValues == null) | ||
| { | ||
| return; | ||
| } | ||
|
|
||
| using var enumerator = headerValues.GetEnumerator(); | ||
|
|
||
| if (!enumerator.MoveNext()) { return; } | ||
|
|
||
| var firstValue = enumerator.Current; | ||
|
|
||
| // Check if there’s more than one value | ||
| if (enumerator.MoveNext()) | ||
| { | ||
| headers.Set("Authorization", firstValue); | ||
| } | ||
| } | ||
| } |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters