Allow to enable HTTP authorization on memory web service (#161)

When deploying the memory service, the instance and its data should
protected requiring a client to be authorized.

This change allows to enable authorization on all the endpoints,
requiring a client to pass one of two valid API keys.
When enabled, the API returns 401 is the API key is not provided, and
403 if an invalid API key is provided.
By default API keys are passed using the `Authorization` header,
although this is configurable.

New configuration block in `appsettings.json`:

```json
"ServiceAuthorization": {
    // Whether clients must provide some credentials to interact with the HTTP API
    "Enabled": false,

    // Currently "APIKey" is the only type supported
    "AuthenticationType": "APIKey",

    // HTTP header name to check
    "HttpHeaderName": "Authorization",

    // Define two separate API Keys, to allow key rotation. Both are active.
    // Keys must be different and case-sensitive, and at least 32 chars long.
    // Contain only alphanumeric chars and allowed symbols.
    // Symbols allowed: . _ - (dot, underscore, minus).
    "AccessKey1": "",
    "AccessKey2": "",
},
```

The code loading the two API keys is in Service/Program.cs, and can
easily be customized to fetch keys from KeyVault or similar solution.

OpenAPI Swagger UI updated to support API keys.

Author: dluc

dotnet/ClientLib/MemoryWebClient.cs

@@ -19,14 +19,25 @@ public class MemoryWebClient : IKernelMemory
 {
     private readonly HttpClient _client;
 
-    public MemoryWebClient(string endpoint) : this(endpoint, new HttpClient())
+    public MemoryWebClient(string endpoint, string apiKey = "", string apiKeyHeader = "Authorization")
+        : this(endpoint, new HttpClient(), apiKey, apiKeyHeader)
     {
     }
 
-    public MemoryWebClient(string endpoint, HttpClient client)
+    public MemoryWebClient(string endpoint, HttpClient client, string apiKey = "", string apiKeyHeader = "Authorization")
     {
         this._client = client;
         this._client.BaseAddress = new Uri(endpoint);
+
+        if (!string.IsNullOrEmpty(apiKey))
+        {
+            if (string.IsNullOrEmpty(apiKeyHeader))
+            {
+                throw new KernelMemoryException("The name of the HTTP header to pass the API Key is empty");
+            }
+
+            this._client.DefaultRequestHeaders.Add(apiKeyHeader, apiKey);
+        }
     }
 
     /// <inheritdoc />
@@ -300,7 +311,10 @@ public async Task<MemoryAnswer> AskAsync(
 
     #region private
 
-    private async Task<string> ImportInternalAsync(string index, DocumentUploadRequest uploadRequest, CancellationToken cancellationToken)
+    private async Task<string> ImportInternalAsync(
+        string index,
+        DocumentUploadRequest uploadRequest,
+        CancellationToken cancellationToken)
     {
         // Populate form with values and files from disk
         using MultipartFormDataContent formData = new();

dotnet/CoreLib/Configuration/KernelMemoryConfig.cs

@@ -100,6 +100,11 @@ public class RetrievalConfig
     /// </summary>
     public string ImageOcrType { get; set; } = string.Empty;
 
+    /// <summary>
+    /// HTTP service authorization settings.
+    /// </summary>
+    public ServiceAuthorizationConfig ServiceAuthorization { get; set; } = new();
+
     /// <summary>
     /// Settings for the upload of documents and memory creation/update.
     /// </summary>

dotnet/CoreLib/Configuration/ServiceAuthorizationConfig.cs

@@ -0,0 +1,88 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Linq;
+
+namespace Microsoft.KernelMemory.Configuration;
+
+public class ServiceAuthorizationConfig
+{
+    public const string APIKeyAuthType = "APIKey";
+    public const int AccessKeyMinLength = 32;
+
+    /// <summary>
+    /// Whether clients must provide some credentials to interact with the HTTP API.
+    /// </summary>
+    public bool Enabled { get; set; } = false;
+
+    /// <summary>
+    /// Currently "APIKey" is the only type supported
+    /// </summary>
+    public string AuthenticationType { get; set; } = APIKeyAuthType;
+
+    /// <summary>
+    /// HTTP header name to check for the access key
+    /// </summary>
+    public string HttpHeaderName { get; set; } = "Authorization";
+
+    /// <summary>
+    /// Access Key 1. Alphanumeric, "-" "_" "." allowed. Min 32 chars.
+    /// Two different keys are always active, to allow secrets rotation.
+    /// </summary>
+    public string AccessKey1 { get; set; } = "";
+
+    /// <summary>
+    /// Access Key 2. Alphanumeric, "-" "_" "." allowed. Min 32 chars.
+    /// Two different keys are always active, to allow secrets rotation.
+    /// </summary>
+    public string AccessKey2 { get; set; } = "";
+
+    public void Validate()
+    {
+        if (!this.Enabled)
+        {
+            return;
+        }
+
+        if (this.AuthenticationType != APIKeyAuthType)
+        {
+            throw new ConfigurationException($"The authorization type '{this.AuthenticationType}' is not supported. Please use '{APIKeyAuthType}'.");
+        }
+
+        if (string.IsNullOrWhiteSpace(this.HttpHeaderName))
+        {
+            throw new ConfigurationException("The HTTP header name cannot be empty");
+        }
+
+        ValidateAccessKey(this.AccessKey1, 1);
+        ValidateAccessKey(this.AccessKey2, 2);
+
+        if (string.Equals(this.AccessKey1, this.AccessKey2, StringComparison.OrdinalIgnoreCase))
+        {
+            throw new ConfigurationException("Access keys 1 and 2 are the same. Please use two different keys.");
+        }
+    }
+
+    private static void ValidateAccessKey(string key, int keyNumber)
+    {
+        if (string.IsNullOrEmpty(key))
+        {
+            throw new ConfigurationException($"Memory Web Service Access Key {keyNumber} is empty.");
+        }
+
+        if (key.Length < AccessKeyMinLength)
+        {
+            throw new ConfigurationException($"Memory Web Service Access Key {keyNumber} is too short, use at least {AccessKeyMinLength} chars.");
+        }
+
+        if (!key.All(IsValidChar))
+        {
+            throw new ConfigurationException($"Memory Web Service Access Key {keyNumber} contains some invalid chars (allowed: A-B, a-b, 0-9, '.', '_', '-')");
+        }
+    }
+
+    private static bool IsValidChar(char c)
+    {
+        return char.IsLetterOrDigit(c) || c == '.' || c == '_' || c == '-';
+    }
+}

dotnet/CoreLib/KernelMemoryBuilder.cs

@@ -236,7 +236,7 @@ public MemoryService BuildAsyncClient()
         return new MemoryService(orchestrator, searchClient);
     }
 
-    public static IKernelMemory BuildWebClient(string endpoint)
+    public static IKernelMemory BuildWebClient(string endpoint, string apiKey = "", string apiKeyHeader = "Authorization")
     {
         if (string.IsNullOrWhiteSpace(endpoint))
         {
@@ -253,7 +253,7 @@ public static IKernelMemory BuildWebClient(string endpoint)
             throw new ConfigurationException("The endpoint is incomplete");
         }
 
-        return new MemoryWebClient(endpoint);
+        return new MemoryWebClient(endpoint: endpoint, apiKey: apiKey, apiKeyHeader: apiKeyHeader);
     }
 
     public KernelMemoryBuilder WithoutDefaultHandlers()

dotnet/Service/Auth/HttpAuthHandler.cs

@@ -0,0 +1,39 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Threading.Tasks;
+using Microsoft.AspNetCore.Http;
+using Microsoft.KernelMemory.Configuration;
+
+namespace Microsoft.KernelMemory.Service.Auth;
+
+public class HttpAuthEndpointFilter : IEndpointFilter
+{
+    private readonly ServiceAuthorizationConfig _config;
+
+    public HttpAuthEndpointFilter(ServiceAuthorizationConfig config)
+    {
+        this._config = config;
+    }
+
+    public async ValueTask<object?> InvokeAsync(
+        EndpointFilterInvocationContext context,
+        EndpointFilterDelegate next)
+    {
+        if (this._config.Enabled)
+        {
+            if (!context.HttpContext.Request.Headers.TryGetValue(this._config.HttpHeaderName, out var apiKey))
+            {
+                return Results.Problem(detail: "API Key missing", statusCode: 401);
+            }
+
+            if (!string.Equals(apiKey, this._config.AccessKey1, StringComparison.Ordinal)
+                && !string.Equals(apiKey, this._config.AccessKey2, StringComparison.Ordinal))
+            {
+                return Results.Problem(detail: "Invalid API Key", statusCode: 403);
+            }
+        }
+
+        return await next(context);
+    }
+}