Throw exception when mixing volatile and persistent data

Directory.Build.props

@@ -2,7 +2,7 @@
 <Project>
     <PropertyGroup>
         <!-- Central version prefix - applies to all nuget packages. -->
-        <Version>0.95.0</Version>
+        <Version>0.96.0</Version>
 
         <!-- C# lang version, https://learn.microsoft.com/dotnet/csharp/whats-new -->
         <LangVersion>12</LangVersion>

examples/001-dotnet-WebClient/Properties/launchSettings.json

@@ -0,0 +1,11 @@
+{
+  "profiles": {
+    "console": {
+      "commandName": "Project",
+      "launchBrowser": false,
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}

examples/002-dotnet-Serverless/Properties/launchSettings.json

@@ -0,0 +1,11 @@
+{
+  "profiles": {
+    "console": {
+      "commandName": "Project",
+      "launchBrowser": false,
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}

examples/007-dotnet-serverless-azure/Properties/launchSettings.json

@@ -0,0 +1,11 @@
+{
+  "profiles": {
+    "console": {
+      "commandName": "Project",
+      "launchBrowser": false,
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}

service/Abstractions/IKernelMemoryBuilder.cs

@@ -26,17 +26,20 @@ public interface IKernelMemoryBuilder
     /// <summary>
     /// Build the memory instance, using defaults and the provided dependencies
     /// and overrides. Depending on the dependencies provided, the resulting
-    /// memory might use either an synchronous or asynchronous pipeline.
+    /// memory might use either a synchronous or asynchronous pipeline.
     /// </summary>
-    public IKernelMemory Build();
+    /// <param name="options">Options for the build process</param>
+    /// <returns>A memory instance</returns>
+    public IKernelMemory Build(KernelMemoryBuilderBuildOptions? options = null);
 
     /// <summary>
     /// Build a specific type of memory instance, e.g. explicitly choosing
     /// between synchronous or asynchronous (queue based) pipeline.
     /// </summary>
     /// <typeparam name="T">Type of memory derived from IKernelMemory</typeparam>
+    /// <param name="options">Options for the build process</param>
     /// <returns>A memory instance</returns>
-    public T Build<T>() where T : class, IKernelMemory;
+    public T Build<T>(KernelMemoryBuilderBuildOptions? options = null) where T : class, IKernelMemory;
 
     /// <summary>
     /// Add a singleton to the builder service collection pool.

service/Abstractions/KernelMemoryBuilderBuildOptions.cs

@@ -0,0 +1,8 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+namespace Microsoft.KernelMemory;
+
+public sealed class KernelMemoryBuilderBuildOptions
+{
+    public bool AllowMixingVolatileAndPersistentData { get; set; } = false;
+}

service/Core/KernelMemoryBuilder.cs

@@ -2,6 +2,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Linq;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.KernelMemory.AI;
 using Microsoft.KernelMemory.AppBuilders;
@@ -116,33 +117,33 @@ public KernelMemoryBuilder(IServiceCollection? hostServiceCollection = null)
     }
 
     ///<inheritdoc />
-    public IKernelMemory Build()
+    public IKernelMemory Build(KernelMemoryBuilderBuildOptions? options = null)
     {
         var type = this.GetBuildType();
         switch (type)
         {
             case ClientTypes.SyncServerless:
-                return this.BuildServerlessClient();
+                return this.BuildServerlessClient(options);
 
             case ClientTypes.AsyncService:
-                return this.BuildAsyncClient();
+                return this.BuildAsyncClient(options);
 
             case ClientTypes.Undefined:
-                throw new KernelMemoryException("Missing dependencies or insufficient configuration provided. " +
-                                                "Try using With...() methods " +
-                                                $"and other configuration methods before calling {nameof(this.Build)}(...)");
+                throw new ConfigurationException("Missing dependencies or insufficient configuration provided. " +
+                                                 "Try using With...() methods " +
+                                                 $"and other configuration methods before calling {nameof(this.Build)}(...)");
 
             default:
                 throw new ArgumentOutOfRangeException(nameof(type), $"Unsupported memory type '{type}'");
         }
     }
 
     ///<inheritdoc />
-    public T Build<T>() where T : class, IKernelMemory
+    public T Build<T>(KernelMemoryBuilderBuildOptions? options = null) where T : class, IKernelMemory
     {
         if (typeof(T) == typeof(MemoryServerless))
         {
-            if (this.BuildServerlessClient() is not T result)
+            if (this.BuildServerlessClient(options) is not T result)
             {
                 throw new InvalidOperationException($"Unable to instantiate '{typeof(MemoryServerless)}'. The instance is NULL.");
             }
@@ -152,7 +153,7 @@ public T Build<T>() where T : class, IKernelMemory
 
         if (typeof(T) == typeof(MemoryService))
         {
-            if (this.BuildAsyncClient() is not T result)
+            if (this.BuildAsyncClient(options) is not T result)
             {
                 throw new InvalidOperationException($"Unable to instantiate '{typeof(MemoryService)}'. The instance is NULL.");
             }
@@ -226,7 +227,7 @@ private static void CopyServiceCollection(
         }
     }
 
-    private MemoryServerless BuildServerlessClient()
+    private MemoryServerless BuildServerlessClient(KernelMemoryBuilderBuildOptions? options)
     {
         try
         {
@@ -240,6 +241,7 @@ private MemoryServerless BuildServerlessClient()
 
             // Recreate the service provider, in order to have the latest dependencies just configured
             serviceProvider = this._memoryServiceCollection.BuildServiceProvider();
+            this.CheckStoragePersistence(options, serviceProvider);
             var memoryClientInstance = ActivatorUtilities.CreateInstance<MemoryServerless>(serviceProvider);
 
             // Load handlers in the memory client
@@ -257,7 +259,7 @@ private MemoryServerless BuildServerlessClient()
         }
     }
 
-    private MemoryService BuildAsyncClient()
+    private MemoryService BuildAsyncClient(KernelMemoryBuilderBuildOptions? options)
     {
         // Add handlers to DI service collection
         if (this._useDefaultHandlers)
@@ -282,9 +284,62 @@ private MemoryService BuildAsyncClient()
 
         // Recreate the service provider, in order to have the latest dependencies just configured
         serviceProvider = this._memoryServiceCollection.BuildServiceProvider();
+        this.CheckStoragePersistence(options, serviceProvider);
         return ActivatorUtilities.CreateInstance<MemoryService>(serviceProvider);
     }
 
+    private void CheckStoragePersistence(KernelMemoryBuilderBuildOptions? options, ServiceProvider serviceProvider)
+    {
+        if (options is { AllowMixingVolatileAndPersistentData: true }) { return; }
+
+        ServiceDescriptor docStoreType = this._memoryServiceCollection.Last<ServiceDescriptor>(x => x.ServiceType == typeof(IDocumentStorage));
+        ServiceDescriptor memStoreType = this._memoryServiceCollection.Last<ServiceDescriptor>(x => x.ServiceType == typeof(IMemoryDb));
+        SimpleFileStorageConfig? simpleFileStorageConfig = serviceProvider.GetService<SimpleFileStorageConfig>();
+        SimpleVectorDbConfig? simpleVectorDbConfig = serviceProvider.GetService<SimpleVectorDbConfig>();
+        SimpleTextDbConfig? simpleTextDbConfig = serviceProvider.GetService<SimpleTextDbConfig>();
+
+        bool persistentDocStore = docStoreType.ImplementationType != typeof(SimpleFileStorage) || simpleFileStorageConfig?.StorageType == FileSystemTypes.Disk;
+        bool persistentMemStore = memStoreType.ImplementationType != typeof(SimpleVectorDb) && memStoreType.ImplementationType != typeof(SimpleTextDb)
+                                  || (memStoreType.ImplementationType == typeof(SimpleVectorDb) && simpleVectorDbConfig?.StorageType == FileSystemTypes.Disk)
+                                  || (memStoreType.ImplementationType == typeof(SimpleTextDb) && simpleTextDbConfig?.StorageType == FileSystemTypes.Disk);
+
+        // No error if both services are volatile or persistent,
+        if (persistentMemStore == persistentDocStore)
+        {
+            return;
+        }
+
+        // Show a service name for a helpful error message
+        var docStoreName = docStoreType.ImplementationType != typeof(SimpleFileStorage)
+            ? docStoreType.ImplementationType?.Name
+            : $"{nameof(SimpleFileStorage)} {simpleFileStorageConfig?.StorageType}";
+
+        var memStoreName = memStoreType.ImplementationType != typeof(SimpleVectorDb) && memStoreType.ImplementationType != typeof(SimpleTextDb)
+            ? memStoreType.ImplementationType?.Name
+            : memStoreType.ImplementationType == typeof(SimpleVectorDb)
+                ? $"{nameof(SimpleVectorDb)} {simpleVectorDbConfig?.StorageType}"
+                : $"{nameof(SimpleTextDb)} {simpleTextDbConfig?.StorageType}";
+
+        // Different error message depending on which service is volatile
+        if (persistentMemStore && !persistentDocStore)
+        {
+            throw new ConfigurationException(
+                $"Using a persistent memory store ({memStoreName}) with a volatile document store ({docStoreName}) will lead to duplicate memory records over multiple executions. " +
+                $"Set up Kernel Memory to use a persistent document store like Azure Blobs, AWS S3, {nameof(SimpleFileStorage)} on disk, etc. " +
+                $"Otherwise, use {nameof(KernelMemoryBuilderBuildOptions)}.{nameof(KernelMemoryBuilderBuildOptions.AllowMixingVolatileAndPersistentData)} " +
+                "to suppress this exception when invoking kernelMemoryBuilder.Build(<options here>). ");
+        }
+
+        if (persistentDocStore && !persistentMemStore)
+        {
+            throw new ConfigurationException(
+                $"Using a volatile memory store ({memStoreName}) with a persistent document store ({docStoreName}) will lead to missing memory records over multiple executions. " +
+                $"Set up Kernel Memory to use a persistent memory store like Azure AI Search, Postgres, Qdrant, {nameof(SimpleVectorDb)} on disk, etc. " +
+                $"Otherwise, use {nameof(KernelMemoryBuilderBuildOptions)}.{nameof(KernelMemoryBuilderBuildOptions.AllowMixingVolatileAndPersistentData)} " +
+                "to suppress this exception when invoking kernelMemoryBuilder.Build(<options here>). ");
+        }
+    }
+
     private KernelMemoryBuilder CompleteServerlessClient(ServiceProvider serviceProvider)
     {
         this.UseDefaultSearchClientIfNecessary(serviceProvider);

service/Service/Program.cs

@@ -60,7 +60,7 @@ public static void Main(string[] args)
         // Run `dotnet run setup` to run this code and set up the service
         if (new[] { "setup", "-setup", "config" }.Contains(args.FirstOrDefault(), StringComparer.OrdinalIgnoreCase))
         {
-            InteractiveSetup.Main.InteractiveSetup(args.Skip(1).ToArray());
+            InteractiveSetup.Program.Main(args.Skip(1).ToArray());
         }
 
         // *************************** APP BUILD *******************************

tools/InteractiveSetup/AppSettings.cs

@@ -1,6 +1,7 @@
 // Copyright (c) Microsoft. All rights reserved.
 
 using System;
+using System.Collections.Generic;
 using System.IO;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
@@ -37,6 +38,11 @@ public static void Change(Action<KernelMemoryConfig> configChanges)
         File.WriteAllText(DevelopmentSettingsFile, json);
     }
 
+    public static void AddService(string serviceName, Dictionary<string, object> config)
+    {
+        Change(x => { x.Services.Add(serviceName, config); });
+    }
+
     public static void GlobalChange(Action<JObject> configChanges)
     {
         CreateFileIfNotExists();

tools/InteractiveSetup/Context.cs

@@ -27,8 +27,10 @@ internal sealed class Context
     public readonly BoundedBoolean CfgAzureOpenAIText = new();
     public readonly BoundedBoolean CfgAzureOpenAIEmbedding = new();
     public readonly BoundedBoolean CfgOpenAI = new();
-    public readonly BoundedBoolean CfgLlamaSharp = new();
-    public readonly BoundedBoolean CfgOllama = new();
+    public readonly BoundedBoolean CfgLlamaSharpEmbedding = new();
+    public readonly BoundedBoolean CfgLlamaSharpText = new();
+    public readonly BoundedBoolean CfgOllamaEmbedding = new();
+    public readonly BoundedBoolean CfgOllamaText = new();
     public readonly BoundedBoolean CfgAzureAIDocIntel = new();
 
     // Vectors

tools/InteractiveSetup/InteractiveSetup.csproj

@@ -2,6 +2,7 @@
 
     <PropertyGroup>
         <TargetFramework>net8.0</TargetFramework>
+        <OutputType>Exe</OutputType>
         <AssemblyName>Microsoft.KernelMemory.InteractiveSetup</AssemblyName>
         <RootNamespace>Microsoft.KernelMemory.InteractiveSetup</RootNamespace>
         <NoWarn>$(NoWarn);CA1031;CA1303;CA1724;</NoWarn>

tools/InteractiveSetup/Main.cs → tools/InteractiveSetup/Program.cs

@@ -7,9 +7,9 @@
 
 namespace Microsoft.KernelMemory.InteractiveSetup;
 
-public static class Main
+public static class Program
 {
-    public static void InteractiveSetup(string[] args)
+    public static void Main(string[] args)
     {
         var ctx = new Context();
 
@@ -207,7 +207,19 @@ private static void EmbeddingGeneratorSetup(Context ctx)
                             ? [x.Retrieval.EmbeddingGeneratorType]
                             : [];
                     });
-                    ctx.CfgOllama.Value = true;
+                    ctx.CfgOllamaEmbedding.Value = true;
+                }),
+
+                new("LlamaSharp library", config.Retrieval.EmbeddingGeneratorType == "LlamaSharp", () =>
+                {
+                    AppSettings.Change(x =>
+                    {
+                        x.Retrieval.EmbeddingGeneratorType = "LlamaSharp";
+                        x.DataIngestion.EmbeddingGeneratorTypes = ctx.CfgEmbeddingGenerationEnabled.Value
+                            ? [x.Retrieval.EmbeddingGeneratorType]
+                            : [];
+                    });
+                    ctx.CfgLlamaSharpEmbedding.Value = true;
                 }),
 
                 new("None/Custom (manually set with code)", string.IsNullOrEmpty(config.Retrieval.EmbeddingGeneratorType), () =>
@@ -248,13 +260,13 @@ private static void TextGeneratorTypeSetup(Context ctx)
                 new("Ollama service", config.TextGeneratorType == "Ollama", () =>
                 {
                     AppSettings.Change(x => { x.TextGeneratorType = "Ollama"; });
-                    ctx.CfgOllama.Value = true;
+                    ctx.CfgOllamaText.Value = true;
                 }),
 
                 new("LlamaSharp library", config.TextGeneratorType == "LlamaSharp", () =>
                 {
                     AppSettings.Change(x => { x.TextGeneratorType = "LlamaSharp"; });
-                    ctx.CfgLlamaSharp.Value = true;
+                    ctx.CfgLlamaSharpText.Value = true;
                 }),
 
                 new("None/Custom (manually set with code)", string.IsNullOrEmpty(config.TextGeneratorType), () =>

tools/InteractiveSetup/Properties/launchSettings.json

@@ -0,0 +1,11 @@
+{
+  "profiles": {
+    "console": {
+      "commandName": "Project",
+      "launchBrowser": false,
+      "environmentVariables": {
+        "ASPNETCORE_ENVIRONMENT": "Development"
+      }
+    }
+  }
+}

tools/InteractiveSetup/Services/AWSS3.cs

@@ -24,6 +24,7 @@ public static void Setup(Context ctx)
                 { "BucketName", "" },
                 { "Endpoint", "https://s3.amazonaws.com" },
             };
+            AppSettings.AddService(ServiceName, config);
         }
 
         // Required to avoid exceptions. "Endpoint" is optional and not defined in appsettings.json

tools/InteractiveSetup/Services/AzureAIDocIntel.cs

@@ -22,6 +22,7 @@ public static void Setup(Context ctx)
                 { "Auth", "ApiKey" },
                 { "APIKey", "" },
             };
+            AppSettings.AddService(ServiceName, config);
         }
 
         SetupUI.AskQuestionWithOptions(new QuestionWithOptions

tools/InteractiveSetup/Services/AzureAISearch.cs

@@ -24,6 +24,7 @@ public static void Setup(Context ctx, bool force = false)
                 { "UseHybridSearch", false },
                 { "UseStickySessions", false }
             };
+            AppSettings.AddService(ServiceName, config);
         }
 
         SetupUI.AskQuestionWithOptions(new QuestionWithOptions

tools/InteractiveSetup/Services/AzureBlobs.cs

@@ -23,6 +23,7 @@ public static void Setup(Context ctx)
                 { "Auth", "ConnectionString" },
                 { "ConnectionString", "" },
             };
+            AppSettings.AddService(ServiceName, config);
         }
 
         SetupUI.AskQuestionWithOptions(new QuestionWithOptions

tools/InteractiveSetup/Services/AzureOpenAIEmbedding.cs

@@ -25,6 +25,7 @@ public static void Setup(Context ctx, bool force = false)
                 { "Auth", "ApiKey" },
                 { "APIKey", "" },
             };
+            AppSettings.AddService(ServiceName, config);
         }
 
         SetupUI.AskQuestionWithOptions(new QuestionWithOptions

tools/InteractiveSetup/Services/AzureOpenAIText.cs

@@ -25,6 +25,7 @@ public static void Setup(Context ctx, bool force = false)
                 { "Auth", "ApiKey" },
                 { "APIKey", "" },
             };
+            AppSettings.AddService(ServiceName, config);
         }
 
         SetupUI.AskQuestionWithOptions(new QuestionWithOptions