Refactoring: merge Fragment into Chunk

Author: dluc

.github/_typos.toml

@@ -17,7 +17,8 @@ extend-exclude = [
     "appsettings.Development.json",
     "appsettings.*.json.*",
     "AzureAISearchFilteringTest.cs",
-    "KernelMemory.sln.DotSettings"
+    "KernelMemory.sln.DotSettings",
+    "doc1.txt",
 ]
 
 [default.extend-words]

KernelMemory.sln

@@ -720,7 +720,6 @@ Global
 		{FD1EB2C1-581E-4EB8-AF4A-BC4773453226}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{FD1EB2C1-581E-4EB8-AF4A-BC4773453226}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{FD1EB2C1-581E-4EB8-AF4A-BC4773453226}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{FD1EB2C1-581E-4EB8-AF4A-BC4773453226}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

docs/how-to/custom-partitioning.md

@@ -59,7 +59,6 @@ For example, with small models supporting up to 256 tokens, something like this
       ...
       "TextPartitioning": {
         "MaxTokensPerParagraph": 256,
-        "MaxTokensPerLine": 256,
         "OverlappingTokens": 50
       },
   ...
@@ -74,7 +73,6 @@ var memory = new KernelMemoryBuilder()
         new TextPartitioningOptions
         {
             MaxTokensPerParagraph = 256,
-            MaxTokensPerLine = 256,
             OverlappingTokens = 50
         })
     .Build<MemoryServerless>();

examples/102-dotnet-custom-partitioning-options/Program.cs

@@ -7,11 +7,9 @@
     .WithOpenAIDefaults(Environment.GetEnvironmentVariable("OPENAI_API_KEY")!)
     .WithCustomTextPartitioningOptions(new TextPartitioningOptions
     {
-        // Max 99 tokens per sentence
-        MaxTokensPerLine = 99,
-        // When sentences are merged into paragraphs (aka partitions), stop at 299 tokens
+        // When splitting text into chunks (aka partitions), stop at 299 tokens
         MaxTokensPerParagraph = 299,
-        // Each paragraph contains the last 47 tokens from the previous one
+        // Each chunk contains the last 47 tokens from the previous one
         OverlappingTokens = 47,
     })
     .Build<MemoryServerless>();

examples/108-dotnet-custom-content-decoders/Program.cs

@@ -91,7 +91,7 @@ public Task<FileContent> DecodeAsync(Stream data, CancellationToken cancellation
         foreach (Page? page in pdfDocument.GetPages().Where(x => x != null))
         {
             string pageContent = (ContentOrderTextExtractor.GetText(page, options) ?? string.Empty).ReplaceLineEndings(" ");
-            result.Sections.Add(new Chunk(page.Number, pageContent, Chunk.Meta(sentencesAreComplete: false)));
+            result.Sections.Add(new Chunk(pageContent, page.Number, Chunk.Meta(sentencesAreComplete: false)));
         }
 
         return Task.FromResult(result);

examples/207-dotnet-expanding-chunks-on-retrieval/Program.cs

@@ -40,7 +40,7 @@ public static class Program
     public static async Task Main()
     {
         // Partition input text in chunks of 100 tokens
-        const int PartitionSize = 100;
+        const int Chunksize = 100;
 
         // Search settings
         const string Query = "astrobiology";
@@ -59,8 +59,7 @@ public static async Task Main()
         // Customize memory records size (in tokens)
         var textPartitioningOptions = new TextPartitioningOptions
         {
-            MaxTokensPerParagraph = PartitionSize,
-            MaxTokensPerLine = PartitionSize,
+            MaxTokensPerParagraph = Chunksize,
             OverlappingTokens = 0,
         };
 

examples/210-KM-without-builder/appsettings.json

@@ -154,14 +154,11 @@
       "ImageOcrType": "None",
       // Partitioning / Chunking settings
       // How does the partitioning work?
-      // * Given a document, text is extracted, and text is split in sentences, called "lines of text".
-      // * Sentences are merged into paragraphs, called "partitions".
-      // * For each partition, one (potentially more) memory is generated.
+      // * Given a document, text is extracted, and text is split in tokens.
+      // * Tokens are merged into chunks, called "partitions", sometimes called "paragraphs"
+      // * For each chunk, one (potentially more) memory is generated.
       "TextPartitioning": {
-        // Maximum length of lines of text (aka sentences), in tokens. Tokens depend on the LLM in use.
-        // Sentences are grouped into paragraphs, see the next setting.
-        "MaxTokensPerLine": 300,
-        // Maximum length of paragraphs (aka partitions), in tokens. Tokens depend on the LLM in use.
+        // Maximum length of chunks in tokens. Tokens depend on the LLM in use.
         "MaxTokensPerParagraph": 1000,
         // How many tokens from a paragraph to keep in the following paragraph.
         "OverlappingTokens": 100

extensions/Chunkers/Chunkers.UnitTests/Chunkers.UnitTests.csproj

@@ -35,6 +35,10 @@
         <Content Include="doc1.txt">
             <CopyToOutputDirectory>Always</CopyToOutputDirectory>
         </Content>
+        <None Remove="doc2.md" />
+        <Content Include="doc2.md">
+            <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+        </Content>
     </ItemGroup>
 
 </Project>

extensions/Chunkers/Chunkers.UnitTests/MarkDownChunkerManualTest.cs

@@ -0,0 +1,50 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Diagnostics;
+using Microsoft.KernelMemory.AI;
+using Microsoft.KernelMemory.Chunkers;
+using Microsoft.KM.TestHelpers;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Microsoft.Chunkers.UnitTests;
+
+public class MarkDownChunkerManualTest(ITestOutputHelper output) : BaseUnitTestCase(output)
+{
+    [Fact]
+    [Trait("Category", "UnitTest")]
+    [Trait("Category", "Chunking")]
+    [Trait("Category", "Manual")]
+    public void ItSplitsMarkdownInASensibleWay()
+    {
+        // Arrange
+        string text = File.ReadAllText("doc2.md");
+        text = $"{text}{text}";
+
+        // Act
+        var w = new Stopwatch();
+        w.Start();
+        var chunks = new MarkDownChunker(new CL100KTokenizer()).Split(text, new MarkDownChunkerOptions { MaxTokensPerChunk = 600, Overlap = 60 });
+        w.Stop();
+
+        Console.WriteLine($"Text length: {text.Length:N0} chars");
+        Console.WriteLine($"Chunks: {chunks.Count}");
+        Console.WriteLine($"Time: {w.ElapsedMilliseconds:N0} ms");
+
+        // Assert
+        Assert.NotEmpty(chunks);
+        DebugChunks(chunks, new CL100KTokenizer());
+    }
+
+    private static void DebugChunks(IEnumerable<string> chunks, ITextTokenizer tokenizer)
+    {
+        var list = chunks.ToList();
+
+        for (int index = 0; index < list.Count; index++)
+        {
+            Console.WriteLine($"************************* {index}: [{tokenizer.CountTokens(list[index])} tokens] *****************************************");
+            Console.WriteLine(list[index]);
+            Console.WriteLine("***********************************************************************************");
+        }
+    }
+}

extensions/Chunkers/Chunkers.UnitTests/MarkDownChunkerTests.cs

@@ -3,7 +3,7 @@
 using Microsoft.Chunkers.UnitTests.Helpers;
 using Microsoft.KernelMemory.AI;
 using Microsoft.KernelMemory.Chunkers;
-using Microsoft.KernelMemory.Chunkers.internals;
+using Microsoft.KernelMemory.DataFormats;
 using Microsoft.KM.TestHelpers;
 using Xunit;
 using Xunit.Abstractions;
@@ -375,7 +375,7 @@ private static void DebugChunks(IEnumerable<string> chunks, ITextTokenizer token
         Console.WriteLine("----------------------------------");
     }
 
-    private static void DebugFragments(List<Fragment> fragments)
+    private static void DebugFragments(List<Chunk> fragments)
     {
         if (fragments.Count == 0)
         {
@@ -384,8 +384,7 @@ private static void DebugFragments(List<Fragment> fragments)
 
         for (int index = 0; index < fragments.Count; index++)
         {
-            Fragment token = fragments[index];
-            Console.WriteLine($"- {index}: Value: \"{token.Content}\"");
+            Console.WriteLine($"- {index}: Value: \"{fragments[index].Content}\"");
         }
     }