VectorIndexDefinition: Adds Support for Partitioned DiskANN

Microsoft.Azure.Cosmos/src/Fluent/Settings/VectorIndexDefinition.cs

@@ -55,6 +55,47 @@ public VectorIndexDefinition<T> Path(
             return this;
         }
 
+        /// <summary>
+        /// Add a path to the current <see cref="VectorIndexPath"/> definition with a particular set of <see cref="VectorIndexType"/>s.
+        /// </summary>
+        /// <param name="quantizationByteSize">The number of bytes used in product quantization of the vectors. A larger value may result
+        /// in better recall for vector searches at the expense of latency. This applies to index types DiskANN and quantizedFlat</param>
+        /// <returns>An instance of the current <see cref="VectorIndexDefinition{T}"/>.</returns>
+        public VectorIndexDefinition<T> WithQuantizationByteSize(
+            int? quantizationByteSize)
+        {
+            this.vectorIndexPath.QuantizationByteSize = quantizationByteSize ?? throw new ArgumentNullException(nameof(quantizationByteSize));
+
+            return this;
+        }
+
+        /// <summary>
+        /// Add a path to the current <see cref="VectorIndexPath"/> definition with a particular set of <see cref="VectorIndexType"/>s.
+        /// </summary>
+        /// <param name="indexingSearchListSize">This represents the size of the candidate list of approximate neighbors stored while building the DiskANN index as part of the optimization processes.
+        /// Large values may improve recall at the expense of latency. This applies to index type DiskANN only.</param>
+        /// <returns>An instance of the current <see cref="VectorIndexDefinition{T}"/>.</returns>
+        public VectorIndexDefinition<T> WithIndexingSearchListSize(
+            int? indexingSearchListSize = 0)
+        {
+            this.vectorIndexPath.IndexingSearchListSize = indexingSearchListSize ?? throw new ArgumentNullException(nameof(indexingSearchListSize));
+
+            return this;
+        }
+
+        /// <summary>
+        /// Add a path to the current <see cref="VectorIndexPath"/> definition with a particular set of <see cref="VectorIndexType"/>s.
+        /// </summary>
+        /// <param name="vectorIndexShardKey">A string array containing the shard keys used for partitioning the vector indexes. This applies to index types DiskANN and quantizedFlat.</param>
+        /// <returns>An instance of the current <see cref="VectorIndexDefinition{T}"/>.</returns>
+        public VectorIndexDefinition<T> WithVectorIndexShardKey(
+            string[] vectorIndexShardKey = default)
+        {
+            this.vectorIndexPath.VectorIndexShardKey = vectorIndexShardKey ?? throw new ArgumentNullException(nameof(vectorIndexShardKey));
+
+            return this;
+        }
+
         /// <summary>
         /// Applies the current definition to the parent.
         /// </summary>

Microsoft.Azure.Cosmos/src/Resource/Settings/Embedding.cs

@@ -38,7 +38,7 @@ class Embedding : IEquatable<Embedding>
         /// Gets or sets a long integer representing the dimensions of a vector. 
         /// </summary>
         [JsonProperty(PropertyName = "dimensions")]
-        public ulong Dimensions { get; set; }
+        public int Dimensions { get; set; }
 
         /// <summary>
         /// Gets or sets the <see cref="Cosmos.DistanceFunction"/> which is used to calculate the respective distance between the vectors. 

Microsoft.Azure.Cosmos/src/Resource/Settings/VectorDataType.cs

@@ -15,12 +15,6 @@ namespace Microsoft.Azure.Cosmos
 #endif
     enum VectorDataType
     {
-        /// <summary>
-        /// Represent a float16 data type.
-        /// </summary>
-        [EnumMember(Value = "float16")]
-        Float16,
-
         /// <summary>
         /// Represent a float32 data type.
         /// </summary>

Microsoft.Azure.Cosmos/src/Resource/Settings/VectorIndexPath.cs

@@ -32,11 +32,16 @@ namespace Microsoft.Azure.Cosmos
     ///         },
     ///         {
     ///             "path": "/vector2",
-    ///             "type": "flat"
+    ///             "type": "quantizedFlat",
+    ///             "quantizationByteSize": 4,
+    ///             "vectorIndexShardKey": ["/Country/City", "ZipCode"]
     ///         },
     ///         {
     ///             "path": "/embeddings/vector",
-    ///             "type": "flat"
+    ///             "type": "DiskANN",
+    ///             "quantizationByteSize": 8,
+    ///             "indexingSearchListSize": 100,
+    ///             "vectorIndexShardKey": ["/Country/City", "ZipCode"]
     ///         }
     ///     ]
     /// }
@@ -62,6 +67,24 @@ sealed class VectorIndexPath
         [JsonConverter(typeof(StringEnumConverter))]
         public VectorIndexType Type { get; set; }
 
+        /// <summary>
+        /// Gets or sets the full path in a document used for vector indexing.
+        /// </summary>
+        [JsonProperty(PropertyName = "quantizationByteSize")]
+        public int? QuantizationByteSize { get; set; }
+
+        /// <summary>
+        /// Gets or sets the full path in a document used for vector indexing.
+        /// </summary>
+        [JsonProperty(PropertyName = "indexingSearchListSize")]
+        public int? IndexingSearchListSize { get; set; }
+
+        /// <summary>
+        /// Gets or sets the full path in a document used for vector indexing.
+        /// </summary>
+        [JsonProperty(PropertyName = "vectorIndexShardKey")]
+        public string[] VectorIndexShardKey { get; set; }
+
         /// <summary>
         /// This contains additional values for scenarios where the SDK is not aware of new fields. 
         /// This ensures that if resource is read and updated none of the fields will be lost in the process.

Microsoft.Azure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Fluent/ContainerSettingsTests.cs

@@ -585,10 +585,15 @@ await databaseForVectorEmbedding.DefineContainer(containerName, partitionKeyPath
                                 .Path(vector1Path, VectorIndexType.Flat)
                              .Attach()
                             .WithVectorIndex()
-                                .Path(vector2Path, VectorIndexType.Flat)
+                                .Path(vector2Path, VectorIndexType.QuantizedFlat)
+                                .WithQuantizationByteSize(4)
+                                .WithVectorIndexShardKey(new string[] { "/Country/City" })
                              .Attach()
                             .WithVectorIndex()
-                                .Path(vector3Path, VectorIndexType.Flat)
+                                .Path(vector3Path, VectorIndexType.DiskANN)
+                                .WithQuantizationByteSize(8)
+                                .WithIndexingSearchListSize(5)
+                                .WithVectorIndexShardKey(new string[] { "/Country/City", "ZipCode" })
                              .Attach()
                         .Attach()
                         .CreateAsync();
@@ -610,9 +615,14 @@ await databaseForVectorEmbedding.DefineContainer(containerName, partitionKeyPath
                 Assert.AreEqual(vector1Path, containerSettings.IndexingPolicy.VectorIndexes[0].Path);
                 Assert.AreEqual(VectorIndexType.Flat, containerSettings.IndexingPolicy.VectorIndexes[0].Type);
                 Assert.AreEqual(vector2Path, containerSettings.IndexingPolicy.VectorIndexes[1].Path);
-                Assert.AreEqual(VectorIndexType.Flat, containerSettings.IndexingPolicy.VectorIndexes[1].Type);
+                Assert.AreEqual(VectorIndexType.QuantizedFlat, containerSettings.IndexingPolicy.VectorIndexes[1].Type);
+                Assert.AreEqual(4, containerSettings.IndexingPolicy.VectorIndexes[1].QuantizationByteSize);
+                CollectionAssert.AreEqual(new string[] { "/Country/City" }, containerSettings.IndexingPolicy.VectorIndexes[1].VectorIndexShardKey);
                 Assert.AreEqual(vector3Path, containerSettings.IndexingPolicy.VectorIndexes[2].Path);
-                Assert.AreEqual(VectorIndexType.Flat, containerSettings.IndexingPolicy.VectorIndexes[2].Type);
+                Assert.AreEqual(VectorIndexType.DiskANN, containerSettings.IndexingPolicy.VectorIndexes[2].Type);
+                Assert.AreEqual(8, containerSettings.IndexingPolicy.VectorIndexes[2].QuantizationByteSize);
+                Assert.AreEqual(5, containerSettings.IndexingPolicy.VectorIndexes[2].IndexingSearchListSize);
+                CollectionAssert.AreEqual(new string[] { "/Country/City", "ZipCode" }, containerSettings.IndexingPolicy.VectorIndexes[2].VectorIndexShardKey);
             }
             finally
             {