Skip to content

Query: Fixes handling of undefined projections in hybrid search #5145

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Apr 24, 2025
Merged

Query: Fixes handling of undefined projections in hybrid search #5145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
3ce4205
Handle undefined projections gracefully in the hybrid search query pi…
neildsh Apr 24, 2025
e2b9e07
Add more test cases for exercising undefined projections in hybrid se…
neildsh Apr 24, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ...ure.Cosmos/src/Query/Core/Pipeline/CrossPartition/HybridSearch/HybridSearchQueryResult.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,9 +54,9 @@ public static HybridSearchQueryResult Create(CosmosElement document)
throw new ArgumentException($"{FieldNames.Payload} must exist.");
}

if (!outerPayload.TryGetValue(FieldNames.Payload, out CosmosObject innerPayload))
if (!outerPayload.TryGetValue(FieldNames.Payload, out CosmosElement innerPayload))
{
throw new ArgumentException($"{FieldNames.Payload} must exist nested within the outer payload field.");
innerPayload = CosmosUndefined.Create();
}

if (!outerPayload.TryGetValue(FieldNames.ComponentScores, out CosmosArray componentScores))
Expand Down
125 changes: 105 additions & 20 deletions ...zure.Cosmos/tests/Microsoft.Azure.Cosmos.EmulatorTests/Query/CosmosUndefinedQueryTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@ public sealed class CosmosUndefinedQueryTests : QueryTestsBase

private const int IntegerValue = 42;

private const string StringValue = "string";
private const string ArrayValue = "[10, 20]";
private const string StringValue = "string";

private const string ArrayValue = "[10, 20]";

private const string ObjectValue = "{\"type\":\"object\"}";

private static readonly int[] PageSizes = new[] { 5, 10, -1 };
Expand All @@ -51,14 +51,76 @@ await this.CreateIngestQueryDeleteAsync(
indexingPolicy: CompositeIndexPolicy);
}

[TestMethod]
public async Task HybridSearchTests()
{
// Removing the await causes the test framework to not run this test
await this.CreateIngestQueryDeleteAsync(
connectionModes: ConnectionModes.Direct | ConnectionModes.Gateway,
collectionTypes: CollectionTypes.MultiPartition | CollectionTypes.SinglePartition,
documents: Documents,
query: HybridSearchTests,
indexingPolicy: CompositeIndexPolicy);
}

private static async Task RunTests(Container container, IReadOnlyList<CosmosObject> _)
{
await OrderByTests(container);
await GroupByTests(container);
await UntypedTests(container);
}

private static async Task UntypedTests(Container container)
private static async Task HybridSearchTests(Container container, IReadOnlyList<CosmosObject> _)
{
UndefinedProjectionTestCase[] testCases = new[]
{
MakeUndefinedProjectionTest(
query: "SELECT c.AlwaysUndefinedField " +
"FROM c " +
"ORDER BY RANK FullTextScore(c.AlwaysUndefinedField, ['needle'])",
expectedCount: DocumentCount),
MakeUndefinedProjectionTest(
query: "SELECT VALUE c.AlwaysUndefinedField " +
"FROM c " +
"ORDER BY RANK FullTextScore(c.AlwaysUndefinedField, ['needle'])",
expectedCount: 0),
MakeUndefinedProjectionTest(
query: "SELECT c.AlwaysUndefinedField " +
"FROM c " +
"ORDER BY RANK RRF(FullTextScore(c.AlwaysUndefinedField, ['needle']), FullTextScore(c.AnotherUndefinedField, ['needle']))",
expectedCount: DocumentCount),
MakeUndefinedProjectionTest(
query: "SELECT VALUE c.AlwaysUndefinedField " +
"FROM c " +
"ORDER BY RANK RRF(FullTextScore(c.AlwaysUndefinedField, ['needle']), FullTextScore(c.AnotherUndefinedField, ['needle']))",
expectedCount: 0),
MakeUndefinedProjectionTest(
query: $"SELECT c.AlwaysUndefinedField " +
$"FROM c " +
$"ORDER BY RANK FullTextScore(c.{nameof(MixedTypeDocument.MixedTypeField)}, ['needle'])",
expectedCount: DocumentCount),
MakeUndefinedProjectionTest(
query: $"SELECT VALUE c.AlwaysUndefinedField " +
$"FROM c " +
$"ORDER BY RANK FullTextScore(c.{nameof(MixedTypeDocument.MixedTypeField)}, ['needle'])",
expectedCount: 0),
MakeUndefinedProjectionTest(
query: $"SELECT c.AlwaysUndefinedField " +
$"FROM c " +
$"ORDER BY RANK RRF(FullTextScore(c.{nameof(MixedTypeDocument.MixedTypeField)}, ['needle']),FullTextScore(c.{nameof(MixedTypeDocument.Index)}, ['needle']))",
expectedCount: DocumentCount),
MakeUndefinedProjectionTest(
query: $"SELECT VALUE c.AlwaysUndefinedField " +
$"FROM c " +
$"ORDER BY RANK RRF(FullTextScore(c.{nameof(MixedTypeDocument.MixedTypeField)}, ['needle']),FullTextScore(c.{nameof(MixedTypeDocument.Index)}, ['needle']))",
expectedCount: 0),
};

await RunUndefinedProjectionTests(container, testCases);
await RunUntypedTestsAsync(container, testCases);
}

private static Task UntypedTests(Container container)
{
UndefinedProjectionTestCase[] undefinedProjectionTestCases = new[]
{
Expand All @@ -79,13 +141,18 @@ private static async Task UntypedTests(Container container)
expectedCount: 0),
MakeUndefinedProjectionTest(
query: $"SELECT VALUE AVG(c.{nameof(MixedTypeDocument.MixedTypeField)}) FROM c",
expectedCount: 0),
MakeUndefinedProjectionTest(
expectedCount: 0),
MakeUndefinedProjectionTest(
query: $"SELECT DISTINCT VALUE SUM(c.{nameof(MixedTypeDocument.MixedTypeField)}) FROM c",
expectedCount: 0)
};

foreach (UndefinedProjectionTestCase testCase in undefinedProjectionTestCases)
return RunUntypedTestsAsync(container, undefinedProjectionTestCases);
}

private static async Task RunUntypedTestsAsync(Container container, IEnumerable<UndefinedProjectionTestCase> testCases)
{
foreach (UndefinedProjectionTestCase testCase in testCases)
{
foreach (int pageSize in PageSizes)
{
Expand Down Expand Up @@ -122,6 +189,24 @@ private static async Task UntypedTests(Container container)
Assert.AreEqual(testCase.ExpectedResultCount, actualCount);
}
}
}

private static async Task RunUndefinedProjectionTests(Container container, IEnumerable<UndefinedProjectionTestCase> testCases)
{
foreach (UndefinedProjectionTestCase testCase in testCases)
{
foreach (int pageSize in PageSizes)
{
List<UndefinedProjection> results = await RunQueryCombinationsAsync<UndefinedProjection>(
container,
testCase.Query,
new QueryRequestOptions { MaxItemCount = pageSize },
QueryDrainingMode.HoldState);

Assert.AreEqual(testCase.ExpectedResultCount, results.Count);
Assert.IsTrue(results.All(x => x is UndefinedProjection));
}
}
}

private static async Task OrderByTests(Container container)
Expand Down Expand Up @@ -237,10 +322,10 @@ private static async Task GroupByTests(Container container)
value: DocumentsPerTypeCount),
MakeGrouping(
key: CosmosArray.Parse(ArrayValue),
value: DocumentsPerTypeCount),
value: DocumentsPerTypeCount),
MakeGrouping(
key: CosmosObject.Parse(ObjectValue),
value: DocumentsPerTypeCount),
value: DocumentsPerTypeCount),
}),
MakeGroupByTest(
query: $"SELECT SUM(c.{nameof(MixedTypeDocument.MixedTypeField)}) as {nameof(GroupByProjection.MixedTypeField)}, " +
Expand Down Expand Up @@ -361,9 +446,9 @@ private static async Task GroupByTests(Container container)
};

foreach (GroupByUndefinedTestCase testCase in mixedTypeTestCases)
{
{
foreach (int pageSize in PageSizes)
{
{
List<GroupByProjection> actual = await QueryWithoutContinuationTokensAsync<GroupByProjection>(
container,
testCase.Query,
Expand Down Expand Up @@ -457,14 +542,14 @@ private static List<MixedTypeDocument> CreateDocuments(int count)
case 4:
mixedTypeElement = CosmosString.Create(StringValue);
break;
case 5:
mixedTypeElement = CosmosArray.Parse(ArrayValue);
break;
case 6:
mixedTypeElement = CosmosObject.Parse(ObjectValue);
break;

case 5:
mixedTypeElement = CosmosArray.Parse(ArrayValue);
break;

case 6:
mixedTypeElement = CosmosObject.Parse(ObjectValue);
break;

default:
mixedTypeElement = null;
Expand Down
Loading