Skip to content

Commit fad4568

Browse files
qixuan0212qixuan0212
committed
Add query vector operation related cases for null and default value support
Signed-off-by: qixuan <[email protected]>
1 parent 4460177 commit fad4568

File tree

7 files changed

+124
-41
lines changed

7 files changed

+124
-41
lines changed

‎tests/milvustestv2/src/main/java/com/zilliz/milvustestv2/common/BaseTest.java‎

Lines changed: 0 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,11 @@ public void initCollection() {
5151
logger.info("**************************************************BeforeSuit**********************");
5252
milvusClientV2.dropAlias(DropAliasReq.builder().alias(CommonData.alias).build());
5353
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(CommonData.defaultFloatVectorCollection).build());
54-
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(CommonData.defaultHasNullCollection).build());
5554
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(CommonData.defaultBFloat16VectorCollection).build());
5655
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(CommonData.defaultFloat16VectorCollection).build());
5756
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(CommonData.defaultBinaryVectorCollection).build());
5857
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(CommonData.defaultSparseFloatVectorCollection).build());
5958
initFloatVectorCollectionForTest();
60-
initHasNoneCollectionForTest();
6159
initBF16VectorForTest();
6260
initFloat16VectorForTest();
6361
initSparseVectorForTest();
@@ -111,36 +109,4 @@ public void initSparseVectorForTest(){
111109
CommonFunction.createNewCollection(CommonData.dim,CommonData.defaultSparseFloatVectorCollection, DataType.SparseFloatVector);
112110
CommonFunction.createIndexAndInsertAndLoad(CommonData.defaultSparseFloatVectorCollection,DataType.SparseFloatVector,true,CommonData.numberEntities);
113111
}
114-
115-
public void initHasNoneCollectionForTest(){
116-
CommonFunction.createNewNullableDefaultValueCollection(CommonData.dim, CommonData.defaultHasNullCollection, DataType.FloatVector);
117-
// insert data
118-
List<JsonObject> jsonObjects = CommonFunction.generateSimpleNullData(0,CommonData.numberEntities, CommonData.dim,DataType.FloatVector);
119-
InsertResp insert = milvusClientV2.insert(InsertReq.builder().collectionName(CommonData.defaultHasNullCollection).data(jsonObjects).build());
120-
CommonFunction.createVectorIndex(CommonData.defaultHasNullCollection,CommonData.fieldFloatVector, IndexParam.IndexType.AUTOINDEX, IndexParam.MetricType.L2);
121-
// Build Scalar Index
122-
List<FieldParam> FieldParamList = new ArrayList<FieldParam>() {{
123-
add(FieldParam.builder().fieldName(CommonData.fieldVarchar).indextype(IndexParam.IndexType.BITMAP).build());
124-
add(FieldParam.builder().fieldName(CommonData.fieldInt8).indextype(IndexParam.IndexType.BITMAP).build());
125-
add(FieldParam.builder().fieldName(CommonData.fieldInt16).indextype(IndexParam.IndexType.BITMAP).build());
126-
add(FieldParam.builder().fieldName(CommonData.fieldInt32).indextype(IndexParam.IndexType.BITMAP).build());
127-
add(FieldParam.builder().fieldName(CommonData.fieldInt64).indextype(IndexParam.IndexType.STL_SORT).build());
128-
add(FieldParam.builder().fieldName(CommonData.fieldBool).indextype(IndexParam.IndexType.BITMAP).build());
129-
add(FieldParam.builder().fieldName(CommonData.fieldArray).indextype(IndexParam.IndexType.BITMAP).build());
130-
}};
131-
CommonFunction.createScalarCommonIndex(CommonData.defaultHasNullCollection, FieldParamList);
132-
log.info("Create Scalar index done{}, scalar index:{}", CommonData.defaultHasNullCollection, FieldParamList);
133-
milvusClientV2.loadCollection(LoadCollectionReq.builder().collectionName(CommonData.defaultHasNullCollection).build());
134-
log.info("insert default float vector collection:" + insert);
135-
// create partition
136-
CommonFunction.createPartition(CommonData.defaultHasNullCollection,CommonData.partitionNameA);
137-
CommonFunction.createPartition(CommonData.defaultHasNullCollection,CommonData.partitionNameB);
138-
CommonFunction.createPartition(CommonData.defaultHasNullCollection,CommonData.partitionNameC);
139-
List<JsonObject> jsonObjectsA = CommonFunction.generateSimpleNullData(0,CommonData.numberEntities, CommonData.dim,DataType.FloatVector);
140-
List<JsonObject> jsonObjectsB = CommonFunction.generateSimpleNullData(0,CommonData.numberEntities*2, CommonData.dim,DataType.FloatVector);
141-
List<JsonObject> jsonObjectsC = CommonFunction.generateSimpleNullData(0,CommonData.numberEntities*3, CommonData.dim,DataType.FloatVector);
142-
milvusClientV2.insert(InsertReq.builder().collectionName(CommonData.defaultHasNullCollection).partitionName(CommonData.partitionNameA).data(jsonObjectsA).build());
143-
milvusClientV2.insert(InsertReq.builder().collectionName(CommonData.defaultHasNullCollection).partitionName(CommonData.partitionNameB).data(jsonObjectsB).build());
144-
milvusClientV2.insert(InsertReq.builder().collectionName(CommonData.defaultHasNullCollection).partitionName(CommonData.partitionNameC).data(jsonObjectsC).build());
145-
}
146112
}

‎tests/milvustestv2/src/main/java/com/zilliz/milvustestv2/common/CommonData.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,6 @@ public class CommonData {
1313
public static String defaultFloat16VectorCollection="Float16VectorCollection";
1414
public static String defaultBFloat16VectorCollection="BFloat16VectorCollection";
1515
public static String defaultSparseFloatVectorCollection="SparseFloatVectorCollection";
16-
public static String defaultHasNullCollection="HasNullableCollection";
1716
public static String partitionNameA="partitionNameA";
1817
public static String partitionNameB="partitionNameB";
1918
public static String partitionNameC="partitionNameC";

‎tests/milvustestv2/src/main/java/com/zilliz/milvustestv2/common/CommonFunction.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -649,6 +649,18 @@ public static List<JsonObject> generateSimpleNullData(long startId,long num, int
649649
}
650650
row.add(CommonData.fieldFloatVector, gson.toJsonTree(vector));
651651
}
652+
if (vectorType == DataType.BinaryVector) {
653+
row.add(CommonData.fieldBinaryVector, gson.toJsonTree(generateBinaryVector(dim).array()));
654+
}
655+
if (vectorType == DataType.Float16Vector) {
656+
row.add(CommonData.fieldFloat16Vector, gson.toJsonTree(generateFloat16Vector(dim).array()));
657+
}
658+
if (vectorType == DataType.BFloat16Vector) {
659+
row.add(CommonData.fieldBF16Vector, gson.toJsonTree(generateBF16Vector(dim).array()));
660+
}
661+
if (vectorType == DataType.SparseFloatVector) {
662+
row.add(CommonData.fieldSparseVector, gson.toJsonTree(generateSparseVector(dim)));
663+
}
652664

653665
JsonObject json = new JsonObject();
654666
json.addProperty(CommonData.fieldInt64, (int) i % 32767);

‎tests/milvustestv2/src/test/java/com/zilliz/milvustestv2/vectorOperation/InsertTest.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,17 +33,19 @@
3333
public class InsertTest extends BaseTest {
3434
private String newCollectionName;
3535
private String simpleCollectionName;
36+
private String nullableDefaultCollectionName;
3637

3738
@BeforeClass(alwaysRun = true)
3839
public void providerCollection() {
3940
newCollectionName = CommonFunction.createNewCollection(CommonData.dim, null, DataType.FloatVector);
4041
simpleCollectionName = CommonFunction.createSimpleCollection(CommonData.dim, null);
42+
nullableDefaultCollectionName = CommonFunction.createNewNullableDefaultValueCollection(CommonData.dim, null, DataType.FloatVector);
4143
}
42-
4344
@AfterClass(alwaysRun = true)
4445
public void cleanTestData() {
4546
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(newCollectionName).build());
4647
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(simpleCollectionName).build());
48+
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(nullableDefaultCollectionName).build());
4749
}
4850

4951
@DataProvider(name = "VectorTypeList")
@@ -88,4 +90,14 @@ public void insertIntoDiffVectorCollection(DataType dataType) {
8890
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(newCollection).build());
8991
}
9092

93+
@Test(description = "insert nullable field collection test", groups = {"Smoke"}, dataProvider = "VectorTypeList")
94+
public void insertNullableCollection(DataType dataType) {
95+
String newCollection = CommonFunction.createNewNullableDefaultValueCollection(CommonData.dim, null, dataType);
96+
List<JsonObject> jsonObjects = CommonFunction.generateSimpleNullData(0,CommonData.numberEntities, CommonData.dim, dataType);
97+
InsertResp insert = milvusClientV2.insert(InsertReq.builder()
98+
.collectionName(newCollection)
99+
.data(jsonObjects).build());
100+
Assert.assertEquals(insert.getInsertCnt(), CommonData.numberEntities);
101+
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(newCollection).build());
102+
}
91103
}

‎tests/milvustestv2/src/test/java/com/zilliz/milvustestv2/vectorOperation/QueryTest.java‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,30 @@
11
package com.zilliz.milvustestv2.vectorOperation;
22

33
import com.google.common.collect.Lists;
4+
import com.google.gson.JsonObject;
45
import com.zilliz.milvustestv2.common.BaseTest;
56
import com.zilliz.milvustestv2.common.CommonData;
7+
import com.zilliz.milvustestv2.common.CommonFunction;
8+
import com.zilliz.milvustestv2.params.FieldParam;
69
import com.zilliz.milvustestv2.utils.DataProviderUtils;
710
import io.milvus.v2.common.ConsistencyLevel;
811
import io.milvus.v2.common.DataType;
12+
import io.milvus.v2.common.IndexParam;
13+
import io.milvus.v2.service.collection.request.DropCollectionReq;
914
import io.milvus.v2.service.collection.request.GetCollectionStatsReq;
15+
import io.milvus.v2.service.collection.request.LoadCollectionReq;
1016
import io.milvus.v2.service.collection.response.GetCollectionStatsResp;
17+
import io.milvus.v2.service.vector.request.InsertReq;
1118
import io.milvus.v2.service.vector.request.QueryReq;
19+
import io.milvus.v2.service.vector.response.InsertResp;
1220
import io.milvus.v2.service.vector.response.QueryResp;
1321
import org.testng.Assert;
22+
import org.testng.annotations.AfterClass;
23+
import org.testng.annotations.BeforeClass;
1424
import org.testng.annotations.DataProvider;
1525
import org.testng.annotations.Test;
1626

27+
import java.util.ArrayList;
1728
import java.util.Arrays;
1829
import java.util.List;
1930

@@ -22,6 +33,9 @@
2233
* @Date 2024/2/19 17:03
2334
*/
2435
public class QueryTest extends BaseTest {
36+
37+
String nullableDefaultCollectionName;
38+
2539
@DataProvider(name = "filterAndExcept")
2640
public Object[][] providerData() {
2741
return new Object[][]{
@@ -88,6 +102,36 @@ public Object[][] providerDiffCollectionWithFilter(){
88102
return objects;
89103
}
90104

105+
@BeforeClass(alwaysRun = true)
106+
public void providerCollection() {
107+
nullableDefaultCollectionName = CommonFunction.createNewNullableDefaultValueCollection(CommonData.dim, null, DataType.FloatVector);
108+
// insert data
109+
List<JsonObject> jsonObjects = CommonFunction.generateSimpleNullData(0,CommonData.numberEntities, CommonData.dim,DataType.FloatVector);
110+
InsertResp insert = milvusClientV2.insert(InsertReq.builder().collectionName(nullableDefaultCollectionName).data(jsonObjects).build());
111+
CommonFunction.createVectorIndex(nullableDefaultCollectionName,CommonData.fieldFloatVector, IndexParam.IndexType.AUTOINDEX, IndexParam.MetricType.L2);
112+
// Build Scalar Index
113+
List<FieldParam> FieldParamList = new ArrayList<FieldParam>() {{
114+
add(FieldParam.builder().fieldName(CommonData.fieldVarchar).indextype(IndexParam.IndexType.BITMAP).build());
115+
add(FieldParam.builder().fieldName(CommonData.fieldInt8).indextype(IndexParam.IndexType.BITMAP).build());
116+
add(FieldParam.builder().fieldName(CommonData.fieldInt16).indextype(IndexParam.IndexType.BITMAP).build());
117+
add(FieldParam.builder().fieldName(CommonData.fieldInt32).indextype(IndexParam.IndexType.BITMAP).build());
118+
add(FieldParam.builder().fieldName(CommonData.fieldInt64).indextype(IndexParam.IndexType.STL_SORT).build());
119+
add(FieldParam.builder().fieldName(CommonData.fieldBool).indextype(IndexParam.IndexType.BITMAP).build());
120+
add(FieldParam.builder().fieldName(CommonData.fieldArray).indextype(IndexParam.IndexType.BITMAP).build());
121+
}};
122+
CommonFunction.createScalarCommonIndex(nullableDefaultCollectionName, FieldParamList);
123+
milvusClientV2.loadCollection(LoadCollectionReq.builder().collectionName(nullableDefaultCollectionName).build());
124+
// create partition
125+
CommonFunction.createPartition(nullableDefaultCollectionName,CommonData.partitionNameA);
126+
List<JsonObject> jsonObjectsA = CommonFunction.generateSimpleNullData(0,CommonData.numberEntities*3, CommonData.dim,DataType.FloatVector);
127+
milvusClientV2.insert(InsertReq.builder().collectionName(nullableDefaultCollectionName).partitionName(CommonData.partitionNameA).data(jsonObjectsA).build());
128+
}
129+
130+
@AfterClass(alwaysRun = true)
131+
public void cleanTestData() {
132+
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(nullableDefaultCollectionName).build());
133+
}
134+
91135
@DataProvider(name = "queryNullableField")
92136
private Object[][] provideNullableFieldQueryParams() {
93137
return new Object[][]{
@@ -178,7 +222,7 @@ public void queryDiffCollection(String collectionName,String filter, long expect
178222
@Test(description = "query with nullable field", groups = {"Smoke"}, dataProvider = "queryNullableField")
179223
public void queryByNullFilter(String filter, long expect) {
180224
QueryResp query = milvusClientV2.query(QueryReq.builder()
181-
.collectionName(CommonData.defaultHasNullCollection)
225+
.collectionName(nullableDefaultCollectionName)
182226
.consistencyLevel(ConsistencyLevel.BOUNDED)
183227
.outputFields(Lists.newArrayList("*"))
184228
.filter(filter)

‎tests/milvustestv2/src/test/java/com/zilliz/milvustestv2/vectorOperation/SearchTest.java‎

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ public class SearchTest extends BaseTest {
5555
int topK = 10;
5656
private MilvusServiceClient milvusServiceClient;
5757
String newCollectionName;
58+
String nullableDefaultCollectionName;
5859

5960
@DataProvider(name = "filterAndExcept")
6061
public Object[][] providerData() {
@@ -149,20 +150,37 @@ private Object[][] provideNullableFieldSearchParams() {
149150
};
150151
}
151152

153+
@DataProvider(name = "collectionNameList")
154+
public Object[][] providerCollectionName() {
155+
return new Object[][]{
156+
{newCollectionName},
157+
{nullableDefaultCollectionName},
158+
};
159+
}
160+
152161
@BeforeClass(alwaysRun = true)
153162
public void providerCollection() {
154163
newCollectionName = CommonFunction.createNewCollection(CommonData.dim, null, DataType.FloatVector);
155164
List<JsonObject> jsonObjects = CommonFunction.generateDefaultData(0,CommonData.numberEntities * 10, CommonData.dim, DataType.FloatVector);
156165
milvusClientV2.insert(InsertReq.builder().collectionName(newCollectionName).data(jsonObjects).build());
166+
nullableDefaultCollectionName = CommonFunction.createNewNullableDefaultValueCollection(CommonData.dim, null, DataType.FloatVector);
167+
// insert data
168+
List<JsonObject> jsonNullableObjects = CommonFunction.generateSimpleNullData(0,CommonData.numberEntities, CommonData.dim,DataType.FloatVector);
169+
milvusClientV2.insert(InsertReq.builder().collectionName(nullableDefaultCollectionName).data(jsonNullableObjects).build());
170+
// create partition
171+
CommonFunction.createPartition(nullableDefaultCollectionName,CommonData.partitionNameA);
172+
List<JsonObject> jsonObjectsA = CommonFunction.generateSimpleNullData(0,CommonData.numberEntities, CommonData.dim,DataType.FloatVector);
173+
milvusClientV2.insert(InsertReq.builder().collectionName(nullableDefaultCollectionName).partitionName(CommonData.partitionNameA).data(jsonObjectsA).build());
157174
}
158175

159176
@AfterClass(alwaysRun = true)
160177
public void cleanTestData() {
161178
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(newCollectionName).build());
179+
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(nullableDefaultCollectionName).build());
162180
}
163181

164-
@Test(description = "Create vector and scalar index", groups = {"Smoke"})
165-
public void createVectorAndScalarIndex() {
182+
@Test(description = "Create vector and scalar index", groups = {"Smoke"}, dataProvider = "collectionNameList")
183+
public void createVectorAndScalarIndex(String newCollectionName) {
166184
// Build Vector index
167185
IndexParam indexParam = IndexParam.builder()
168186
.fieldName(CommonData.fieldFloatVector)
@@ -359,11 +377,11 @@ public void searchScalarIndexCollection(String filter, int expect) {
359377
Assert.assertEquals(search.getSearchResults().get(0).size(), expect);
360378
}
361379

362-
@Test(description = "search collection with nullable field", groups = {"Smoke"}, dataProvider = "searchNullableField")
380+
@Test(description = "search collection with nullable field", groups = {"Smoke"}, dependsOnMethods = {"createVectorAndScalarIndex"}, dataProvider = "searchNullableField")
363381
public void searchNullableCollection(String filter, int expect) {
364382
List<BaseVector> data = CommonFunction.providerBaseVector(CommonData.nq, CommonData.dim, DataType.FloatVector);
365383
SearchReq searchParams = SearchReq.builder()
366-
.collectionName(CommonData.defaultHasNullCollection)
384+
.collectionName(nullableDefaultCollectionName)
367385
.filter(filter)
368386
.outputFields(Lists.newArrayList("*"))
369387
.consistencyLevel(ConsistencyLevel.STRONG)

‎tests/milvustestv2/src/test/java/com/zilliz/milvustestv2/vectorOperation/UpsertTest.java‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,17 +34,20 @@
3434
*/
3535
public class UpsertTest extends BaseTest {
3636
String newCollectionName;
37+
String nullableDefaultCollectionName;
3738

3839
@BeforeClass(alwaysRun = true)
3940
public void providerCollection() {
4041
newCollectionName = CommonFunction.createNewCollection(CommonData.dim, null, DataType.FloatVector);
4142
List<JsonObject> jsonObjects = CommonFunction.generateDefaultData(0,CommonData.numberEntities, CommonData.dim, DataType.FloatVector);
4243
milvusClientV2.insert(InsertReq.builder().collectionName(newCollectionName).data(jsonObjects).build());
44+
nullableDefaultCollectionName = CommonFunction.createNewNullableDefaultValueCollection(CommonData.dim, null, DataType.FloatVector);
4345
}
4446

4547
@AfterClass(alwaysRun = true)
4648
public void cleanTestData() {
4749
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(newCollectionName).build());
50+
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(nullableDefaultCollectionName).build());
4851
}
4952

5053
@DataProvider(name = "DifferentCollection")
@@ -114,4 +117,33 @@ public void simpleUpsert() {
114117
System.out.println(upsert);
115118
}
116119

120+
@Test(description = "upsert nullable collection", groups = {"Smoke"}, dataProvider = "DifferentCollection")
121+
public void nullableCollectionUpsert( DataType vectorType) {
122+
String collectionName = CommonFunction.createNewNullableDefaultValueCollection(CommonData.dim, null, vectorType);
123+
CommonFunction.createIndexAndInsertAndLoad(collectionName,vectorType,true,CommonData.numberEntities);
124+
125+
List<JsonObject> jsonObjects = CommonFunction.generateSimpleNullData(0,1, CommonData.dim, vectorType);
126+
for (int i = 1; i < 10; i++) {
127+
JsonObject jsonObject0 = jsonObjects.get(0).deepCopy();
128+
jsonObject0.addProperty(CommonData.fieldInt64, i);
129+
jsonObjects.add(jsonObject0);
130+
}
131+
UpsertResp upsert = milvusClientV2.upsert(UpsertReq.builder()
132+
.collectionName(collectionName)
133+
.data(jsonObjects)
134+
.partitionName("_default")
135+
.build());
136+
System.out.println(upsert);
137+
Assert.assertEquals(upsert.getUpsertCnt(), 10);
138+
139+
// query
140+
QueryResp query = milvusClientV2.query(QueryReq.builder()
141+
.collectionName(collectionName)
142+
.filter(CommonData.fieldInt32 + " == 1")
143+
.partitionNames(Lists.newArrayList(CommonData.defaultPartitionName))
144+
.outputFields(Lists.newArrayList(CommonData.fieldInt64, CommonData.fieldInt32))
145+
.consistencyLevel(ConsistencyLevel.STRONG).build());
146+
Assert.assertEquals(query.getQueryResults().size(),10);
147+
milvusClientV2.dropCollection(DropCollectionReq.builder().collectionName(collectionName).build());
148+
}
117149
}

0 commit comments

Comments
 (0)