Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cpp/bench/ann/src/diskann/diskann_wrapper.h
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ class diskann_ssd : public algo<T> {
template <typename T>
diskann_ssd<T>::diskann_ssd(Metric metric, int dim, const build_param& param) : algo<T>(metric, dim)
{
// Currently set the indexing RAM budget and the search RAM budget to max value to avoid sharding
// Currently we set the indexing RAM budget and the search RAM budget to max value to avoid sharding
uint32_t build_dram_budget = std::numeric_limits<uint32_t>::max();
uint32_t search_dram_budget = std::numeric_limits<uint32_t>::max();
index_build_params_str =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,6 @@ groups:
base:
build:
search:
test:
build:
search:
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,10 @@ groups:
graph_build_algo: ["NN_DESCENT"]
search:
L_search: [10, 20, 30, 40, 50, 100, 200, 300]
test:
build:
graph_degree: [32]
intermediate_graph_degree: [32]
graph_build_algo: ["NN_DESCENT"]
search:
L_search: [10]
7 changes: 7 additions & 0 deletions python/cuvs_bench/cuvs_bench/config/algos/cuvs_vamana.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,3 +8,10 @@ groups:
search:
L_search: [10, 20, 30, 40, 50, 100, 200, 300]
num_threads: [32]
test:
build:
graph_degree: [32]
visited_size: [128]
alpha: [1.2]
search:
L_search: [10]
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,11 @@ groups:
num_threads: [32]
search:
L_search: [10, 20, 30, 40, 50, 100, 200, 300]
test:
build:
R: [32]
L_build: [128]
alpha: [1.2]
num_threads: [32]
search:
L_search: [10]
151 changes: 147 additions & 4 deletions python/cuvs_bench/cuvs_bench/tests/test_cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@ def test_run_command_creates_results(temp_datasets_dir: Path):

python -m cuvs_bench.run --dataset test-data --dataset-path datasets/ \
--algorithms faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,\
cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq \
cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,cuvs_vamana,\
diskann_memory \
--batch-size 100 -k 10 --groups test -m latency --force

It then verifies that the set of expected result files
Expand All @@ -70,7 +71,7 @@ def test_run_command_creates_results(temp_datasets_dir: Path):
"--dataset-path",
dataset_path_arg,
"--algorithms",
"faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq", # noqa: E501
"faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,cuvs_vamana,diskann_memory", # noqa: E501
"--batch-size",
"100",
"-k",
Expand Down Expand Up @@ -159,6 +160,27 @@ def test_run_command_creates_results(temp_datasets_dir: Path):
],
"rows": 1,
},
"test-data/result/build/cuvs_vamana,test.csv": {
"header": common_build_header
+ [
"GPU",
"alpha",
"graph_degree",
"num_threads",
"visited_size",
],
"rows": 1,
},
"test-data/result/build/diskann_memory,test.csv": {
"header": common_build_header
+ [
"L_build",
"R",
"alpha",
"num_threads",
],
"rows": 1,
},
# Search files:
"test-data/result/search/cuvs_cagra_hnswlib,test,k10,bs100,raw.csv": {
"header": common_search_header
Expand Down Expand Up @@ -421,6 +443,126 @@ def test_run_command_creates_results(temp_datasets_dir: Path):
],
"rows": 2,
},
"test-data/result/search/cuvs_vamana,test,k10,bs100,raw.csv": {
"header": common_search_header
+ [
"GPU",
"L_search",
"end_to_end",
"k",
"n_queries",
"num_threads",
"total_queries",
"build time",
"build threads",
"build cpu_time",
"build GPU",
"alpha",
"graph_degree",
"build num_threads",
"visited_size",
],
"rows": 1,
},
"test-data/result/search/cuvs_vamana,test,k10,bs100,latency.csv": {
"header": common_search_header
+ [
"GPU",
"L_search",
"end_to_end",
"k",
"n_queries",
"num_threads",
"total_queries",
"build time",
"build threads",
"build cpu_time",
"build GPU",
"alpha",
"graph_degree",
"build num_threads",
"visited_size",
],
"rows": 1,
},
"test-data/result/search/cuvs_vamana,test,k10,bs100,throughput.csv": {
"header": common_search_header
+ [
"GPU",
"L_search",
"end_to_end",
"k",
"n_queries",
"num_threads",
"total_queries",
"build time",
"build threads",
"build cpu_time",
"build GPU",
"alpha",
"graph_degree",
"build num_threads",
"visited_size",
],
"rows": 1,
},
"test-data/result/search/diskann_memory,test,k10,bs100,raw.csv": {
"header": common_search_header
+ [
"L_search",
"end_to_end",
"k",
"n_queries",
"num_threads",
"total_queries",
"build time",
"build threads",
"build cpu_time",
"L_build",
"R",
"alpha",
"build_num_threads",
],
"rows": 1,
},
"test-data/result/search/diskann_memory,test,k10,bs100,latency.csv": {
"header": common_search_header
+ [
"L_search",
"end_to_end",
"k",
"n_queries",
"num_threads",
"total_queries",
"build time",
"build threads",
"build cpu_time",
"L_build",
"R",
"alpha",
"build_num_threads",
],
"rows": 1,
},
"test-data/result/search/diskann_memory,test,k10,bs100,throughput.csv": {
"header": common_search_header
+ [
"L_search",
"end_to_end",
"k",
"n_queries",
"num_threads",
"total_queries",
"build time",
"build threads",
"build cpu_time",
"L_build",
"R",
"alpha",
"build_num_threads",
],
"rows": 1,
},
}

for rel_path, expectations in expected_files.items():
Expand Down Expand Up @@ -448,7 +590,8 @@ def test_plot_command_creates_png_files(temp_datasets_dir: Path):

python -m cuvs_bench.plot --dataset test-data --dataset-path datasets/ \
--algorithms faiss_gpu_ivf_flat,faiss_gpu_ivf_sq, \
cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq \
cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,\
cuvs_vamana,diskann_memory \
--batch-size 100 -k 10 --groups test -m latency

and then verifies that the following files are produced in the
Expand All @@ -472,7 +615,7 @@ def test_plot_command_creates_png_files(temp_datasets_dir: Path):
"--output-filepath",
dataset_path_arg,
"--algorithms",
"faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq", # noqa: E501
"faiss_gpu_ivf_flat,faiss_gpu_ivf_sq,cuvs_ivf_flat,cuvs_cagra,ggnn,cuvs_cagra_hnswlib,cuvs_ivf_pq,cuvs_vamana,diskann_memory", # noqa: E501
"--batch-size",
"100",
"-k",
Expand Down