Skip to content

Commit

Permalink
[test][fix] Add ivf pq boundary test timestamp.
Browse files Browse the repository at this point in the history
  • Loading branch information
Haijun Yu committed Dec 5, 2023
1 parent 520d0d8 commit 048250e
Showing 1 changed file with 55 additions and 47 deletions.
102 changes: 55 additions & 47 deletions test/test_vector_index_raw_ivf_pq_boundary.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
#include <vector>

#include "butil/status.h"
#include "common/helper.h"
#include "faiss/MetricType.h"
#include "proto/common.pb.h"
#include "proto/error.pb.h"
Expand Down Expand Up @@ -131,7 +132,7 @@ TEST_F(VectorIndexRawIvfPqBoundaryTest, Create) {
int max_internal_nbits_per_idx = 16;
int max_internal_ncentroids = 256;

for (int internal_dimension = 1; internal_dimension <= max_internal_dimension; internal_dimension++) {
for (int internal_dimension = 1; internal_dimension <= max_internal_dimension; internal_dimension *= 2) {
// // create random data
// {
// std::mt19937 rng;
Expand All @@ -155,9 +156,9 @@ TEST_F(VectorIndexRawIvfPqBoundaryTest, Create) {
// internal_dimension)
// << '\n';
for (int internal_nbits_per_idx = 1; internal_nbits_per_idx <= max_internal_nbits_per_idx;
internal_nbits_per_idx++) {
for (int internal_nsubvector = 1; internal_nsubvector <= internal_dimension; internal_nsubvector++)
for (int internal_ncentroids = 1; internal_ncentroids <= max_internal_ncentroids; internal_ncentroids++) {
internal_nbits_per_idx *= 2) {
for (int internal_nsubvector = 1; internal_nsubvector <= internal_dimension; internal_nsubvector *= 2)
for (int internal_ncentroids = 1; internal_ncentroids <= max_internal_ncentroids; internal_ncentroids *= 2) {
// create random data
{
std::mt19937 rng;
Expand Down Expand Up @@ -305,21 +306,23 @@ TEST_F(VectorIndexRawIvfPqBoundaryTest, Create) {
}
butil::Status ok = raw_ivf->Train(internal_data_base);
if (!ok.ok()) {
std::cout << fmt::format(
"data_base_size : {} dimension : {} nbits_per_idx : {} nsubvector:{} ncentroids:{} {} "
"train failed",
data_base_size, internal_dimension, internal_nbits_per_idx, internal_nsubvector,
internal_ncentroids, name)
<< "\n"
<< "\n";

outfile << fmt::format(
"data_base_size : {} dimension : {} nbits_per_idx : {} nsubvector:{} ncentroids:{} {} "
"train failed",
data_base_size, internal_dimension, internal_nbits_per_idx, internal_nsubvector,
internal_ncentroids, name)
<< "\n"
<< "\n";
std::cout
<< fmt::format(
"[{}] data_base_size : {} dimension : {} nbits_per_idx : {} nsubvector:{} ncentroids:{} {} "
"train failed",
Helper::NowTime(), data_base_size, internal_dimension, internal_nbits_per_idx,
internal_nsubvector, internal_ncentroids, name)
<< "\n"
<< "\n";

outfile
<< fmt::format(
"[{}] data_base_size : {} dimension : {} nbits_per_idx : {} nsubvector:{} ncentroids:{} {} "
"train failed",
Helper::NowTime(), data_base_size, internal_dimension, internal_nbits_per_idx,
internal_nsubvector, internal_ncentroids, name)
<< "\n"
<< "\n";
outfile.flush();
lambda_set_error(name, true);
return;
Expand Down Expand Up @@ -355,18 +358,20 @@ TEST_F(VectorIndexRawIvfPqBoundaryTest, Create) {
}
butil::Status ok = raw_ivf->Add(vector_with_ids);
if (!ok.ok()) {
std::cout << fmt::format(
"data_base_size : {} dimension : {} nbits_per_idx : {} nsubvector:{} ncentroids:{} {} "
"add failed",
data_base_size, internal_dimension, internal_nbits_per_idx, internal_nsubvector,
internal_ncentroids, name)
<< "\n"
<< "\n";
std::cout
<< fmt::format(
"[{}] data_base_size : {} dimension : {} nbits_per_idx : {} nsubvector:{} ncentroids:{} {} "
"add failed",
Helper::NowTime(), data_base_size, internal_dimension, internal_nbits_per_idx,
internal_nsubvector, internal_ncentroids, name)
<< "\n"
<< "\n";
outfile << fmt::format(
"data_base_size : {} dimension : {} nbits_per_idx : {} nsubvector:{} ncentroids:{} {} add "
"[{}] data_base_size : {} dimension : {} nbits_per_idx : {} nsubvector:{} ncentroids:{} "
"{} add "
"failed",
data_base_size, internal_dimension, internal_nbits_per_idx, internal_nsubvector,
internal_ncentroids, name)
Helper::NowTime(), data_base_size, internal_dimension, internal_nbits_per_idx,
internal_nsubvector, internal_ncentroids, name)
<< "\n"
<< "\n";
lambda_set_error(name, true);
Expand Down Expand Up @@ -413,21 +418,23 @@ TEST_F(VectorIndexRawIvfPqBoundaryTest, Create) {
std::vector<pb::index::VectorWithDistanceResult> results;
butil::Status ok = raw_ivf->Search(vector_with_ids_clone, topk, {filter}, false, parameter, results);
if (!ok.ok()) {
std::cout << fmt::format(
"data_base_size : {} dimension : {} nbits_per_idx : {} nsubvector:{} ncentroids:{} {} "
"search failed",
data_base_size, internal_dimension, internal_nbits_per_idx, internal_nsubvector,
internal_ncentroids, name)
<< "\n"
<< "\n";

outfile << fmt::format(
"data_base_size : {} dimension : {} nbits_per_idx : {} nsubvector:{} ncentroids:{} {} "
"search failed",
data_base_size, internal_dimension, internal_nbits_per_idx, internal_nsubvector,
internal_ncentroids, name)
<< "\n"
<< "\n";
std::cout
<< fmt::format(
"[{}] data_base_size : {} dimension : {} nbits_per_idx : {} nsubvector:{} ncentroids:{} {} "
"search failed",
Helper::NowTime(), data_base_size, internal_dimension, internal_nbits_per_idx,
internal_nsubvector, internal_ncentroids, name)
<< "\n"
<< "\n";

outfile
<< fmt::format(
"[{}] data_base_size : {} dimension : {} nbits_per_idx : {} nsubvector:{} ncentroids:{} {} "
"search failed",
Helper::NowTime(), data_base_size, internal_dimension, internal_nbits_per_idx,
internal_nsubvector, internal_ncentroids, name)
<< "\n"
<< "\n";
outfile.flush();
lambda_set_error(name, true);
return;
Expand All @@ -443,9 +450,10 @@ TEST_F(VectorIndexRawIvfPqBoundaryTest, Create) {
t.join();
}

lambda_output_info(fmt::format(
"data_base_size : {} dimension : {} nbits_per_idx : {} nsubvector:{} ncentroids:{}", data_base_size,
internal_dimension, internal_nbits_per_idx, internal_nsubvector, internal_ncentroids));
lambda_output_info(
fmt::format("[{}] data_base_size : {} dimension : {} nbits_per_idx : {} nsubvector:{} ncentroids:{}",
Helper::NowTime(), data_base_size, internal_dimension, internal_nbits_per_idx,
internal_nsubvector, internal_ncentroids));

outfile.flush();
}
Expand Down

0 comments on commit 048250e

Please sign in to comment.