From acfa4a3f61eaff5582b8d6c683a69c14adc761d8 Mon Sep 17 00:00:00 2001 From: Haijun Yu Date: Tue, 23 Apr 2024 16:52:32 +0800 Subject: [PATCH] [chore][coprocessor] Remove deprecated function. --- src/vector/vector_index_flat.cc | 30 ----------------------- src/vector/vector_index_flat.h | 9 ------- src/vector/vector_index_utils.cc | 23 ----------------- src/vector/vector_index_utils.h | 8 ------ test/unit_test/test_vector_index_utils.cc | 2 +- 5 files changed, 1 insertion(+), 71 deletions(-) diff --git a/src/vector/vector_index_flat.cc b/src/vector/vector_index_flat.cc index 0d76bedef..1fddd111c 100644 --- a/src/vector/vector_index_flat.cc +++ b/src/vector/vector_index_flat.cc @@ -385,22 +385,6 @@ butil::Status VectorIndexFlat::GetMemorySize(int64_t& memory_size) { bool VectorIndexFlat::IsExceedsMaxElements() { return false; } -void VectorIndexFlat::SearchWithParam(faiss::idx_t n, const faiss::Index::component_t* x, faiss::idx_t k, - faiss::Index::distance_t* distances, faiss::idx_t* labels, - std::shared_ptr filters) { - faiss::SearchParameters param; - param.sel = filters.get(); - raw_index_->search(n, x, k, distances, labels, ¶m); - faiss::idx_t* li = labels; - // TODO: running by regular user may cause abort exception below - // libgomp: Thread creation failed: Resource temporarily unavailable - // please run by root, or adjust ulimit for regular user, then you can increase FLAGS_omp_num_threads -#pragma omp parallel for - for (faiss::idx_t i = 0; i < n * k; ++i) { - li[i] = li[i] < 0 ? li[i] : index_id_map2_->id_map[li[i]]; - } -} - bool VectorIndexFlat::NeedToSave(int64_t last_save_log_behind) { RWLockReadGuard guard(&rw_lock_); @@ -419,21 +403,7 @@ bool VectorIndexFlat::NeedToSave(int64_t last_save_log_behind) { return false; } -void VectorIndexFlat::DoRangeSearch(faiss::idx_t n, const faiss::Index::component_t* x, faiss::Index::distance_t radius, - faiss::RangeSearchResult* result, - std::vector> filters) { - faiss::SearchParameters flat_search_parameters; - auto flat_filter = std::make_shared(filters); - auto flat_filter_wrapper = std::make_shared(index_id_map2_->id_map, flat_filter.get()); - flat_search_parameters.sel = flat_filter_wrapper.get(); - index_id_map2_->index->range_search(n, x, radius, result, &flat_search_parameters); - -#pragma omp parallel for - for (faiss::idx_t i = 0; i < result->lims[result->nq]; i++) { - result->labels[i] = result->labels[i] < 0 ? result->labels[i] : index_id_map2_->id_map[result->labels[i]]; - } -} template std::vector VectorIndexFlat::GetExistVectorIds(const T& ids, size_t size) { diff --git a/src/vector/vector_index_flat.h b/src/vector/vector_index_flat.h index e4464076d..6ed4dc64d 100644 --- a/src/vector/vector_index_flat.h +++ b/src/vector/vector_index_flat.h @@ -112,15 +112,6 @@ class VectorIndexFlat : public VectorIndex { bool NeedToSave(int64_t last_save_log_behind) override; private: - [[deprecated("faiss fix bug. never use.")]] void SearchWithParam(faiss::idx_t n, const faiss::Index::component_t* x, - faiss::idx_t k, faiss::Index::distance_t* distances, - faiss::idx_t* labels, - std::shared_ptr filters); - - [[deprecated("faiss add feature. never use.")]] void DoRangeSearch( - faiss::idx_t n, const faiss::Index::component_t* x, faiss::Index::distance_t radius, - faiss::RangeSearchResult* result, std::vector> filters); - template std::vector GetExistVectorIds(const T& ids, size_t size); diff --git a/src/vector/vector_index_utils.cc b/src/vector/vector_index_utils.cc index 30cb0e1c5..43a4c1f4c 100644 --- a/src/vector/vector_index_utils.cc +++ b/src/vector/vector_index_utils.cc @@ -895,11 +895,6 @@ butil::Status VectorIndexUtils::ValidateVectorIndexParameter( return VectorIndexUtils::ValidateVectorScalarSchema(vector_index_parameter.scalar_schema()); } -butil::Status VectorIndexUtils::ValidateScalarIndexParameter( - const pb::common::ScalarIndexParameter& /*scalar_index_parameter*/) { - return butil::Status::OK(); -} - butil::Status VectorIndexUtils::ValidateVectorScalarSchema(const pb::common::ScalarSchema& scalar_schema) { DINGO_LOG_IF(INFO, FLAGS_dingo_log_switch_scalar_speed_up_detail) << fmt::format("scalar_schema : {}", scalar_schema.DebugString()); @@ -985,24 +980,6 @@ butil::Status VectorIndexUtils::ValidateVectorScalarData(const pb::common::Scala return butil::Status::OK(); } -butil::Status VectorIndexUtils::SplitVectorScalarData( - const pb::common::ScalarSchema& scalar_schema, const pb::common::VectorScalardata& vector_scalar_data, - std::vector>& scalar_key_value_pairs, // NOLINT - pb::common::VectorScalardata& other_vector_scalar_data) { - other_vector_scalar_data = vector_scalar_data; - for (const auto& field : scalar_schema.fields()) { - if (field.enable_speed_up()) { - const std::string& key = field.key(); - auto iter = other_vector_scalar_data.scalar_data().find(key); - if (iter != other_vector_scalar_data.scalar_data().end()) { - scalar_key_value_pairs.push_back({iter->first, iter->second}); - } - } - } - - return butil::Status::OK(); -} - butil::Status VectorIndexUtils::SplitVectorScalarData( const pb::common::ScalarSchema& scalar_schema, const pb::common::VectorScalardata& vector_scalar_data, std::vector>& scalar_key_value_pairs) { diff --git a/src/vector/vector_index_utils.h b/src/vector/vector_index_utils.h index ff917f8af..eaebb915e 100644 --- a/src/vector/vector_index_utils.h +++ b/src/vector/vector_index_utils.h @@ -183,19 +183,11 @@ class VectorIndexUtils { const pb::common::VectorIndexParameter& target); static butil::Status ValidateVectorIndexParameter(const pb::common::VectorIndexParameter& vector_index_parameter); - [[deprecated("Use another function with the same name")]] static butil::Status ValidateScalarIndexParameter( - const pb::common::ScalarIndexParameter& scalar_index_parameter); - static butil::Status ValidateVectorScalarSchema(const pb::common::ScalarSchema& scalar_schema); static butil::Status ValidateVectorScalarData(const pb::common::ScalarSchema& scalar_schema, const pb::common::VectorScalardata& vector_scalar_data); - [[deprecated("Use another function with the same name")]] static butil::Status SplitVectorScalarData( - const pb::common::ScalarSchema& scalar_schema, const pb::common::VectorScalardata& vector_scalar_data, - std::vector>& scalar_key_value_pairs, // NOLINT - pb::common::VectorScalardata& other_vector_scalar_data); // NOLINT - static butil::Status SplitVectorScalarData( const pb::common::ScalarSchema& scalar_schema, const pb::common::VectorScalardata& vector_scalar_data, std::vector>& scalar_key_value_pairs); // NOLINT diff --git a/test/unit_test/test_vector_index_utils.cc b/test/unit_test/test_vector_index_utils.cc index 29744c056..f0ab7ede0 100644 --- a/test/unit_test/test_vector_index_utils.cc +++ b/test/unit_test/test_vector_index_utils.cc @@ -169,7 +169,7 @@ TEST_F(VectorIndexUtilsTest, ValidateVectorScalarSchema) { TEST_F(VectorIndexUtilsTest, ValidateScalarIndexParameter) { butil::Status ok; pb::common::ScalarIndexParameter scalar_index_parameter; - ok = VectorIndexUtils::ValidateScalarIndexParameter(scalar_index_parameter); + // ok = VectorIndexUtils::ValidateScalarIndexParameter(scalar_index_parameter); EXPECT_EQ(ok.error_code(), pb::error::Errno::OK); }