Skip to content

Commit

Permalink
[chore][coprocessor] Remove deprecated function.
Browse files Browse the repository at this point in the history
  • Loading branch information
Haijun Yu committed Apr 23, 2024
1 parent 3fce387 commit 6bf3c3a
Show file tree
Hide file tree
Showing 5 changed files with 1 addition and 71 deletions.
30 changes: 0 additions & 30 deletions src/vector/vector_index_flat.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -385,22 +385,6 @@ butil::Status VectorIndexFlat::GetMemorySize(int64_t& memory_size) {

bool VectorIndexFlat::IsExceedsMaxElements() { return false; }

void VectorIndexFlat::SearchWithParam(faiss::idx_t n, const faiss::Index::component_t* x, faiss::idx_t k,
faiss::Index::distance_t* distances, faiss::idx_t* labels,
std::shared_ptr<FlatIDSelector> filters) {
faiss::SearchParameters param;
param.sel = filters.get();
raw_index_->search(n, x, k, distances, labels, &param);
faiss::idx_t* li = labels;
// TODO: running by regular user may cause abort exception below
// libgomp: Thread creation failed: Resource temporarily unavailable
// please run by root, or adjust ulimit for regular user, then you can increase FLAGS_omp_num_threads
#pragma omp parallel for
for (faiss::idx_t i = 0; i < n * k; ++i) {
li[i] = li[i] < 0 ? li[i] : index_id_map2_->id_map[li[i]];
}
}

bool VectorIndexFlat::NeedToSave(int64_t last_save_log_behind) {
RWLockReadGuard guard(&rw_lock_);

Expand All @@ -419,21 +403,7 @@ bool VectorIndexFlat::NeedToSave(int64_t last_save_log_behind) {
return false;
}

void VectorIndexFlat::DoRangeSearch(faiss::idx_t n, const faiss::Index::component_t* x, faiss::Index::distance_t radius,
faiss::RangeSearchResult* result,
std::vector<std::shared_ptr<VectorIndex::FilterFunctor>> filters) {
faiss::SearchParameters flat_search_parameters;
auto flat_filter = std::make_shared<FlatIDSelector>(filters);
auto flat_filter_wrapper = std::make_shared<faiss::IDSelectorTranslated>(index_id_map2_->id_map, flat_filter.get());
flat_search_parameters.sel = flat_filter_wrapper.get();

index_id_map2_->index->range_search(n, x, radius, result, &flat_search_parameters);

#pragma omp parallel for
for (faiss::idx_t i = 0; i < result->lims[result->nq]; i++) {
result->labels[i] = result->labels[i] < 0 ? result->labels[i] : index_id_map2_->id_map[result->labels[i]];
}
}

template <typename T>
std::vector<faiss::idx_t> VectorIndexFlat::GetExistVectorIds(const T& ids, size_t size) {
Expand Down
9 changes: 0 additions & 9 deletions src/vector/vector_index_flat.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,15 +112,6 @@ class VectorIndexFlat : public VectorIndex {
bool NeedToSave(int64_t last_save_log_behind) override;

private:
[[deprecated("faiss fix bug. never use.")]] void SearchWithParam(faiss::idx_t n, const faiss::Index::component_t* x,
faiss::idx_t k, faiss::Index::distance_t* distances,
faiss::idx_t* labels,
std::shared_ptr<FlatIDSelector> filters);

[[deprecated("faiss add feature. never use.")]] void DoRangeSearch(
faiss::idx_t n, const faiss::Index::component_t* x, faiss::Index::distance_t radius,
faiss::RangeSearchResult* result, std::vector<std::shared_ptr<VectorIndex::FilterFunctor>> filters);

template <typename T>
std::vector<faiss::idx_t> GetExistVectorIds(const T& ids, size_t size);

Expand Down
23 changes: 0 additions & 23 deletions src/vector/vector_index_utils.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -895,11 +895,6 @@ butil::Status VectorIndexUtils::ValidateVectorIndexParameter(
return VectorIndexUtils::ValidateVectorScalarSchema(vector_index_parameter.scalar_schema());
}

butil::Status VectorIndexUtils::ValidateScalarIndexParameter(
const pb::common::ScalarIndexParameter& /*scalar_index_parameter*/) {
return butil::Status::OK();
}

butil::Status VectorIndexUtils::ValidateVectorScalarSchema(const pb::common::ScalarSchema& scalar_schema) {
DINGO_LOG_IF(INFO, FLAGS_dingo_log_switch_scalar_speed_up_detail)
<< fmt::format("scalar_schema : {}", scalar_schema.DebugString());
Expand Down Expand Up @@ -985,24 +980,6 @@ butil::Status VectorIndexUtils::ValidateVectorScalarData(const pb::common::Scala
return butil::Status::OK();
}

butil::Status VectorIndexUtils::SplitVectorScalarData(
const pb::common::ScalarSchema& scalar_schema, const pb::common::VectorScalardata& vector_scalar_data,
std::vector<std::pair<std::string, pb::common::ScalarValue>>& scalar_key_value_pairs, // NOLINT
pb::common::VectorScalardata& other_vector_scalar_data) {
other_vector_scalar_data = vector_scalar_data;
for (const auto& field : scalar_schema.fields()) {
if (field.enable_speed_up()) {
const std::string& key = field.key();
auto iter = other_vector_scalar_data.scalar_data().find(key);
if (iter != other_vector_scalar_data.scalar_data().end()) {
scalar_key_value_pairs.push_back({iter->first, iter->second});
}
}
}

return butil::Status::OK();
}

butil::Status VectorIndexUtils::SplitVectorScalarData(
const pb::common::ScalarSchema& scalar_schema, const pb::common::VectorScalardata& vector_scalar_data,
std::vector<std::pair<std::string, pb::common::ScalarValue>>& scalar_key_value_pairs) {
Expand Down
8 changes: 0 additions & 8 deletions src/vector/vector_index_utils.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -183,19 +183,11 @@ class VectorIndexUtils {
const pb::common::VectorIndexParameter& target);
static butil::Status ValidateVectorIndexParameter(const pb::common::VectorIndexParameter& vector_index_parameter);

[[deprecated("Use another function with the same name")]] static butil::Status ValidateScalarIndexParameter(
const pb::common::ScalarIndexParameter& scalar_index_parameter);

static butil::Status ValidateVectorScalarSchema(const pb::common::ScalarSchema& scalar_schema);

static butil::Status ValidateVectorScalarData(const pb::common::ScalarSchema& scalar_schema,
const pb::common::VectorScalardata& vector_scalar_data);

[[deprecated("Use another function with the same name")]] static butil::Status SplitVectorScalarData(
const pb::common::ScalarSchema& scalar_schema, const pb::common::VectorScalardata& vector_scalar_data,
std::vector<std::pair<std::string, pb::common::ScalarValue>>& scalar_key_value_pairs, // NOLINT
pb::common::VectorScalardata& other_vector_scalar_data); // NOLINT

static butil::Status SplitVectorScalarData(
const pb::common::ScalarSchema& scalar_schema, const pb::common::VectorScalardata& vector_scalar_data,
std::vector<std::pair<std::string, pb::common::ScalarValue>>& scalar_key_value_pairs); // NOLINT
Expand Down
2 changes: 1 addition & 1 deletion test/unit_test/test_vector_index_utils.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,7 @@ TEST_F(VectorIndexUtilsTest, ValidateVectorScalarSchema) {
TEST_F(VectorIndexUtilsTest, ValidateScalarIndexParameter) {
butil::Status ok;
pb::common::ScalarIndexParameter scalar_index_parameter;
ok = VectorIndexUtils::ValidateScalarIndexParameter(scalar_index_parameter);
// ok = VectorIndexUtils::ValidateScalarIndexParameter(scalar_index_parameter);
EXPECT_EQ(ok.error_code(), pb::error::Errno::OK);
}

Expand Down

0 comments on commit 6bf3c3a

Please sign in to comment.