Skip to content

Commit

Permalink
[fix][store] Delete some log print.
Browse files Browse the repository at this point in the history
  • Loading branch information
@rock-git @ketor
rock-git authored and ketor committed Apr 18, 2024
1 parent f872578 commit f617bbe
Show file tree
Hide file tree
Showing 5 changed files with 14 additions and 23 deletions.
19 changes: 9 additions & 10 deletions src/vector/vector_index_flat.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,11 @@ bvar::LatencyRecorder g_flat_range_search_latency("dingo_flat_range_search_laten
bvar::LatencyRecorder g_flat_delete_latency("dingo_flat_delete_latency");
bvar::LatencyRecorder g_flat_load_latency("dingo_flat_load_latency");

template std::vector<faiss::idx_t> VectorIndexFlat::GetRepeatedIds(const std::unique_ptr<faiss::idx_t[]>& ids,
size_t size);
template std::vector<faiss::idx_t> VectorIndexFlat::GetExistVectorIds(const std::unique_ptr<faiss::idx_t[]>& ids,
size_t size);

template std::vector<faiss::idx_t> VectorIndexFlat::GetRepeatedIds(const std::vector<faiss::idx_t>& ids, size_t size);
template std::vector<faiss::idx_t> VectorIndexFlat::GetExistVectorIds(const std::vector<faiss::idx_t>& ids,
size_t size);

VectorIndexFlat::VectorIndexFlat(int64_t id, const pb::common::VectorIndexParameter& vector_index_parameter,
const pb::common::RegionEpoch& epoch, const pb::common::Range& range,
Expand Down Expand Up @@ -106,7 +107,7 @@ butil::Status VectorIndexFlat::AddOrUpsert(const std::vector<pb::common::VectorW

// delete id exists.
if (!index_id_map2_->rev_map.empty()) {
std::vector<faiss::idx_t> internal_ids = GetRepeatedIds(ids, vector_with_ids.size());
std::vector<faiss::idx_t> internal_ids = GetExistVectorIds(ids, vector_with_ids.size());

if (!internal_ids.empty()) {
faiss::IDSelectorBatch sel(internal_ids.size(), internal_ids.data());
Expand Down Expand Up @@ -139,7 +140,7 @@ butil::Status VectorIndexFlat::Delete(const std::vector<int64_t>& delete_ids) {

// delete id exists.
if (!index_id_map2_->rev_map.empty()) {
std::vector<faiss::idx_t> internal_ids = GetRepeatedIds(delete_ids, delete_ids.size());
std::vector<faiss::idx_t> internal_ids = GetExistVectorIds(delete_ids, delete_ids.size());

if (!internal_ids.empty()) {
faiss::IDSelectorBatch sel(internal_ids.size(), internal_ids.data());
Expand Down Expand Up @@ -266,9 +267,7 @@ butil::Status VectorIndexFlat::Save(const std::string& path) {
try {
faiss::write_index(index_id_map2_.get(), path.c_str());
} catch (std::exception& e) {
DINGO_LOG(ERROR) << fmt::format("[vector_index.flat][id({})] write index failed, exception: {} path: {}", Id(),
e.what(), path);
return butil::Status(pb::error::Errno::EINTERNAL, fmt::format("write index exception: ", e.what()));
return butil::Status(pb::error::Errno::EINTERNAL, fmt::format("write index exception: {}", e.what()));
}

return butil::Status();
Expand All @@ -287,7 +286,7 @@ butil::Status VectorIndexFlat::Load(const std::string& path) {
internal_raw_index = faiss::read_index(path.c_str(), 0);
} catch (std::exception& e) {
delete internal_raw_index;
return butil::Status(pb::error::Errno::EINTERNAL, fmt::format("read index exception: {}", path, e.what()));
return butil::Status(pb::error::Errno::EINTERNAL, fmt::format("read index exception: {} {}", path, e.what()));
}

faiss::IndexIDMap2* internal_index = dynamic_cast<faiss::IndexIDMap2*>(internal_raw_index);
Expand Down Expand Up @@ -437,7 +436,7 @@ void VectorIndexFlat::DoRangeSearch(faiss::idx_t n, const faiss::Index::componen
}

template <typename T>
std::vector<faiss::idx_t> VectorIndexFlat::GetRepeatedIds(const T& ids, size_t size) {
std::vector<faiss::idx_t> VectorIndexFlat::GetExistVectorIds(const T& ids, size_t size) {
std::vector<faiss::idx_t> internal_ids;
internal_ids.reserve(size);
for (int i = 0; i < size; i++) {
Expand Down
2 changes: 1 addition & 1 deletion src/vector/vector_index_flat.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ class VectorIndexFlat : public VectorIndex {
faiss::RangeSearchResult* result, std::vector<std::shared_ptr<VectorIndex::FilterFunctor>> filters);

template <typename T>
std::vector<faiss::idx_t> GetRepeatedIds(const T& ids, size_t size);
std::vector<faiss::idx_t> GetExistVectorIds(const T& ids, size_t size);

// Dimension of the elements
faiss::idx_t dimension_;
Expand Down
4 changes: 1 addition & 3 deletions src/vector/vector_index_ivf_flat.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -315,9 +315,7 @@ butil::Status VectorIndexIvfFlat::Save(const std::string& path) {
try {
faiss::write_index(index_.get(), path.c_str());
} catch (std::exception& e) {
DINGO_LOG(ERROR) << fmt::format("[vector_index.ivf_flat][id({})] write index failed, exception: {} path: {}", Id(),
e.what(), path);
return butil::Status(pb::error::Errno::EINTERNAL, fmt::format("write index exception: ", e.what()));
return butil::Status(pb::error::Errno::EINTERNAL, fmt::format("write index exception: {}", e.what()));
}

return butil::Status();
Expand Down
2 changes: 0 additions & 2 deletions src/vector/vector_index_ivf_pq.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -521,8 +521,6 @@ bool VectorIndexIvfPq::IsTrainedImpl() {
is_trained = false;
}

DINGO_LOG(DEBUG) << fmt::format("[vector_index.ivf_pq][id({})] is train {}", Id(), is_trained);

return is_trained;
}

Expand Down
10 changes: 3 additions & 7 deletions src/vector/vector_index_raw_ivf_pq.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -238,8 +238,7 @@ butil::Status VectorIndexRawIvfPq::RangeSearch(const std::vector<pb::common::Vec
RWLockReadGuard guard(&rw_lock_);

if (BAIDU_UNLIKELY(!IsTrainedImpl())) {
std::string s = fmt::format("raw ivf pq not train. train first. ignored");
DINGO_LOG(WARNING) << s;
DINGO_LOG(WARNING) << fmt::format("raw ivf pq not train. train first. ignored");

for (size_t row = 0; row < vector_with_ids.size(); ++row) {
auto& result = results.emplace_back();
Expand Down Expand Up @@ -298,9 +297,7 @@ butil::Status VectorIndexRawIvfPq::Save(const std::string& path) {
try {
faiss::write_index(index_.get(), path.c_str());
} catch (std::exception& e) {
DINGO_LOG(ERROR) << fmt::format("[vector_index.raw_ivf_pq][id({})] write index failed, exception: {} path: {}",
Id(), e.what(), path);
return butil::Status(pb::error::Errno::EINTERNAL, fmt::format("write index exception: ", e.what()));
return butil::Status(pb::error::Errno::EINTERNAL, fmt::format("write index exception: {}", e.what()));
}

return butil::Status();
Expand All @@ -318,7 +315,7 @@ butil::Status VectorIndexRawIvfPq::Load(const std::string& path) {

} catch (std::exception& e) {
delete internal_raw_index;
return butil::Status(pb::error::Errno::EINTERNAL, fmt::format("read index exception: {}", path, e.what()));
return butil::Status(pb::error::Errno::EINTERNAL, fmt::format("read index exception: {} {}", path, e.what()));
}

faiss::IndexIVFPQ* internal_index = dynamic_cast<faiss::IndexIVFPQ*>(internal_raw_index);
Expand Down Expand Up @@ -578,7 +575,6 @@ bool VectorIndexRawIvfPq::IsTrainedImpl() {
? index_->is_trained
: false;

DINGO_LOG(DEBUG) << fmt::format("[vector_index.raw_ivf_pq][id({})] is train {}", Id(), is_trained);
return is_trained;
}

Expand Down

0 comments on commit f617bbe

Please sign in to comment.