Skip to content

Commit

Permalink
relaxed memory ordering of refcount increment/decrement
Browse files Browse the repository at this point in the history 
implemented prefetching in ordered trees for pointer keys and keys with a data() member function which returns a pointer (string, string_view, vector)
implemented prefetching for child pointers in make_persistent and ~InternalNodeBase, and in modification operations when current node is marked persistent
  • Loading branch information
@jeffplaisance
jeffplaisance committed Oct 9, 2023
1 parent 0ce3305 commit 5cad5d9
Show file tree
Hide file tree
Showing 4 changed files with 87 additions and 3 deletions.
40 changes: 39 additions & 1 deletion benchmarks/benchmark.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ inline void random_benchmarks(std::integral_constant<N, n>) {
}
}
if constexpr (true) {
using TreeType = BppTreeMap<int, int>::internal_node_bytes<256>::leaf_node_bytes<1024>::Persistent;
using TreeType = BppTreeMap<int, int>::Persistent;
TreeType tree{};
cout << "BppTreeMap<int, int>::Persistent : " << n << endl;
cout << "=============================================================" << endl;
Expand Down Expand Up @@ -155,13 +155,51 @@ inline void sequential_benchmark(T&& tree, std::string const& message) {
cout << sum << endl << endl;
}

template <typename T, typename = void>
struct HasPushBack : std::false_type {};

template <typename T>
struct HasPushBack<T, std::void_t<decltype(std::declval<T>().push_back(std::make_pair(0, 0)))>> : std::true_type {};

template <auto n, typename T>
inline void iterator_benchmark(T&& tree, std::string const& message) {
cout << "Running iterator benchmark using " << message << " with size " << n << endl;
cout << "=============================================================" << endl;
auto startTime = std::chrono::steady_clock::now();
for (int i = 0; i < n; i++) {
if constexpr (HasPushBack<T>::value) {
tree.push_back(std::make_pair(i, i));
} else {
tree[i] = i;
}
}
auto endTime = std::chrono::steady_clock::now();
std::chrono::duration<double> elapsed = endTime - startTime;
cout << elapsed.count() << 's' << endl;

int64_t sum = 0;
startTime = std::chrono::steady_clock::now();
for (auto const& p: std::as_const(tree)) {
sum += p.second;
}
endTime = std::chrono::steady_clock::now();
elapsed = endTime - startTime;
cout << elapsed.count() << 's' << endl;
cout << sum << endl << endl;
}

template <typename N, auto n>
inline void sequential_benchmarks(std::integral_constant<N, n>) {
for (int j = 0; j < 5; ++j) {
sequential_benchmark<n>(absl::btree_map<int, int>(), "absl::btree_map<int, int>");
sequential_benchmark<n>(tlx::btree_map<int, int>(), "tlx::btree_map<int, int>");
sequential_benchmark<n>(BppTreeMap<int, int>::Transient(), "BppTreeMap<int, int>::Transient");
sequential_benchmark<n>(std::map<int, int>(), "std::map<int, int>");

iterator_benchmark<n>(absl::btree_map<int, int>(), "absl::btree_map<int, int>");
iterator_benchmark<n>(tlx::btree_map<int, int>(), "tlx::btree_map<int, int>");
iterator_benchmark<n>(BppTreeMap<int, int>::Transient(), "BppTreeMap<int, int>::Transient");
iterator_benchmark<n>(std::map<int, int>(), "std::map<int, int>");
}
if constexpr (true) {
using TreeType = BppTreeMap<int, int>::internal_node_bytes<256>::leaf_node_bytes<1024>::Persistent;
Expand Down
18 changes: 18 additions & 0 deletions include/bpptree/detail/internalnodebase.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,18 @@ struct InternalNodeBase : public Parent {

NodePtr<ChildType> pointers[internal_size]{};

void prefetch_children(bool condition) {
if (condition) {
for (IndexType i = 0; i < this->length; ++i) {
__builtin_prefetch(&*pointers[i], 1, 3);
}
}
}

~InternalNodeBase() {
prefetch_children(true);
}

static IndexType get_index(uint64_t it) {
return (it >> it_shift) & it_mask;
}
Expand Down Expand Up @@ -303,6 +315,7 @@ struct InternalNodeBase : public Parent {
template <typename T, typename F, typename R, typename S, typename... Args>
void insert(T const& search_val, F&& finder, R&& do_replace, S&& do_split, size_t& size, uint64_t& iter, bool right_most, Args&&... args) {
auto [index, remainder] = finder(this->self(), search_val);
prefetch_children(this->persistent);
pointers[index]->insert(remainder,
finder,
DoReplace<R>(this->self(), index, do_replace, iter),
Expand All @@ -316,6 +329,7 @@ struct InternalNodeBase : public Parent {
template <typename T, typename F, typename R, typename... Args>
void assign(T const& search_val, F&& finder, R&& do_replace, uint64_t& iter, Args&&... args) {
auto [index, remainder] = finder(this->self(), search_val);
prefetch_children(this->persistent);
pointers[index]->assign(remainder,
finder,
DoReplace<R>(this->self(), index, do_replace, iter),
Expand Down Expand Up @@ -390,6 +404,7 @@ struct InternalNodeBase : public Parent {
template <typename T, typename F, typename R, typename E>
void erase(T const& search_val, F&& finder, R&& do_replace, E&& do_erase, size_t& size, uint64_t& iter, bool right_most) {
auto [index, remainder] = finder(this->self(), search_val);
prefetch_children(this->persistent);
pointers[index]->erase(remainder,
finder,
DoReplace<R>(this->self(), index, do_replace, iter),
Expand All @@ -402,6 +417,7 @@ struct InternalNodeBase : public Parent {
template <typename T, typename F, typename R, typename U>
void update(T const& search_val, F&& finder, R&& do_replace, uint64_t& iter, U&& updater) {
auto [index, remainder] = finder(this->self(), search_val);
prefetch_children(this->persistent);
pointers[index]->update(remainder,
finder,
DoReplace<R>(this->self(), index, do_replace, iter),
Expand All @@ -413,6 +429,7 @@ struct InternalNodeBase : public Parent {
template <typename T, typename F, typename R, typename U>
void update2(T const& search_val, F&& finder, R&& do_replace, uint64_t& iter, U&& updater) {
auto [index, remainder] = finder(this->self(), search_val);
prefetch_children(this->persistent);
pointers[index]->update2(remainder,
finder,
DoReplace<R>(this->self(), index, do_replace, iter),
Expand All @@ -428,6 +445,7 @@ struct InternalNodeBase : public Parent {
void make_persistent() {
if (!this->persistent) {
this->persistent = true;
prefetch_children(true);
for (IndexType i = 0; i < this->length; ++i) {
pointers[i]->make_persistent();
}
Expand Down
5 changes: 3 additions & 2 deletions include/bpptree/detail/nodeptr.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,8 @@ class NodePtr {

void dec_ref() {
if (ptr != nullptr) {
if (--ptr->ref_count == 0) {
if (ptr->ref_count.fetch_sub(1, std::memory_order_release) == 1) {
std::atomic_thread_fence(std::memory_order_acquire);
delete ptr;
if constexpr (count_allocations) ++deallocations;
}
Expand All @@ -48,7 +49,7 @@ class NodePtr {

void inc_ref() const {
if (ptr != nullptr) {
++ptr->ref_count;
ptr->ref_count.fetch_add(1, std::memory_order_relaxed);
if constexpr (count_allocations) ++increments;
}
}
Expand Down
27 changes: 27 additions & 0 deletions include/bpptree/detail/ordered_detail.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,12 @@

namespace bpptree::detail {

template <typename T, typename = void>
struct HasDataPtr : std::false_type {};

template <typename T>
struct HasDataPtr<T, std::enable_if_t<std::is_pointer_v<decltype(std::declval<T const&>().data())>>> : std::true_type {};

template <typename KeyValue, typename KeyValueExtractor, typename LessThan, bool binary_search>
struct OrderedDetail {
private:
Expand All @@ -32,6 +38,16 @@ struct OrderedDetail {

template <typename Comp>
IndexType find_key_index(Key const& search_val, Comp const& comp) const {
if constexpr (std::is_pointer_v<Key>) {
for (IndexType i = 0; i < this->length; ++i) {
__builtin_prefetch(extractor.get_key(this->values[i]), 0, 3);
}
}
if constexpr (HasDataPtr<Key>::value) {
for (IndexType i = 0; i < this->length; ++i) {
__builtin_prefetch(extractor.get_key(this->values[i]).data(), 0, 3);
}
}
if constexpr (!binary_search) {
IndexType index = 0;
while (index < this->length) {
Expand Down Expand Up @@ -238,6 +254,16 @@ struct OrderedDetail {

template <typename Comp>
IndexType find_key_index(Key const& search_val, Comp const& comp) const {
if constexpr (std::is_pointer_v<Key>) {
for (IndexType i = 0; i < this->length; ++i) {
__builtin_prefetch(keys[i], 0, 3);
}
}
if constexpr (HasDataPtr<Key>::value) {
for (IndexType i = 0; i < this->length; ++i) {
__builtin_prefetch(keys[i].data(), 0, 3);
}
}
if constexpr (!binary_search) {
IndexType index = 0;
while (index < this->length - 1) {
Expand Down Expand Up @@ -269,6 +295,7 @@ struct OrderedDetail {
void insert_or_assign(Key const& search_val, F&& finder, R&& do_replace, S&& do_split,
size_t& size, uint64_t& iter, bool right_most, Args&& ... args) {
auto [index, remainder] = finder(this->self(), search_val);
this->prefetch_children(this->persistent);
this->pointers[index]->template insert_or_assign<duplicate_policy>(remainder,
finder,
typename Parent::template DoReplace<R>(this->self(), index, do_replace, iter),
Expand Down

0 comments on commit 5cad5d9

Please sign in to comment.