Skip to content

Commit

Permalink
feat: Allow querying block number for tree indices (#10332)
Browse files Browse the repository at this point in the history 
This PR make the following changes.

1. Captures and propagates more of the errors generated in the merkle
trees out to the TS interface.
2. Introduces the `block_number_t` typedef within the native world
state.
3. Introduces a new DB in the native world state. This DB maps block
numbers to the size of the tree at that block. It then uses this DB to
fulfill queries looking to identify which block given notes were
included within.
  • Loading branch information
@PhilWindle
PhilWindle authored Dec 2, 2024
1 parent 2a07355 commit cf05a7a
Show file tree
Hide file tree
Showing 33 changed files with 1,402 additions and 193 deletions.
2 changes: 1 addition & 1 deletion barretenberg/cpp/scripts/merkle_tree_tests.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ set -e
# run commands relative to parent directory
cd $(dirname $0)/..

DEFAULT_TESTS=PersistedIndexedTreeTest.*:PersistedAppendOnlyTreeTest.*:LMDBStoreTest.*:PersistedContentAddressedIndexedTreeTest.*:PersistedContentAddressedAppendOnlyTreeTest.*
DEFAULT_TESTS=PersistedIndexedTreeTest.*:PersistedAppendOnlyTreeTest.*:LMDBTreeStoreTest.*:PersistedContentAddressedIndexedTreeTest.*:PersistedContentAddressedAppendOnlyTreeTest.*
TEST=${1:-$DEFAULT_TESTS}
PRESET=${PRESET:-clang16}

Expand Down
120 changes: 95 additions & 25 deletions ...c/barretenberg/crypto/merkle_tree/append_only_tree/content_addressed_append_only_tree.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,17 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
using StoreType = Store;

// Asynchronous methods accept these callback function types as arguments
using AppendCompletionCallback = std::function<void(const TypedResponse<AddDataResponse>&)>;
using MetaDataCallback = std::function<void(const TypedResponse<TreeMetaResponse>&)>;
using HashPathCallback = std::function<void(const TypedResponse<GetSiblingPathResponse>&)>;
using FindLeafCallback = std::function<void(const TypedResponse<FindLeafIndexResponse>&)>;
using GetLeafCallback = std::function<void(const TypedResponse<GetLeafResponse>&)>;
using AppendCompletionCallback = std::function<void(TypedResponse<AddDataResponse>&)>;
using MetaDataCallback = std::function<void(TypedResponse<TreeMetaResponse>&)>;
using HashPathCallback = std::function<void(TypedResponse<GetSiblingPathResponse>&)>;
using FindLeafCallback = std::function<void(TypedResponse<FindLeafIndexResponse>&)>;
using GetLeafCallback = std::function<void(TypedResponse<GetLeafResponse>&)>;
using CommitCallback = std::function<void(TypedResponse<CommitResponse>&)>;
using RollbackCallback = std::function<void(const Response&)>;
using RollbackCallback = std::function<void(Response&)>;
using RemoveHistoricBlockCallback = std::function<void(TypedResponse<RemoveHistoricResponse>&)>;
using UnwindBlockCallback = std::function<void(TypedResponse<UnwindResponse>&)>;
using FinaliseBlockCallback = std::function<void(const Response&)>;
using FinaliseBlockCallback = std::function<void(Response&)>;
using GetBlockForIndexCallback = std::function<void(TypedResponse<BlockForIndexResponse>&)>;

// Only construct from provided store and thread pool, no copies or moves
ContentAddressedAppendOnlyTree(std::unique_ptr<Store> store,
Expand Down Expand Up @@ -90,7 +91,7 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
* @param includeUncommitted Whether to include uncommitted changes
*/
void get_sibling_path(const index_t& index,
const index_t& blockNumber,
const block_number_t& blockNumber,
const HashPathCallback& on_completion,
bool includeUncommitted) const;

Expand Down Expand Up @@ -131,7 +132,7 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
* @param includeUncommitted Whether to include uncommitted changes
* @param on_completion Callback to be called on completion
*/
void get_meta_data(const index_t& blockNumber,
void get_meta_data(const block_number_t& blockNumber,
bool includeUncommitted,
const MetaDataCallback& on_completion) const;

Expand All @@ -151,7 +152,7 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
* @param on_completion Callback to be called on completion
*/
void get_leaf(const index_t& index,
const index_t& blockNumber,
const block_number_t& blockNumber,
bool includeUncommitted,
const GetLeafCallback& completion) const;

Expand All @@ -164,7 +165,7 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
* @brief Returns the index of the provided leaf in the tree
*/
void find_leaf_index(const fr& leaf,
const index_t& blockNumber,
const block_number_t& blockNumber,
bool includeUncommitted,
const FindLeafCallback& on_completion) const;

Expand All @@ -181,10 +182,23 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
*/
void find_leaf_index_from(const fr& leaf,
const index_t& start_index,
const index_t& blockNumber,
const block_number_t& blockNumber,
bool includeUncommitted,
const FindLeafCallback& on_completion) const;

/**
* @brief Returns the block numbers that correspond to the given indices values
*/
void find_block_numbers(const std::vector<index_t>& indices, const GetBlockForIndexCallback& on_completion) const;

/**
* @brief Returns the block numbers that correspond to the given indices values, from the perspective of a
* historical block number
*/
void find_block_numbers(const std::vector<index_t>& indices,
const block_number_t& blockNumber,
const GetBlockForIndexCallback& on_completion) const;

/**
* @brief Commit the tree to the backing store
*/
Expand All @@ -200,11 +214,11 @@ template <typename Store, typename HashingPolicy> class ContentAddressedAppendOn
*/
uint32_t depth() const { return depth_; }

void remove_historic_block(const index_t& blockNumber, const RemoveHistoricBlockCallback& on_completion);
void remove_historic_block(const block_number_t& blockNumber, const RemoveHistoricBlockCallback& on_completion);

void unwind_block(const index_t& blockNumber, const UnwindBlockCallback& on_completion);
void unwind_block(const block_number_t& blockNumber, const UnwindBlockCallback& on_completion);

void finalise_block(const index_t& blockNumber, const FinaliseBlockCallback& on_completion);
void finalise_block(const block_number_t& blockNumber, const FinaliseBlockCallback& on_completion);

protected:
using ReadTransaction = typename Store::ReadTransaction;
Expand Down Expand Up @@ -326,7 +340,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_meta_data(bool in
}

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_meta_data(const index_t& blockNumber,
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_meta_data(const block_number_t& blockNumber,
bool includeUncommitted,
const MetaDataCallback& on_completion) const
{
Expand Down Expand Up @@ -361,7 +375,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_sibling_path(cons

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_sibling_path(const index_t& index,
const index_t& blockNumber,
const block_number_t& blockNumber,
const HashPathCallback& on_completion,
bool includeUncommitted) const
{
Expand Down Expand Up @@ -393,6 +407,62 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_sibling_path(cons
workers_->enqueue(job);
}

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::find_block_numbers(
const std::vector<index_t>& indices, const GetBlockForIndexCallback& on_completion) const
{
auto job = [=, this]() {
execute_and_report<BlockForIndexResponse>(
[=, this](TypedResponse<BlockForIndexResponse>& response) {
response.inner.blockNumbers.reserve(indices.size());
TreeMeta meta;
ReadTransactionPtr tx = store_->create_read_transaction();
store_->get_meta(meta, *tx, true);
index_t maxIndex = meta.committedSize;
for (index_t index : indices) {
bool outOfRange = index >= maxIndex;
std::optional<block_number_t> block =
outOfRange ? std::nullopt : store_->find_block_for_index(index, *tx);
response.inner.blockNumbers.emplace_back(block);
}
},
on_completion);
};
workers_->enqueue(job);
}

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::find_block_numbers(
const std::vector<index_t>& indices,
const block_number_t& blockNumber,
const GetBlockForIndexCallback& on_completion) const
{
auto job = [=, this]() {
execute_and_report<BlockForIndexResponse>(
[=, this](TypedResponse<BlockForIndexResponse>& response) {
response.inner.blockNumbers.reserve(indices.size());
TreeMeta meta;
BlockPayload blockPayload;
ReadTransactionPtr tx = store_->create_read_transaction();
store_->get_meta(meta, *tx, true);
if (!store_->get_block_data(blockNumber, blockPayload, *tx)) {
throw std::runtime_error(format("Unable to find block numbers for indices for block ",
blockNumber,
", failed to get block data."));
}
index_t maxIndex = std::min(meta.committedSize, blockPayload.size);
for (index_t index : indices) {
bool outOfRange = index >= maxIndex;
std::optional<block_number_t> block =
outOfRange ? std::nullopt : store_->find_block_for_index(index, *tx);
response.inner.blockNumbers.emplace_back(block);
}
},
on_completion);
};
workers_->enqueue(job);
}

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_subtree_sibling_path(
uint32_t subtree_depth, const HashPathCallback& on_completion, bool includeUncommitted) const
Expand Down Expand Up @@ -473,7 +543,7 @@ std::optional<fr> ContentAddressedAppendOnlyTree<Store, HashingPolicy>::find_lea
NodePayload nodePayload;
bool success = store_->get_node_by_hash(hash, nodePayload, tx, requestContext.includeUncommitted);
if (!success) {
// std::cout << "No root" << std::endl;
// std::cout << "No root " << hash << std::endl;
return std::nullopt;
}
// std::cout << "Found root at depth " << i << " : " << hash << std::endl;
Expand Down Expand Up @@ -601,7 +671,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_leaf(const index_

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::get_leaf(const index_t& leaf_index,
const index_t& blockNumber,
const block_number_t& blockNumber,
bool includeUncommitted,
const GetLeafCallback& on_completion) const
{
Expand Down Expand Up @@ -657,7 +727,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::find_leaf_index(const

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::find_leaf_index(const fr& leaf,
const index_t& blockNumber,
const block_number_t& blockNumber,
bool includeUncommitted,
const FindLeafCallback& on_completion) const
{
Expand Down Expand Up @@ -696,7 +766,7 @@ template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::find_leaf_index_from(
const fr& leaf,
const index_t& start_index,
const index_t& blockNumber,
const block_number_t& blockNumber,
bool includeUncommitted,
const FindLeafCallback& on_completion) const
{
Expand Down Expand Up @@ -788,7 +858,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::rollback(const Rollba

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::remove_historic_block(
const index_t& blockNumber, const RemoveHistoricBlockCallback& on_completion)
const block_number_t& blockNumber, const RemoveHistoricBlockCallback& on_completion)
{
auto job = [=, this]() {
execute_and_report<RemoveHistoricResponse>(
Expand All @@ -804,7 +874,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::remove_historic_block
}

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::unwind_block(const index_t& blockNumber,
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::unwind_block(const block_number_t& blockNumber,
const UnwindBlockCallback& on_completion)
{
auto job = [=, this]() {
Expand All @@ -821,7 +891,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::unwind_block(const in
}

template <typename Store, typename HashingPolicy>
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::finalise_block(const index_t& blockNumber,
void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::finalise_block(const block_number_t& blockNumber,
const FinaliseBlockCallback& on_completion)
{
auto job = [=, this]() {
Expand Down Expand Up @@ -981,7 +1051,7 @@ void ContentAddressedAppendOnlyTree<Store, HashingPolicy>::add_batch_internal(
new_root = new_hash;
meta.root = new_hash;
meta.size = new_size;
// std::cout << "New size: " << meta.size << std::endl;
// std::cout << "New size: " << meta.size << ", root " << meta.root << std::endl;
store_->put_meta(meta);
}

Expand Down
Loading

1 comment on commit cf05a7a

@AztecBot
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Performance Alert ⚠️

Possible performance regression was detected for benchmark 'C++ Benchmark'.
Benchmark result of this commit is worse than the previous benchmark result exceeding threshold 1.05.

Benchmark suite Current: cf05a7a Previous: 887c011 Ratio
nativeconstruct_proof_ultrahonk_power_of_2/20 5038.110503999988 ms/iter 4583.430852999996 ms/iter 1.10
Goblin::merge(t) 140911800 ns/iter 133738848 ns/iter 1.05

This comment was automatically generated by workflow using github-action-benchmark.

CC: @ludamad @codygunton

Please sign in to comment.