From 8cbd9158c81bccb1b542fd324dc12c5e1c6289f6 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Tue, 26 Sep 2023 08:29:46 +0100 Subject: [PATCH] feat: add benchmarking suite (#25) --- .github/workflows/test.yml | 2 +- .gitignore | 2 +- Makefile | 48 ++++ README.md | 214 ++++++++++++++++++ bench_test.go | 50 ----- benchmarks/bench_leaf_test.go | 72 ++++++ benchmarks/bench_smst_test.go | 385 +++++++++++++++++++++++++++++++++ benchmarks/bench_smt_test.go | 385 +++++++++++++++++++++++++++++++++ benchmarks/bench_utils_test.go | 127 +++++++++++ benchmarks/proof_sizes_test.go | 175 +++++++++++++++ 10 files changed, 1408 insertions(+), 52 deletions(-) create mode 100644 Makefile delete mode 100644 bench_test.go create mode 100644 benchmarks/bench_leaf_test.go create mode 100644 benchmarks/bench_smst_test.go create mode 100644 benchmarks/bench_smt_test.go create mode 100644 benchmarks/bench_utils_test.go create mode 100644 benchmarks/proof_sizes_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 80c020f..74eb870 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,7 +37,7 @@ jobs: - name: Create coverage report and run tests run: | set -euo pipefail - GODEBUG=netdns=cgo go test -p 1 -json ./... -mod=readonly -timeout 8m -race -coverprofile=coverage.txt -covermode=atomic 2>&1 | tee test_results.json + GODEBUG=netdns=cgo go test -p 1 -json ./ -mod=readonly -timeout 8m -race -coverprofile=coverage.txt -covermode=atomic 2>&1 | tee test_results.json - name: Sanitize test results # We're utilizing `tee` above which can capture non-json stdout output so we need to remove non-json lines before additional parsing and submitting it to the external github action. if: ${{ always() && env.TARGET_GOLANG_VERSION == matrix.go }} diff --git a/.gitignore b/.gitignore index fb94ebf..9f11b75 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1 @@ -testdata/ \ No newline at end of file +.idea/ diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a7ecad8 --- /dev/null +++ b/Makefile @@ -0,0 +1,48 @@ +SHELL := /bin/bash + +.SILENT: + +.PHONY: help +.DEFAULT_GOAL := help +help: ## Prints all the targets in all the Makefiles + @grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +.PHONY: list +list: ## List all make targets + @${MAKE} -pRrn : -f $(MAKEFILE_LIST) 2>/dev/null | awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}' | egrep -v -e '^[^[:alnum:]]' -e '^$@$$' | sort + +.PHONY: test_all +test_all: ## runs the test suite + go test -v -p 1 ./ -mod=readonly -race + +.PHONY: benchmark_all +bechmark_all: ## runs all benchmarks + go test -benchmem -run=^$ -bench Benchmark ./benchmarks -timeout 0 + +.PHONY: benchmark_smt +benchmark_smt: ## runs all benchmarks for the SMT + go test -benchmem -run=^$ -bench=BenchmarkSparseMerkleTree ./benchmarks -timeout 0 + +.PHONY: benchmark_smt_fill +benchmark_smt_fill: ## runs a benchmark on filling the SMT with different amounts of values + go test -benchmem -run=^$ -bench=BenchmarkSparseMerkleTree_Fill ./benchmarks -timeout 0 -benchtime 10x + +.PHONY: benchmark_smt_ops +benchmark_smt_ops: ## runs the benchmarks testing different operations on the SMT against different sized trees + go test -benchmem -run=^$ -bench='BenchmarkSparseMerkleTree_(Update|Get|Prove|Delete)' ./benchmarks -timeout 0 + +.PHONY: benchmark_smst +benchmark_smst: ## runs all benchmarks for the SMST + go test -benchmem -run=^$ -bench=BenchmarkSparseMerkleSumTree ./benchmarks -timeout 0 + +.PHONY: benchmark_smst_fill +benchmark_smst_fill: ## runs a benchmark on filling the SMST with different amounts of values + go test -benchmem -run=^$ -bench=BenchmarkSparseMerkleSumTree_Fill ./benchmarks -timeout 0 -benchtime 10x + +.PHONY: benchmark_smst_ops +benchmark_smst_ops: ## runs the benchmarks testing different operations on the SMST against different sized trees + go test -benchmem -run=^$ -bench='BenchmarkSparseMerkleSumTree_(Update|Get|Prove|Delete)' ./benchmarks -timeout 0 + +.PHONY: bechmark_proof_sizes +benchmark_proof_sizes: ## runs the benchmarks testing the proof sizes for different sized trees + go test -v ./benchmarks -run ProofSizes diff --git a/README.md b/README.md index 56726e2..5d91879 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,13 @@ Note: **Requires Go 1.19+** - [Overview](#overview) - [Documentation](#documentation) +- [Benchmarks](#benchmarks) + - [SMT](#smt) + - [Fill](#fill) + - [Operations](#operations) + - [SMST](#smst) + - [Fill](#fill-1) + - [Operations](#operations-1) ## Overview @@ -19,4 +26,211 @@ This is a Go library that implements a Sparse Merkle tree for a key-value map. T Documentation for the different aspects of this library can be found in the [docs](./docs/) directory. +## Benchmarks + +Benchmarks for the different aspects of this SMT library can be found in [benchmarks](./benchmarks/). In order to run the entire benchmarking suite use the following command: + +```sh +make benchmark_all +``` + +### Definitions + +Below is a list of terms used in the benchmarks' results that may need clarification. + +#### Bytes/Operation (B/op) + - This refers to the number of bytes allocated for each operation. + +#### Commit + - The `Commit` term refers to the `Commit` method of the tree. This takes all changes (which are made in memory) to the tree and writes them to the underlying database. + +#### Sizing + - The tests use the following sizes: 0.1M, 0.5M, 1M, 5M, 10M. The `M` refers to millions hence: + - 0.1M = 100,000 (One hundred thousand) + - 0.5M = 500,000 (Five hundred thousand) + - 1M = 1,000,000 (One million) + - 5M = 5,000,000 (Five million) + - 10M = 10,000,000 (Ten million) + - These sizes refer to the number of key-value pairs or key-value-sum triples inserted into the tree either beforehand or during the benchmark depending on which benchmark it is. + +_NOTE: Unless otherwise stated the benchmarks in this document were ran on a 2023 14-inch Macbook Pro M2 Max with 32GB of RAM. The trees tested are using the `sha256.New()` hasher._ + +_TODO: There is an opportunity to do a fuzz test where we commit every `N` updates, if this ever becomes a bottlneck_ + +### SMT + +In order to run the SMT benchmarks use the following command: + +```sh +make benchmark_smt +``` + +#### Fill + +The "fill" benchmarks cover the time taken to insert `N` key-value pairs into the SMT, as well as how long it takes to do this and commit these changes to disk. This gives us an insight into how long it takes to build a tree of a certain size. + +In order to run the SMT filling benchmarks use the following command: + +```sh +make benchmark_smt_fill +``` + +| Benchmark | # Values | Iterations | Time (s/op) | Bytes (B/op) | Allocations (allocs/op) | +| --------------- |----------| ---------- | --------------- | --------------- | ----------------------- | +| Fill | 0.1M | 10 | 0.162967196 | 159,479,499 | 2,371,598 | +| Fill & Commit | 0.1M | 10 | 2.877307858 | 972,961,486 | 15,992,605 | +| Fill | 0.5M | 10 | 0.926864771 | 890,408,326 | 13,021,258 | +| Fill & Commit | 0.5M | 10 | 16.043430012 | 5,640,034,396 | 82,075,720 | +| Fill | 1M | 10 | 2.033616088 | 1,860,523,968 | 27,041,639 | +| Fill & Commit | 1M | 10 | 32.617249642 | 12,655,347,004 | 166,879,661 | +| Fill | 5M | 10 | 12.502309738 | 10,229,139,731 | 146,821,675 | +| Fill & Commit | 5M | 10 | 175.421250979 | 78,981,342,709 | 870,235,579 | +| Fill | 10M | 10 | 29.718092496 | 21,255,245,031 | 303,637,210 | +| Fill & Commit | 10M | 10 | 396.142675962 | 173,053,933,624 | 1,775,304,977 | + +#### Operations + +The "operations" benchmarks cover the time taken to perform a single operation on an SMT of a given size, and also how long doing this operation followed by a commit would take. This gives us insight into how the SMT operates when filled to differing degrees. + +In order to run the SMT operation benchmarks use the following command: + +```sh +make benchmark_smt_ops +``` + +| Benchmark | Prefilled Values | Iterations | Time (ns/op) | Bytes (B/op) | Allocations (allocs/op) | +| --------------- |------------------| ---------- | ------------ | ------------ | ----------------------- | +| Update | 0.1M | 740,618 | 1,350 | 1,753 | 25 | +| Update & Commit | 0.1M | 21,022 | 54,665 | 13,110 | 281 | +| Update | 0.5M | 605,348 | 1,682 | 1,957 | 26 | +| Update & Commit | 0.5M | 11,697 | 91,028 | 21,501 | 468 | +| Update | 1M | 545,701 | 1,890 | 2,112 | 28 | +| Update & Commit | 1M | 9,540 | 119,347 | 24,983 | 545 | +| Update | 5M | 466,688 | 2,226 | 2,453 | 31 | +| Update & Commit | 5M | 7,906 | 186,026 | 52,621 | 722 | +| Update | 10M | 284,580 | 5,263 | 2,658 | 33 | +| Update & Commit | 10M | 4,484 | 298,376 | 117,923 | 844 | +| Get | 0.1M | 3,923,601 | 303.2 | 48 | 3 | +| Get | 0.5M | 2,209,981 | 577.7 | 48 | 3 | +| Get | 1M | 1,844,431 | 661.6 | 48 | 3 | +| Get | 5M | 1,196,467 | 1,030 | 48 | 3 | +| Get | 10M | 970,195 | 2,667 | 48 | 3 | +| Prove | 0.1M | 829,801 | 1,496 | 2,177 | 17 | +| Prove | 0.5M | 610,402 | 1,835 | 2,747 | 17 | +| Prove | 1M | 605,799 | 1,905 | 2,728 | 17 | +| Prove | 5M | 566,930 | 2,129 | 2,731 | 17 | +| Prove | 10M | 458,472 | 7,113 | 2,735 | 17 | +| Delete | 0.1M | 12,081,112 | 96.18 | 50 | 3 | +| Delete & Commit | 0.1M | 26,490 | 39,568 | 7,835 | 177 | +| Delete | 0.5M | 7,253,522 | 140.3 | 64 | 3 | +| Delete & Commit | 0.5M | 12,766 | 80,518 | 16,696 | 376 | +| Delete | 1M | 1,624,569 | 629.6 | 196 | 4 | +| Delete & Commit | 1M | 9,811 | 135,606 | 20,254 | 456 | +| Delete | 5M | 856,424 | 1,400 | 443 | 6 | +| Delete & Commit | 5M | 8,431 | 151,107 | 74,133 | 626 | +| Delete | 10M | 545,876 | 4,173 | 556 | 6 | +| Delete & Commit | 10M | 3,916 | 271,332 | 108,396 | 772 | + +### SMST + +In order to run the SMST benchmarks use the following command: + +```sh +make benchmark_smst +``` + +#### Fill + +The "fill" benchmarks cover the time taken to insert `N` key-value-sum triples into the SMST, as well as how long it takes to do this and commit these changes to disk. This gives us an insight into how long it takes to build a tree of a certain size. + +In order to run the SMST filling benchmarks use the following command: + +```sh +make benchmark_smst_fill +``` + +| Benchmark | # Values | Iterations | Time (s/op) | Bytes (B/op) | Allocations (allocs/op) | +| --------------- |----------| ---------- | --------------- | --------------- | ----------------------- | +| Fill | 0.1M | 10 | 0.157951888 | 165,878,234 | 2,471,593 | +| Fill & Commit | 0.1M | 10 | 3.011097462 | 1,058,069,050 | 16,664,811 | +| Fill | 0.5M | 10 | 0.927521862 | 922,408,350 | 13,521,259 | +| Fill & Commit | 0.5M | 10 | 15.338199979 | 6,533,439,773 | 85,880,046 | +| Fill | 1M | 10 | 1.982756162 | 1,924,516,467 | 28,041,610 | +| Fill & Commit | 1M | 10 | 31.197517821 | 14,874,342,889 | 175,474,251 | +| Fill | 5M | 10 | 12.054370871 | 10,549,075,488 | 151,821,423 | +| Fill & Commit | 5M | 10 | 176.912009238 | 89,667,234,678 | 914,653,740 | +| Fill | 10M | 10 | 26.859672362 | 21,894,837,504 | 313,635,611 | +| Fill & Commit | 10M | 10 | 490.805535617 | 197,997,807,905 | 1,865,882,489 | + +#### Operations + +The "operations" benchmarks cover the time taken to perform a single operation on an SMST of a given size, and also how long doing this operation followed by a commit would take. This gives us insight into how the SMST operates when filled to differing degrees. + +In order to run the SMST operation benchmarks use the following command: + +```sh +make benchmark_smst_ops +``` + +| Benchmark | Prefilled Values | Iterations | Time (ns/op) | Bytes (B/op) | Allocations (allocs/op) | +| --------------- | ---------------- | ---------- | ------------ | ------------ | ----------------------- | +| Update | 0.1M | 913,760 | 1,477 | 1,843 | 25 | +| Update & Commit | 0.1M | 20,318 | 49,705 | 13,440 | 256 | +| Update | 0.5M | 687,813 | 1,506 | 1,965 | 27 | +| Update & Commit | 0.5M | 14,526 | 83,295 | 37,604 | 428 | +| Update | 1M | 630,310 | 1,679 | 2,076 | 28 | +| Update & Commit | 1M | 11,678 | 122,568 | 25,760 | 501 | +| Update | 5M | 644,193 | 1,850 | 2,378 | 31 | +| Update & Commit | 5M | 6,214 | 184,533 | 60,755 | 723 | +| Update | 10M | 231,714 | 4,962 | 2,616 | 33 | +| Update & Commit | 10M | 4,284 | 279,893 | 77,377 | 830 | +| Get | 0.1M | 3,924,031 | 281.3 | 40 | 2 | +| Get | 0.5M | 2,080,167 | 559.6 | 40 | 2 | +| Get | 1M | 1,609,478 | 718.6 | 40 | 2 | +| Get | 5M | 1,015,630 | 1,105 | 40 | 2 | +| Get | 10M | 352,980 | 2,949 | 40 | 2 | +| Prove | 0.1M | 717,380 | 1,692 | 2,344 | 18 | +| Prove | 0.5M | 618,265 | 1,972 | 3,040 | 19 | +| Prove | 1M | 567,594 | 2,117 | 3,044 | 19 | +| Prove | 5M | 446,062 | 2,289 | 3,045 | 19 | +| Prove | 10M | 122,347 | 11,215 | 3,046 | 19 | +| Delete | 0.1M | 1,000,000 | 1,022 | 1,110 | 7 | +| Delete & Commit | 0.1M | 1,000,000 | 1,039 | 1,110 | 7 | +| Delete | 0.5M | 1,046,163 | 1,159 | 1,548 | 7 | +| Delete & Commit | 0.5M | 907,071 | 1,143 | 1,548 | 7 | +| Delete | 1M | 852,918 | 1,246 | 1,552 | 8 | +| Delete & Commit | 1M | 807,847 | 1,303 | 1,552 | 8 | +| Delete | 5M | 625,662 | 1,604 | 1,552 | 8 | +| Delete & Commit | 5M | 864,432 | 1,382 | 1,552 | 8 | +| Delete | 10M | 232,544 | 4,618 | 1,552 | 8 | +| Delete & Commit | 10M | 224,767 | 5,048 | 1,552 | 8 | + +### Proofs + +To run the tests to average the proof size for numerous prefilled trees use the following command: + +```sh +make benchmark_proof_sizes +``` + +#### SMT + +| Prefilled Size | Average Serialised Proof Size (bytes) | Min (bytes) | Max (bytes) | Average Serialised Compacted Proof Size (bytes) | Min (bytes) | Max (bytes) | +|----------------|---------------------------------------|-------------|-------------|-------------------------------------------------|-------------|-------------| +| 100,000 | 780 | 650 | 1310 | 790 | 692 | 925 | +| 500,000 | 856 | 716 | 1475 | 866 | 758 | 1024 | +| 1,000,000 | 890 | 716 | 1475 | 900 | 758 | 1057 | +| 5,000,000 | 966 | 815 | 1739 | 976 | 858 | 1156 | +| 10,000,000 | 999 | 848 | 1739 | 1010 | 891 | 1189 | + +#### SMST + +| Prefilled Size | Average Serialised Proof Size (bytes) | Min (bytes) | Max (bytes) | Average Serialised Compacted Proof Size (bytes) | Min (bytes) | Max (bytes) | +|----------------|---------------------------------------|-------------|-------------|-------------------------------------------------|-------------|-------------| +| 100,000 | 935 | 780 | 1590 | 937 | 822 | 1101 | +| 500,000 | 1030 | 862 | 1795 | 1032 | 904 | 1224 | +| 1,000,000 | 1071 | 868 | 1795 | 1073 | 910 | 1265 | +| 5,000,000 | 1166 | 975 | 2123 | 1169 | 1018 | 1388 | +| 10,000,000 | 1207 | 1026 | 2123 | 1210 | 1059 | 1429 | + [libra whitepaper]: https://diem-developers-components.netlify.app/papers/the-diem-blockchain/2020-05-26.pdf diff --git a/bench_test.go b/bench_test.go deleted file mode 100644 index 103b07b..0000000 --- a/bench_test.go +++ /dev/null @@ -1,50 +0,0 @@ -package smt - -import ( - "crypto/sha256" - "strconv" - "testing" - - "github.com/stretchr/testify/require" -) - -func BenchmarkSparseMerkleTree_Update(b *testing.B) { - smn, err := NewKVStore("") - require.NoError(b, err) - smv, err := NewKVStore("") - require.NoError(b, err) - smt := NewSMTWithStorage(smn, smv, sha256.New()) - - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - s := strconv.Itoa(i) - _ = smt.Update([]byte(s), []byte(s)) - } - - require.NoError(b, smn.Stop()) - require.NoError(b, smv.Stop()) -} - -func BenchmarkSparseMerkleTree_Delete(b *testing.B) { - smn, err := NewKVStore("") - require.NoError(b, err) - smv, err := NewKVStore("") - require.NoError(b, err) - smt := NewSMTWithStorage(smn, smv, sha256.New()) - - for i := 0; i < 100000; i++ { - s := strconv.Itoa(i) - _ = smt.Update([]byte(s), []byte(s)) - } - - b.ResetTimer() - b.ReportAllocs() - for i := 0; i < b.N; i++ { - s := strconv.Itoa(i) - _ = smt.Delete([]byte(s)) - } - - require.NoError(b, smn.Stop()) - require.NoError(b, smv.Stop()) -} diff --git a/benchmarks/bench_leaf_test.go b/benchmarks/bench_leaf_test.go new file mode 100644 index 0000000..529a065 --- /dev/null +++ b/benchmarks/bench_leaf_test.go @@ -0,0 +1,72 @@ +package smt + +import ( + "crypto/sha256" + "fmt" + "github.com/pokt-network/smt" + "github.com/stretchr/testify/require" + "strconv" + "testing" +) + +func BenchmarkSMTLeafSizes_Fill(b *testing.B) { + treeSizes := []int{100000, 500000, 1000000, 5000000, 10000000} // number of leaves + leafSizes := []int{256, 512, 1024, 2048, 4096, 8192, 16384} // number of bytes per leaf + nodes, err := smt.NewKVStore("") + require.NoError(b, err) + for _, treeSize := range treeSizes { + for _, leafSize := range leafSizes { + leaf := make([]byte, leafSize) + for _, operation := range []string{"Fill", "Fill & Commit"} { + tree := smt.NewSparseMerkleTree(nodes, sha256.New(), smt.WithValueHasher(nil)) + b.ResetTimer() + b.Run( + fmt.Sprintf("%s [Leaf Size: %d bytes] (%d)", operation, leafSize, treeSize), + func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < treeSize; i++ { + require.NoError(b, tree.Update([]byte(strconv.Itoa(i)), leaf)) + } + if operation == "Fill & Commit" { + require.NoError(b, tree.Commit()) + } + }, + ) + require.NoError(b, nodes.ClearAll()) + } + } + } + require.NoError(b, nodes.Stop()) +} + +func BenchmarkSMSTLeafSizes_Fill(b *testing.B) { + treeSizes := []int{100000, 500000, 1000000, 5000000, 10000000} // number of leaves + leafSizes := []int{256, 512, 1024, 2048, 4096, 8192, 16384} // number of bytes per leaf + nodes, err := smt.NewKVStore("") + require.NoError(b, err) + for _, treeSize := range treeSizes { + for _, leafSize := range leafSizes { + leaf := make([]byte, leafSize) + for _, operation := range []string{"Fill", "Fill & Commit"} { + tree := smt.NewSparseMerkleSumTree(nodes, sha256.New(), smt.WithValueHasher(nil)) + b.ResetTimer() + b.Run( + fmt.Sprintf("%s [Leaf Size: %d bytes] (%d)", operation, leafSize, treeSize), + func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + for i := 0; i < treeSize; i++ { + require.NoError(b, tree.Update([]byte(strconv.Itoa(i)), leaf, uint64(i))) + } + if operation == "Fill & Commit" { + require.NoError(b, tree.Commit()) + } + }, + ) + require.NoError(b, nodes.ClearAll()) + } + } + } + require.NoError(b, nodes.Stop()) +} diff --git a/benchmarks/bench_smst_test.go b/benchmarks/bench_smst_test.go new file mode 100644 index 0000000..9b8a455 --- /dev/null +++ b/benchmarks/bench_smst_test.go @@ -0,0 +1,385 @@ +package smt + +import ( + "strconv" + "testing" + + "github.com/pokt-network/smt" + "github.com/stretchr/testify/require" +) + +func BenchmarkSparseMerkleSumTree_Fill(b *testing.B) { + testCases := []struct { + name string + treeSize int + commit bool + persistent bool + }{ + { + name: "Fill (100000)", + treeSize: 100000, + commit: false, + persistent: false, + }, + { + name: "Fill & Commit (100000)", + treeSize: 100000, + commit: true, + persistent: false, + }, + { + name: "Fill (500000)", + treeSize: 500000, + commit: false, + persistent: false, + }, + { + name: "Fill & Commit (500000)", + treeSize: 500000, + commit: true, + persistent: false, + }, + { + name: "Fill (1000000)", + treeSize: 1000000, + commit: false, + persistent: false, + }, + { + name: "Fill & Commit (1000000)", + treeSize: 1000000, + commit: true, + persistent: false, + }, + { + name: "Fill (5000000)", + treeSize: 5000000, + commit: false, + persistent: false, + }, + { + name: "Fill & Commit (5000000)", + treeSize: 5000000, + commit: true, + persistent: false, + }, + { + name: "Fill (10000000)", + treeSize: 10000000, + commit: false, + persistent: false, + }, + { + name: "Fill & Commit (10000000)", + treeSize: 10000000, + commit: true, + persistent: false, + }, + } + + for _, tc := range testCases { + b.ResetTimer() + b.Run(tc.name, func(b *testing.B) { + tree := setupSMST(b, tc.persistent, tc.treeSize) + b.ResetTimer() + b.StartTimer() + b.ReportAllocs() + for n := 0; n < b.N; n++ { + for i := 0; i < tc.treeSize; i++ { + s := strconv.Itoa(i) + require.NoError(b, tree.Update([]byte(s), []byte(s), uint64(i))) + } + if tc.commit { + require.NoError(b, tree.Commit()) + } + } + b.StopTimer() + }) + } +} + +func BenchmarkSparseMerkleSumTree_Update(b *testing.B) { + testCases := []struct { + name string + treeSize int + commit bool + persistent bool + fn func(*smt.SMST, uint64) error + }{ + { + name: "Update (Prefilled: 100000)", + treeSize: 100000, + commit: false, + persistent: false, + fn: updSMST, + }, + { + name: "Update & Commit (Prefilled: 100000)", + treeSize: 100000, + commit: true, + persistent: false, + fn: updSMST, + }, + { + name: "Update (Prefilled: 500000)", + treeSize: 500000, + commit: false, + persistent: false, + fn: updSMST, + }, + { + name: "Update & Commit (Prefilled: 500000)", + treeSize: 500000, + commit: true, + persistent: false, + fn: updSMST, + }, + { + name: "Update (Prefilled: 1000000)", + treeSize: 1000000, + commit: false, + persistent: false, + fn: updSMST, + }, + { + name: "Update & Commit (Prefilled: 1000000)", + treeSize: 1000000, + commit: true, + persistent: false, + fn: updSMST, + }, + { + name: "Update (Prefilled: 5000000)", + treeSize: 5000000, + commit: false, + persistent: false, + fn: updSMST, + }, + { + name: "Update & Commit (Prefilled: 5000000)", + treeSize: 5000000, + commit: true, + persistent: false, + fn: updSMST, + }, + { + name: "Update (Prefilled: 10000000)", + treeSize: 10000000, + commit: false, + persistent: false, + fn: updSMST, + }, + { + name: "Update & Commit (Prefilled: 10000000)", + treeSize: 10000000, + commit: true, + persistent: false, + fn: updSMST, + }, + } + + for _, tc := range testCases { + b.ResetTimer() + b.Run(tc.name, func(b *testing.B) { + tree := setupSMST(b, tc.persistent, tc.treeSize) + benchmarkSMST(b, tree, tc.commit, tc.fn) + }) + } +} + +func BenchmarkSparseMerkleSumTree_Get(b *testing.B) { + testCases := []struct { + name string + treeSize int + commit bool + persistent bool + fn func(*smt.SMST, uint64) error + }{ + { + name: "Get (Prefilled: 100000)", + treeSize: 100000, + commit: false, + persistent: false, + fn: getSMST, + }, + { + name: "Get (Prefilled: 500000)", + treeSize: 500000, + commit: false, + persistent: false, + fn: getSMST, + }, + { + name: "Get (Prefilled: 1000000)", + treeSize: 1000000, + commit: false, + persistent: false, + fn: getSMST, + }, + { + name: "Get (Prefilled: 5000000)", + treeSize: 5000000, + commit: false, + persistent: false, + fn: getSMST, + }, + { + name: "Get (Prefilled: 10000000)", + treeSize: 10000000, + commit: false, + persistent: false, + fn: getSMST, + }, + } + + for _, tc := range testCases { + b.ResetTimer() + b.Run(tc.name, func(b *testing.B) { + tree := setupSMST(b, tc.persistent, tc.treeSize) + benchmarkSMST(b, tree, tc.commit, tc.fn) + }) + } +} + +func BenchmarkSparseMerkleSumTree_Prove(b *testing.B) { + testCases := []struct { + name string + treeSize int + commit bool + persistent bool + fn func(*smt.SMST, uint64) error + }{ + { + name: "Prove (Prefilled: 100000)", + treeSize: 100000, + commit: false, + persistent: false, + fn: proSMST, + }, + { + name: "Prove (Prefilled: 500000)", + treeSize: 500000, + commit: false, + persistent: false, + fn: proSMST, + }, + { + name: "Prove (Prefilled: 1000000)", + treeSize: 1000000, + commit: false, + persistent: false, + fn: proSMST, + }, + { + name: "Prove (Prefilled: 5000000)", + treeSize: 5000000, + commit: false, + persistent: false, + fn: proSMST, + }, + { + name: "Prove (Prefilled: 10000000)", + treeSize: 10000000, + commit: false, + persistent: false, + fn: proSMST, + }, + } + + for _, tc := range testCases { + b.ResetTimer() + b.Run(tc.name, func(b *testing.B) { + tree := setupSMST(b, tc.persistent, tc.treeSize) + benchmarkSMST(b, tree, tc.commit, tc.fn) + }) + } +} + +func BenchmarkSparseMerkleSumTree_Delete(b *testing.B) { + testCases := []struct { + name string + treeSize int + commit bool + persistent bool + fn func(*smt.SMST, uint64) error + }{ + { + name: "Delete (Prefilled: 100000)", + treeSize: 100000, + commit: false, + persistent: false, + fn: delSMST, + }, + { + name: "Delete & Commit (Prefilled: 100000)", + treeSize: 100000, + commit: true, + persistent: false, + fn: delSMST, + }, + { + name: "Delete (Prefilled: 500000)", + treeSize: 500000, + commit: false, + persistent: false, + fn: delSMST, + }, + { + name: "Delete & Commit (Prefilled: 500000)", + treeSize: 500000, + commit: true, + persistent: false, + fn: delSMST, + }, + { + name: "Delete (Prefilled: 1000000)", + treeSize: 1000000, + commit: false, + persistent: false, + fn: delSMST, + }, + { + name: "Delete & Commit (Prefilled: 1000000)", + treeSize: 1000000, + commit: true, + persistent: false, + fn: delSMST, + }, + { + name: "Delete (Prefilled: 5000000)", + treeSize: 5000000, + commit: false, + persistent: false, + fn: delSMST, + }, + { + name: "Delete & Commit (Prefilled: 5000000)", + treeSize: 5000000, + commit: true, + persistent: false, + fn: delSMST, + }, + { + name: "Delete (Prefilled: 10000000)", + treeSize: 10000000, + commit: false, + persistent: false, + fn: delSMST, + }, + { + name: "Delete & Commit (Prefilled: 10000000)", + treeSize: 10000000, + commit: true, + persistent: false, + fn: delSMST, + }, + } + + for _, tc := range testCases { + b.ResetTimer() + b.Run(tc.name, func(b *testing.B) { + tree := setupSMST(b, tc.persistent, tc.treeSize) + benchmarkSMST(b, tree, tc.commit, tc.fn) + }) + } +} diff --git a/benchmarks/bench_smt_test.go b/benchmarks/bench_smt_test.go new file mode 100644 index 0000000..3f85f93 --- /dev/null +++ b/benchmarks/bench_smt_test.go @@ -0,0 +1,385 @@ +package smt + +import ( + "strconv" + "testing" + + "github.com/pokt-network/smt" + "github.com/stretchr/testify/require" +) + +func BenchmarkSparseMerkleTree_Fill(b *testing.B) { + testCases := []struct { + name string + treeSize int + commit bool + persistent bool + }{ + { + name: "Fill (100000)", + treeSize: 100000, + commit: false, + persistent: false, + }, + { + name: "Fill & Commit (100000)", + treeSize: 100000, + commit: true, + persistent: false, + }, + { + name: "Fill (500000)", + treeSize: 500000, + commit: false, + persistent: false, + }, + { + name: "Fill & Commit (500000)", + treeSize: 500000, + commit: true, + persistent: false, + }, + { + name: "Fill (1000000)", + treeSize: 1000000, + commit: false, + persistent: false, + }, + { + name: "Fill & Commit (1000000)", + treeSize: 1000000, + commit: true, + persistent: false, + }, + { + name: "Fill (5000000)", + treeSize: 5000000, + commit: false, + persistent: false, + }, + { + name: "Fill & Commit (5000000)", + treeSize: 5000000, + commit: true, + persistent: false, + }, + { + name: "Fill (10000000)", + treeSize: 10000000, + commit: false, + persistent: false, + }, + { + name: "Fill & Commit (10000000)", + treeSize: 10000000, + commit: true, + persistent: false, + }, + } + + for _, tc := range testCases { + b.ResetTimer() + b.Run(tc.name, func(b *testing.B) { + tree := setupSMT(b, tc.persistent, tc.treeSize) + b.ResetTimer() + b.StartTimer() + b.ReportAllocs() + for n := 0; n < b.N; n++ { + for i := 0; i < tc.treeSize; i++ { + s := strconv.Itoa(i) + require.NoError(b, tree.Update([]byte(s), []byte(s))) + } + if tc.commit { + require.NoError(b, tree.Commit()) + } + } + b.StopTimer() + }) + } +} + +func BenchmarkSparseMerkleTree_Update(b *testing.B) { + testCases := []struct { + name string + treeSize int + commit bool + persistent bool + fn func(*smt.SMT, []byte) error + }{ + { + name: "Update (Prefilled: 100000)", + treeSize: 100000, + commit: false, + persistent: false, + fn: updSMT, + }, + { + name: "Update & Commit (Prefilled: 100000)", + treeSize: 100000, + commit: true, + persistent: false, + fn: updSMT, + }, + { + name: "Update (Prefilled: 500000)", + treeSize: 500000, + commit: false, + persistent: false, + fn: updSMT, + }, + { + name: "Update & Commit (Prefilled: 500000)", + treeSize: 500000, + commit: true, + persistent: false, + fn: updSMT, + }, + { + name: "Update (Prefilled: 1000000)", + treeSize: 1000000, + commit: false, + persistent: false, + fn: updSMT, + }, + { + name: "Update & Commit (Prefilled: 1000000)", + treeSize: 1000000, + commit: true, + persistent: false, + fn: updSMT, + }, + { + name: "Update (Prefilled: 5000000)", + treeSize: 5000000, + commit: false, + persistent: false, + fn: updSMT, + }, + { + name: "Update & Commit (Prefilled: 5000000)", + treeSize: 5000000, + commit: true, + persistent: false, + fn: updSMT, + }, + { + name: "Update (Prefilled: 10000000)", + treeSize: 10000000, + commit: false, + persistent: false, + fn: updSMT, + }, + { + name: "Update & Commit (Prefilled: 10000000)", + treeSize: 10000000, + commit: true, + persistent: false, + fn: updSMT, + }, + } + + for _, tc := range testCases { + b.ResetTimer() + b.Run(tc.name, func(b *testing.B) { + tree := setupSMT(b, tc.persistent, tc.treeSize) + benchmarkSMT(b, tree, tc.commit, tc.fn) + }) + } +} + +func BenchmarkSparseMerkleTree_Get(b *testing.B) { + testCases := []struct { + name string + treeSize int + commit bool + persistent bool + fn func(*smt.SMT, []byte) error + }{ + { + name: "Get (Prefilled: 100000)", + treeSize: 100000, + commit: false, + persistent: false, + fn: getSMT, + }, + { + name: "Get (Prefilled: 500000)", + treeSize: 500000, + commit: false, + persistent: false, + fn: getSMT, + }, + { + name: "Get (Prefilled: 1000000)", + treeSize: 1000000, + commit: false, + persistent: false, + fn: getSMT, + }, + { + name: "Get (Prefilled: 5000000)", + treeSize: 5000000, + commit: false, + persistent: false, + fn: getSMT, + }, + { + name: "Get (Prefilled: 10000000)", + treeSize: 10000000, + commit: false, + persistent: false, + fn: getSMT, + }, + } + + for _, tc := range testCases { + b.ResetTimer() + b.Run(tc.name, func(b *testing.B) { + tree := setupSMT(b, tc.persistent, tc.treeSize) + benchmarkSMT(b, tree, tc.commit, tc.fn) + }) + } +} + +func BenchmarkSparseMerkleTree_Prove(b *testing.B) { + testCases := []struct { + name string + treeSize int + commit bool + persistent bool + fn func(*smt.SMT, []byte) error + }{ + { + name: "Prove (Prefilled: 100000)", + treeSize: 100000, + commit: false, + persistent: false, + fn: proSMT, + }, + { + name: "Prove (Prefilled: 500000)", + treeSize: 500000, + commit: false, + persistent: false, + fn: proSMT, + }, + { + name: "Prove (Prefilled: 1000000)", + treeSize: 1000000, + commit: false, + persistent: false, + fn: proSMT, + }, + { + name: "Prove (Prefilled: 5000000)", + treeSize: 5000000, + commit: false, + persistent: false, + fn: proSMT, + }, + { + name: "Prove (Prefilled: 10000000)", + treeSize: 10000000, + commit: false, + persistent: false, + fn: proSMT, + }, + } + + for _, tc := range testCases { + b.ResetTimer() + b.Run(tc.name, func(b *testing.B) { + tree := setupSMT(b, tc.persistent, tc.treeSize) + benchmarkSMT(b, tree, tc.commit, tc.fn) + }) + } +} + +func BenchmarkSparseMerkleTree_Delete(b *testing.B) { + testCases := []struct { + name string + treeSize int + commit bool + persistent bool + fn func(*smt.SMT, []byte) error + }{ + { + name: "Delete (Prefilled: 100000)", + treeSize: 100000, + commit: false, + persistent: false, + fn: delSMT, + }, + { + name: "Delete & Commit (Prefilled: 100000)", + treeSize: 100000, + commit: true, + persistent: false, + fn: delSMT, + }, + { + name: "Delete (Prefilled: 500000)", + treeSize: 500000, + commit: false, + persistent: false, + fn: delSMT, + }, + { + name: "Delete & Commit (Prefilled: 500000)", + treeSize: 500000, + commit: true, + persistent: false, + fn: delSMT, + }, + { + name: "Delete (Prefilled: 1000000)", + treeSize: 1000000, + commit: false, + persistent: false, + fn: delSMT, + }, + { + name: "Delete & Commit (Prefilled: 1000000)", + treeSize: 1000000, + commit: true, + persistent: false, + fn: delSMT, + }, + { + name: "Delete (Prefilled: 5000000)", + treeSize: 5000000, + commit: false, + persistent: false, + fn: delSMT, + }, + { + name: "Delete & Commit (Prefilled: 5000000)", + treeSize: 5000000, + commit: true, + persistent: false, + fn: delSMT, + }, + { + name: "Delete (Prefilled: 10000000)", + treeSize: 10000000, + commit: false, + persistent: false, + fn: delSMT, + }, + { + name: "Delete & Commit (Prefilled: 10000000)", + treeSize: 10000000, + commit: true, + persistent: false, + fn: delSMT, + }, + } + + for _, tc := range testCases { + b.ResetTimer() + b.Run(tc.name, func(b *testing.B) { + tree := setupSMT(b, tc.persistent, tc.treeSize) + benchmarkSMT(b, tree, tc.commit, tc.fn) + }) + } +} diff --git a/benchmarks/bench_utils_test.go b/benchmarks/bench_utils_test.go new file mode 100644 index 0000000..f2fc4a6 --- /dev/null +++ b/benchmarks/bench_utils_test.go @@ -0,0 +1,127 @@ +package smt + +import ( + "crypto/sha256" + "encoding/binary" + "os" + "strconv" + "testing" + + "github.com/pokt-network/smt" + "github.com/stretchr/testify/require" +) + +var ( + updSMT = func(s *smt.SMT, b []byte) error { + return s.Update(b, b) + } + getSMT = func(s *smt.SMT, b []byte) error { + _, err := s.Get(b) + return err + } + proSMT = func(s *smt.SMT, b []byte) error { + _, err := s.Prove(b) + return err + } + delSMT = func(s *smt.SMT, b []byte) error { + return s.Delete(b) + } + + updSMST = func(s *smt.SMST, i uint64) error { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, i) + return s.Update(b, b, i) + } + getSMST = func(s *smt.SMST, i uint64) error { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, i) + _, _, err := s.Get(b) + return err + } + proSMST = func(s *smt.SMST, i uint64) error { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, i) + _, err := s.Prove(b) + return err + } + delSMST = func(s *smt.SMST, i uint64) error { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, i) + return s.Delete(b) + } +) + +func setupSMT(b *testing.B, persistent bool, numLeaves int) *smt.SMT { + b.Helper() + path := "" + if persistent { + path = b.TempDir() + } + nodes, err := smt.NewKVStore(path) + require.NoError(b, err) + tree := smt.NewSparseMerkleTree(nodes, sha256.New()) + for i := 0; i < numLeaves; i++ { + s := strconv.Itoa(i) + require.NoError(b, tree.Update([]byte(s), []byte(s))) + } + require.NoError(b, tree.Commit()) + b.Cleanup(func() { + require.NoError(b, nodes.ClearAll()) + require.NoError(b, nodes.Stop()) + if path != "" { + require.NoError(b, os.RemoveAll(path)) + } + }) + return tree +} + +func benchmarkSMT(b *testing.B, tree *smt.SMT, commit bool, fn func(*smt.SMT, []byte) error) { + b.ResetTimer() + b.ReportAllocs() + b.StartTimer() + for i := 0; i < b.N; i++ { + s := strconv.Itoa(i) + _ = fn(tree, []byte(s)) + } + if commit { + require.NoError(b, tree.Commit()) + } + b.StopTimer() +} + +func setupSMST(b *testing.B, persistent bool, numLeaves int) *smt.SMST { + b.Helper() + path := "" + if persistent { + path = b.TempDir() + } + nodes, err := smt.NewKVStore(path) + require.NoError(b, err) + tree := smt.NewSparseMerkleSumTree(nodes, sha256.New()) + for i := 0; i < numLeaves; i++ { + s := strconv.Itoa(i) + require.NoError(b, tree.Update([]byte(s), []byte(s), uint64(i))) + } + require.NoError(b, tree.Commit()) + b.Cleanup(func() { + require.NoError(b, nodes.ClearAll()) + require.NoError(b, nodes.Stop()) + if path != "" { + require.NoError(b, os.RemoveAll(path)) + } + }) + return tree +} + +func benchmarkSMST(b *testing.B, tree *smt.SMST, commit bool, fn func(*smt.SMST, uint64) error) { + b.ResetTimer() + b.StartTimer() + b.ReportAllocs() + for i := 0; i < b.N; i++ { + _ = fn(tree, uint64(i)) + } + if commit { + require.NoError(b, tree.Commit()) + } + b.StopTimer() +} diff --git a/benchmarks/proof_sizes_test.go b/benchmarks/proof_sizes_test.go new file mode 100644 index 0000000..dc80d80 --- /dev/null +++ b/benchmarks/proof_sizes_test.go @@ -0,0 +1,175 @@ +package smt + +import ( + "crypto/sha256" + "encoding/binary" + "github.com/pokt-network/smt" + "github.com/stretchr/testify/require" + "testing" +) + +func TestSMT_ProofSizes(t *testing.T) { + nodes, err := smt.NewKVStore("") + require.NoError(t, err) + testCases := []struct { + name string + treeSize int + }{ + { + name: "Proof Size (Prefilled: 100000)", + treeSize: 100000, + }, + { + name: "Proof Size (Prefilled: 500000)", + treeSize: 500000, + }, + { + name: "Proof Size (Prefilled: 1000000)", + treeSize: 1000000, + }, + { + name: "Proof Size (Prefilled: 5000000)", + treeSize: 5000000, + }, + { + name: "Proof Size (Prefilled: 10000000)", + treeSize: 10000000, + }, + } + for _, tc := range testCases { + tree := smt.NewSparseMerkleTree(nodes, sha256.New()) + t.Run(tc.name, func(t *testing.T) { + for i := 0; i < tc.treeSize; i++ { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(i)) + require.NoError(t, tree.Update(b, b)) + } + require.NoError(t, tree.Commit()) + avgProof := uint64(0) + maxProof := uint64(0) + minProof := uint64(0) + avgCompact := uint64(0) + maxCompact := uint64(0) + minCompact := uint64(0) + for i := 0; i < tc.treeSize; i++ { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(i)) + proof, err := tree.Prove(b) + require.NoError(t, err) + require.NotNil(t, proof) + compacted, err := smt.CompactProof(proof, tree.Spec()) + require.NoError(t, err) + require.NotNil(t, compacted) + proofBz, err := proof.Marshal() + require.NoError(t, err) + require.NotNil(t, proofBz) + compactedBz, err := compacted.Marshal() + require.NoError(t, err) + require.NotNil(t, compactedBz) + avgProof += uint64(len(proofBz)) + if uint64(len(proofBz)) > maxProof { + maxProof = uint64(len(proofBz)) + } + if uint64(len(proofBz)) < minProof || i == 0 { + minProof = uint64(len(proofBz)) + } + avgCompact += uint64(len(compactedBz)) + if uint64(len(compactedBz)) > maxCompact { + maxCompact = uint64(len(compactedBz)) + } + if uint64(len(compactedBz)) < minCompact || i == 0 { + minCompact = uint64(len(compactedBz)) + } + } + avgProof /= uint64(tc.treeSize) + avgCompact /= uint64(tc.treeSize) + t.Logf("Average Serialised Proof Size: %d bytes [Min: %d || Max: %d] (Prefilled: %d)", avgProof, minProof, maxProof, tc.treeSize) + t.Logf("Average Serialised Compacted Proof Size: %d bytes [Min: %d || Max: %d] (Prefilled: %d)", avgCompact, minCompact, maxCompact, tc.treeSize) + }) + require.NoError(t, nodes.ClearAll()) + } + require.NoError(t, nodes.Stop()) +} + +func TestSMST_ProofSizes(t *testing.T) { + nodes, err := smt.NewKVStore("") + require.NoError(t, err) + testCases := []struct { + name string + treeSize int + }{ + { + name: "Proof Size (Prefilled: 100000)", + treeSize: 100000, + }, + { + name: "Proof Size (Prefilled: 500000)", + treeSize: 500000, + }, + { + name: "Proof Size (Prefilled: 1000000)", + treeSize: 1000000, + }, + { + name: "Proof Size (Prefilled: 5000000)", + treeSize: 5000000, + }, + { + name: "Proof Size (Prefilled: 10000000)", + treeSize: 10000000, + }, + } + for _, tc := range testCases { + tree := smt.NewSparseMerkleSumTree(nodes, sha256.New()) + t.Run(tc.name, func(t *testing.T) { + for i := 0; i < tc.treeSize; i++ { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(i)) + require.NoError(t, tree.Update(b, b, uint64(i))) + } + require.NoError(t, tree.Commit()) + avgProof := uint64(0) + maxProof := uint64(0) + minProof := uint64(0) + avgCompact := uint64(0) + maxCompact := uint64(0) + minCompact := uint64(0) + for i := 0; i < tc.treeSize; i++ { + b := make([]byte, 8) + binary.BigEndian.PutUint64(b, uint64(i)) + proof, err := tree.Prove(b) + require.NoError(t, err) + require.NotNil(t, proof) + compacted, err := smt.CompactProof(proof, tree.Spec()) + require.NoError(t, err) + require.NotNil(t, compacted) + proofBz, err := proof.Marshal() + require.NoError(t, err) + require.NotNil(t, proofBz) + compactedBz, err := compacted.Marshal() + require.NoError(t, err) + require.NotNil(t, compactedBz) + avgProof += uint64(len(proofBz)) + if uint64(len(proofBz)) > maxProof { + maxProof = uint64(len(proofBz)) + } + if uint64(len(proofBz)) < minProof || i == 0 { + minProof = uint64(len(proofBz)) + } + avgCompact += uint64(len(compactedBz)) + if uint64(len(compactedBz)) > maxCompact { + maxCompact = uint64(len(compactedBz)) + } + if uint64(len(compactedBz)) < minCompact || i == 0 { + minCompact = uint64(len(compactedBz)) + } + } + avgProof /= uint64(tc.treeSize) + avgCompact /= uint64(tc.treeSize) + t.Logf("Average Serialised Proof Size: %d bytes [Min: %d || Max: %d] (Prefilled: %d)", avgProof, minProof, maxProof, tc.treeSize) + t.Logf("Average Serialised Compacted Proof Size: %d bytes [Min: %d || Max: %d] (Prefilled: %d)", avgCompact, minCompact, maxCompact, tc.treeSize) + }) + require.NoError(t, nodes.ClearAll()) + } + require.NoError(t, nodes.Stop()) +}