Skip to content

Commit

Permalink
perf: store/cachekv: Correct the naming and implementation for existi…
Browse files Browse the repository at this point in the history
…ng benchmarks (cosmos#10116)

## Description

Cref discussion in cosmos#10026, updates the CacheKV Benchmark naming and implementation to correspond to whats actually going on, and remove many irrelevant/incorrect components from being in the benchmarks timing.

Basically the old Benchmark's iterator creation was very flawed, and never hit the complex case, only repeatedly performing the best case performance of the iterator. Instead it really benchmarked iterator set and next operations.

This PR splits out the benchmarks for those two accordingly.

---

### Author Checklist

*All items are required. Please add a note to the item if the item is not applicable and
please add links to any relevant follow up issues.*

I have...

- [x] included the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [x] added `!` to the type prefix if API or client breaking change
- [x] targeted the correct branch (see [PR Targeting](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#pr-targeting))
- [x] provided a link to the relevant issue or specification
- [x] followed the guidelines for [building modules](https://github.com/cosmos/cosmos-sdk/blob/master/docs/building-modules)
- [x] included the necessary unit and integration [tests](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#testing)
- [x] added a changelog entry to `CHANGELOG.md` - N/A imo
- [x] included comments for [documenting Go code](https://blog.golang.org/godoc)
- [x] updated the relevant documentation or specification
- [x] reviewed "Files changed" and left comments if necessary
- [x] confirmed all CI checks have passed - lint failure unrelated

### Reviewers Checklist

*All items are required. Please add a note if the item is not applicable and please add
your handle next to the items reviewed if you only reviewed selected items.*

I have...

- [ ] confirmed the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [ ] confirmed `!` in the type prefix if API or client breaking change
- [ ] confirmed all author checklist items have been addressed 
- [ ] reviewed state machine logic
- [ ] reviewed API design and naming
- [ ] reviewed documentation is accurate
- [ ] reviewed tests and test coverage
- [ ] manually tested (if applicable)
  • Loading branch information
ValarDragon authored and blewater committed Dec 8, 2021
1 parent a72f992 commit f340afe
Show file tree
Hide file tree
Showing 2 changed files with 113 additions and 23 deletions.
44 changes: 44 additions & 0 deletions store/cachekv/bench_helper_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
package cachekv_test

import "crypto/rand"

func randSlice(sliceSize int) []byte {
bz := make([]byte, sliceSize)
_, _ = rand.Read(bz)
return bz
}

func incrementByteSlice(bz []byte) {
for index := len(bz) - 1; index >= 0; index-- {
if bz[index] < 255 {
bz[index]++
break
} else {
bz[index] = 0
}
}
}

// Generate many keys starting at startKey, and are in sequential order
func generateSequentialKeys(startKey []byte, numKeys int) [][]byte {
toReturn := make([][]byte, 0, numKeys)
cur := make([]byte, len(startKey))
copy(cur, startKey)
for i := 0; i < numKeys; i++ {
newKey := make([]byte, len(startKey))
copy(newKey, cur)
toReturn = append(toReturn, newKey)
incrementByteSlice(cur)
}
return toReturn
}

// Generate many random, unsorted keys
func generateRandomKeys(keySize int, numKeys int) [][]byte {
toReturn := make([][]byte, 0, numKeys)
for i := 0; i < numKeys; i++ {
newKey := randSlice(keySize)
toReturn = append(toReturn, newKey)
}
return toReturn
}
92 changes: 69 additions & 23 deletions store/cachekv/store_bench_test.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package cachekv_test

import (
"crypto/rand"
"sort"
"testing"

dbm "github.com/tendermint/tm-db"
Expand All @@ -11,37 +9,85 @@ import (
"github.com/cosmos/cosmos-sdk/store/dbadapter"
)

func benchmarkCacheKVStoreIterator(numKVs int, b *testing.B) {
var sink interface{}

const defaultValueSizeBz = 1 << 12

// This benchmark measures the time of iterator.Next() when the parent store is blank
func benchmarkBlankParentIteratorNext(b *testing.B, keysize int) {
mem := dbadapter.Store{DB: dbm.NewMemDB()}
kvstore := cachekv.NewStore(mem)
// Use a singleton for value, to not waste time computing it
value := randSlice(defaultValueSizeBz)
// Use simple values for keys, pick a random start,
// and take next b.N keys sequentially after.]
startKey := randSlice(32)

// Add 1 to avoid issues when b.N = 1
keys := generateSequentialKeys(startKey, b.N+1)
for _, k := range keys {
kvstore.Set(k, value)
}

b.ReportAllocs()
b.ResetTimer()

iter := kvstore.Iterator(keys[0], keys[b.N])
defer iter.Close()

for _ = iter.Key(); iter.Valid(); iter.Next() {
// deadcode elimination stub
sink = iter
}
}

// Benchmark setting New keys to a store, where the new keys are in sequence.
func benchmarkBlankParentAppend(b *testing.B, keysize int) {
mem := dbadapter.Store{DB: dbm.NewMemDB()}
cstore := cachekv.NewStore(mem)
keys := make([]string, numKVs)
kvstore := cachekv.NewStore(mem)

for i := 0; i < numKVs; i++ {
key := make([]byte, 32)
value := make([]byte, 32)
// Use a singleton for value, to not waste time computing it
value := randSlice(32)
// Use simple values for keys, pick a random start,
// and take next b.N keys sequentially after.
startKey := randSlice(32)

_, _ = rand.Read(key)
_, _ = rand.Read(value)
keys := generateSequentialKeys(startKey, b.N)

keys[i] = string(key)
cstore.Set(key, value)
b.ReportAllocs()
b.ResetTimer()

for _, k := range keys {
kvstore.Set(k, value)
}
}

sort.Strings(keys)
// Benchmark setting New keys to a store, where the new keys are random.
// the speed of this function does not depend on the values in the parent store
func benchmarkRandomSet(b *testing.B, keysize int) {
mem := dbadapter.Store{DB: dbm.NewMemDB()}
kvstore := cachekv.NewStore(mem)

for n := 0; n < b.N; n++ {
iter := cstore.Iterator([]byte(keys[0]), []byte(keys[numKVs-1]))
// Use a singleton for value, to not waste time computing it
value := randSlice(defaultValueSizeBz)
keys := generateRandomKeys(keysize, b.N)

for _ = iter.Key(); iter.Valid(); iter.Next() {
}
b.ReportAllocs()
b.ResetTimer()

iter.Close()
for _, k := range keys {
kvstore.Set(k, value)
}
}

func BenchmarkCacheKVStoreIterator500(b *testing.B) { benchmarkCacheKVStoreIterator(500, b) }
func BenchmarkCacheKVStoreIterator1000(b *testing.B) { benchmarkCacheKVStoreIterator(1000, b) }
func BenchmarkCacheKVStoreIterator10000(b *testing.B) { benchmarkCacheKVStoreIterator(10000, b) }
func BenchmarkCacheKVStoreIterator50000(b *testing.B) { benchmarkCacheKVStoreIterator(50000, b) }
func BenchmarkCacheKVStoreIterator100000(b *testing.B) { benchmarkCacheKVStoreIterator(100000, b) }
func BenchmarkBlankParentIteratorNextKeySize32(b *testing.B) {
benchmarkBlankParentIteratorNext(b, 32)
}

func BenchmarkBlankParentAppendKeySize32(b *testing.B) {
benchmarkBlankParentAppend(b, 32)
}

func BenchmarkSetKeySize32(b *testing.B) {
benchmarkRandomSet(b, 32)
}

0 comments on commit f340afe

Please sign in to comment.