feat: ADR-040: Add RootStore implementation

CHANGELOG.md

@@ -54,6 +54,7 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * [\#10379](https://github.com/cosmos/cosmos-sdk/pull/10379) Add validation to `x/upgrade` CLI `software-upgrade` command `--plan-info` value.
 * [\#10561](https://github.com/cosmos/cosmos-sdk/pull/10561) Add configurable IAVL cache size to app.toml
 * [\10507](https://github.com/cosmos/cosmos-sdk/pull/10507) Add middleware for tx priority.
+* [\#10430](https://github.com/cosmos/cosmos-sdk/pull/10430) ADR-040: Add store/v2 `MultiStore` implementation
 
 ### Improvements
 

Makefile

@@ -14,6 +14,9 @@ HTTPS_GIT := https://github.com/cosmos/cosmos-sdk.git
 DOCKER := $(shell which docker)
 DOCKER_BUF := $(DOCKER) run --rm -v $(CURDIR):/workspace --workdir /workspace bufbuild/buf:1.0.0-rc8
 PROJECT_NAME = $(shell git remote get-url origin | xargs basename -s .git)
+# RocksDB is a native dependency, so we don't assume the library is installed.
+# Instead, it must be explicitly enabled and we warn when it is not.
+ENABLE_ROCKSDB ?= false
 
 export GO111MODULE = on
 
@@ -61,6 +64,13 @@ ldflags = -X github.com/cosmos/cosmos-sdk/version.Name=sim \
 		  -X "github.com/cosmos/cosmos-sdk/version.BuildTags=$(build_tags_comma_sep)" \
 			-X github.com/tendermint/tendermint/version.TMCoreSemVer=$(TMVERSION)
 
+ifeq ($(ENABLE_ROCKSDB),true)
+  BUILD_TAGS += rocksdb_build
+  test_tags += rocksdb_build
+else
+  $(warning RocksDB support is disabled; to build and test with RocksDB support, set ENABLE_ROCKSDB=true)
+endif
+
 # DB backend selection
 ifeq (cleveldb,$(findstring cleveldb,$(COSMOS_BUILD_OPTIONS)))
   ldflags += -X github.com/cosmos/cosmos-sdk/types.DBBackend=cleveldb
@@ -71,6 +81,9 @@ ifeq (badgerdb,$(findstring badgerdb,$(COSMOS_BUILD_OPTIONS)))
 endif
 # handle rocksdb
 ifeq (rocksdb,$(findstring rocksdb,$(COSMOS_BUILD_OPTIONS)))
+  ifneq ($(ENABLE_ROCKSDB),true)
+    $(error Cannot use RocksDB backend unless ENABLE_ROCKSDB=true)
+  endif
   CGO_ENABLED=1
   BUILD_TAGS += rocksdb
   ldflags += -X github.com/cosmos/cosmos-sdk/types.DBBackend=rocksdb
@@ -132,6 +145,7 @@ mockgen_cmd=go run github.com/golang/mock/mockgen
 mocks: $(MOCKS_DIR)
 	$(mockgen_cmd) -source=client/account_retriever.go -package mocks -destination tests/mocks/account_retriever.go
 	$(mockgen_cmd) -package mocks -destination tests/mocks/tendermint_tm_db_DB.go github.com/tendermint/tm-db DB
+	$(mockgen_cmd) -source db/types.go -package mocks -destination tests/mocks/db/types.go
 	$(mockgen_cmd) -source=types/module/module.go -package mocks -destination tests/mocks/types_module_module.go
 	$(mockgen_cmd) -source=types/invariant.go -package mocks -destination tests/mocks/types_invariant.go
 	$(mockgen_cmd) -source=types/router.go -package mocks -destination tests/mocks/types_router.go
@@ -204,22 +218,24 @@ TEST_TARGETS := test-unit test-unit-amino test-unit-proto test-ledger-mock test-
 # Test runs-specific rules. To add a new test target, just add
 # a new rule, customise ARGS or TEST_PACKAGES ad libitum, and
 # append the new rule to the TEST_TARGETS list.
-test-unit: ARGS=-tags='cgo ledger test_ledger_mock norace'
-test-unit-amino: ARGS=-tags='ledger test_ledger_mock test_amino norace'
-test-ledger: ARGS=-tags='cgo ledger norace'
-test-ledger-mock: ARGS=-tags='ledger test_ledger_mock norace'
-test-race: ARGS=-race -tags='cgo ledger test_ledger_mock'
+test-unit: test_tags += cgo ledger test_ledger_mock norace
+test-unit-amino: test_tags += ledger test_ledger_mock test_amino norace
+test-ledger: test_tags += cgo ledger norace
+test-ledger-mock: test_tags += ledger test_ledger_mock norace
+test-race: test_tags += cgo ledger test_ledger_mock
+test-race: ARGS=-race
 test-race: TEST_PACKAGES=$(PACKAGES_NOSIMULATION)
 $(TEST_TARGETS): run-tests
 
 # check-* compiles and collects tests without running them
 # note: go test -c doesn't support multiple packages yet (https://github.com/golang/go/issues/15513)
 CHECK_TEST_TARGETS := check-test-unit check-test-unit-amino
-check-test-unit: ARGS=-tags='cgo ledger test_ledger_mock norace'
-check-test-unit-amino: ARGS=-tags='ledger test_ledger_mock test_amino norace'
+check-test-unit: test_tags += cgo ledger test_ledger_mock norace
+check-test-unit-amino: test_tags += ledger test_ledger_mock test_amino norace
 $(CHECK_TEST_TARGETS): EXTRA_ARGS=-run=none
 $(CHECK_TEST_TARGETS): run-tests
 
+ARGS += -tags "$(test_tags)"
 SUB_MODULES = $(shell find . -type f -name 'go.mod' -print0 | xargs -0 -n1 dirname | sort)
 CURRENT_DIR = $(shell pwd)
 run-tests:
@@ -484,7 +500,7 @@ localnet-build-dlv:
 
 localnet-build-nodes:
 	$(DOCKER) run --rm -v $(CURDIR)/.testnets:/data cosmossdk/simd \
-    		  testnet init-files --v 4 --starting-ip-address 192.168.10.2 --keyring-backend=test
+			 testnet init-files --v 4 --starting-ip-address 192.168.10.2 --keyring-backend=test
 	docker-compose up -d
 
 localnet-stop:

db/go.mod

@@ -31,6 +31,6 @@ require (
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )
 
-// FIXME: gorocksdb bindings for OptimisticTransactionDB are not merged upstream, so we use a fork
+// Note: gorocksdb bindings for OptimisticTransactionDB are not merged upstream, so we use a fork
 // See https://github.com/tecbot/gorocksdb/pull/216
-replace github.com/tecbot/gorocksdb => github.com/roysc/gorocksdb v1.1.1
+replace github.com/tecbot/gorocksdb => github.com/cosmos/gorocksdb v1.1.1

db/go.sum

@@ -14,6 +14,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/cosmos/gorocksdb v1.1.1 h1:N0OqpEKXgsi2qtDm8T1+AlNMXkTm6s1jowYf7/4pH5I=
+github.com/cosmos/gorocksdb v1.1.1/go.mod h1:b/U29r/CtguX3TF7mKG1Jjn4APDqh4wECshxXdiWHpA=
 github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -99,8 +101,6 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/roysc/gorocksdb v1.1.1 h1:5qKNwi7V/AchRMjyVf5TMCcZP70ro+VyaRmQxzpRvd4=
-github.com/roysc/gorocksdb v1.1.1/go.mod h1:b/U29r/CtguX3TF7mKG1Jjn4APDqh4wECshxXdiWHpA=
 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=

db/memdb/db.go

@@ -25,7 +25,9 @@ const (
 //
 // Versioning is implemented by maintaining references to copy-on-write clones of the backing btree.
 //
-// TODO: Currently transactions do not detect write conflicts, so writers cannot be used concurrently.
+// Note: Currently, transactions do not detect write conflicts, so multiple writers cannot be
+// safely committed to overlapping domains. Because of this, the number of open writers is
+// limited to 1.
 type MemDB struct {
 	btree       *btree.BTree            // Main contents
 	mtx         sync.RWMutex            // Guards version history

db/prefix/prefix.go

@@ -1,39 +1,62 @@
+// Prefixed DB reader/writer types let you namespace multiple DBs within a single DB.
+
 package prefix
 
 import (
 	dbm "github.com/cosmos/cosmos-sdk/db"
 )
 
-// Prefix Reader/Writer lets you namespace multiple DBs within a single DB.
+// prefixed Reader
 type prefixR struct {
 	db     dbm.DBReader
 	prefix []byte
 }
 
+// prefixed ReadWriter
 type prefixRW struct {
 	db     dbm.DBReadWriter
 	prefix []byte
 }
 
+// prefixed Writer
+type prefixW struct {
+	db     dbm.DBWriter
+	prefix []byte
+}
+
 var _ dbm.DBReader = (*prefixR)(nil)
 var _ dbm.DBReadWriter = (*prefixRW)(nil)
+var _ dbm.DBWriter = (*prefixW)(nil)
 
+// NewPrefixReader returns a DBReader that only has access to the subset of DB keys
+// that contain the given prefix.
 func NewPrefixReader(db dbm.DBReader, prefix []byte) prefixR {
 	return prefixR{
 		prefix: prefix,
 		db:     db,
 	}
 }
 
+// NewPrefixReadWriter returns a DBReader that only has access to the subset of DB keys
+// that contain the given prefix.
 func NewPrefixReadWriter(db dbm.DBReadWriter, prefix []byte) prefixRW {
 	return prefixRW{
 		prefix: prefix,
 		db:     db,
 	}
 }
 
+// NewPrefixWriter returns a DBWriter that reads/writes only from the subset of DB keys
+// that contain the given prefix
+func NewPrefixWriter(db dbm.DBWriter, prefix []byte) prefixW {
+	return prefixW{
+		prefix: prefix,
+		db:     db,
+	}
+}
+
 func prefixed(prefix, key []byte) []byte {
-	return append(prefix, key...)
+	return append(cp(prefix), key...)
 }
 
 // Get implements DBReader.
@@ -135,15 +158,42 @@ func (pdb prefixRW) Commit() error { return pdb.db.Commit() }
 // Discard implements DBReadWriter.
 func (pdb prefixRW) Discard() error { return pdb.db.Discard() }
 
-// Returns a slice of the same length (big endian), but incremented by one.
+// Set implements DBReadWriter.
+func (pdb prefixW) Set(key []byte, value []byte) error {
+	if len(key) == 0 {
+		return dbm.ErrKeyEmpty
+	}
+	return pdb.db.Set(prefixed(pdb.prefix, key), value)
+}
+
+// Delete implements DBWriter.
+func (pdb prefixW) Delete(key []byte) error {
+	if len(key) == 0 {
+		return dbm.ErrKeyEmpty
+	}
+	return pdb.db.Delete(prefixed(pdb.prefix, key))
+}
+
+// Close implements DBWriter.
+func (pdb prefixW) Commit() error { return pdb.db.Commit() }
+
+// Discard implements DBReadWriter.
+func (pdb prefixW) Discard() error { return pdb.db.Discard() }
+
+func cp(bz []byte) (ret []byte) {
+	ret = make([]byte, len(bz))
+	copy(ret, bz)
+	return ret
+}
+
+// Returns a new slice of the same length (big endian), but incremented by one.
 // Returns nil on overflow (e.g. if bz bytes are all 0xFF)
 // CONTRACT: len(bz) > 0
 func cpIncr(bz []byte) (ret []byte) {
 	if len(bz) == 0 {
 		panic("cpIncr expects non-zero bz length")
 	}
-	ret = make([]byte, len(bz))
-	copy(ret, bz)
+	ret = cp(bz)
 	for i := len(bz) - 1; i >= 0; i-- {
 		if ret[i] < byte(0xFF) {
 			ret[i]++

db/rocksdb/batch.go

@@ -1,3 +1,5 @@
+//go:build rocksdb_build
+
 package rocksdb
 
 import (

db/rocksdb/db.go

@@ -1,3 +1,5 @@
+//go:build rocksdb_build
+
 package rocksdb
 
 import (

db/rocksdb/db_test.go

@@ -1,7 +1,10 @@
+//go:build rocksdb_build
+
 package rocksdb
 
 import (
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -46,18 +49,29 @@ func TestRevertRecovery(t *testing.T) {
 	dir := t.TempDir()
 	db, err := NewDB(dir)
 	require.NoError(t, err)
-	_, err = db.SaveNextVersion()
-	require.NoError(t, err)
 	txn := db.Writer()
 	require.NoError(t, txn.Set([]byte{1}, []byte{1}))
+	require.NoError(t, txn.Commit())
+	_, err = db.SaveNextVersion()
+	require.NoError(t, err)
+	txn = db.Writer()
 	require.NoError(t, txn.Set([]byte{2}, []byte{2}))
 	require.NoError(t, txn.Commit())
 
-	// make checkpoints dir temporarily unreadable to trigger an error
-	require.NoError(t, os.Chmod(db.checkpointsDir(), 0000))
+	// move checkpoints dir temporarily to trigger an error
+	hideDir := filepath.Join(dir, "hide_checkpoints")
+	require.NoError(t, os.Rename(db.checkpointsDir(), hideDir))
 	require.Error(t, db.Revert())
+	require.NoError(t, os.Rename(hideDir, db.checkpointsDir()))
 
-	require.NoError(t, os.Chmod(db.checkpointsDir(), 0755))
 	db, err = NewDB(dir)
 	require.NoError(t, err)
+	view := db.Reader()
+	val, err := view.Get([]byte{1})
+	require.NoError(t, err)
+	require.Equal(t, []byte{1}, val)
+	val, err = view.Get([]byte{2})
+	require.NoError(t, err)
+	require.Nil(t, val)
+	view.Discard()
 }

db/rocksdb/iterator.go

@@ -1,3 +1,5 @@
+//go:build rocksdb_build
+
 package rocksdb
 
 import (

docs/core/store.md

@@ -232,26 +232,48 @@ Additional information about state streaming configuration can be found in the [
 
 When `KVStore.Set` or `KVStore.Delete` methods are called, `listenkv.Store` automatically writes the operations to the set of `Store.listeners`.
 
-## New Store package (`store/v2`)
+# New Store package (`store/v2`)
 
 The SDK is in the process of transitioning to use the types listed here as the default interface for state storage. At the time of writing, these cannot be used within an application and are not directly compatible with the `CommitMultiStore` and related types.
 
-### `BasicKVStore` interface
+These types use the new `db` sub-module of Cosmos-SDK (`github.com/cosmos/cosmos-sdk/db`), rather than TM-DB (`github.com/tendermint/tm-db`).
+
+See [ADR-040](../architecture/adr-040-storage-and-smt-state-commitments.md) for the motivations and design specifications of the change.
+
+## `BasicKVStore` interface
 
 An interface providing only the basic CRUD functionality (`Get`, `Set`, `Has`, and `Delete` methods), without iteration or caching. This is used to partially expose components of a larger store, such as a `flat.Store`.
 
-### Flat Store
+## MultiStore
+
+This is the new interface (or set of interfaces) for the main client store, replacing the function of `MultiStore`. There are a few significant differences in behavior compared with `MultiStore`:
+  * Commits are atomic and are performed on the entire store state; individual substores cannot be committed separately and cannot have different version numbers.
+  * The store's current version and version history track that of the backing `db.DBConnection`. Past versions are accessible read-only.
+  * The set of valid substores is defined in at initialization and cannot be updated dynamically in an existing store instance.
+
+### `CommitMultiStore`
+
+This is the main interface for persisent application state, analogous to the original `CommitMultiStore`.
+  * Past versions are accessed with `GetVersion`, which returns a `BasicMultiStore`.
+  * Substores are accessed with `GetKVStore`. Trying to get a substore that was not defined at initialization will cause a panic.
+  * `Close` must be called to release the DB resources being used by the store.
+
+### `BasicMultiStore`
+
+A minimal interface that only allows accessing substores. Note: substores returned by `BasicMultiStore.GetKVStore` are read-only.
+
+### Implementation (`root.Store`)
 
-`flat.Store` is the new default persistent store, which internally decouples the concerns of state storage and commitment scheme. Values are stored directly in the backing key-value database (the "storage" bucket), while the value's hash is mapped in a separate store which is able to generate a cryptographic commitment (the "state commitment" bucket, implmented with `smt.Store`).
+The canonical implementation of `MultiStore` is in `store/v2/root`. It internally decouples the concerns of state storage and state commitment: values are stored in, and read directly from, the backing key-value database, but are also mapped in a logically separate store which generates cryptographic proofs (the *state-commitment* store).
 
-This can optionally be constructed to use different backend databases for each bucket.
+The state-commitment component of each substore is implemented as an independent `smt.Store`. Internally, each substore is allocated in a separate partition within the backing DB, such that commits apply to the state of all substores. Likewise, past version state includes the state of all substore storage and state-commitment stores.
 
-<!-- TODO: add link +++ https://github.com/cosmos/cosmos-sdk/blob/v0.44.0/store/v2/flat/store.go -->
+This store can optionally be configured to use different backend databases for each bucket (e.g., `badgerdb` for the state storage DB and `memdb` for the state-commitment DB).
 
-### SMT Store
+## SMT Store
 
-A `BasicKVStore` which is used to partially expose functions of an underlying store (for instance, to allow access to the commitment store in `flat.Store`).
+`store/v2/smt.Store` maps values into a Sparse Merkle Tree (SMT), and supports a `BasicKVStore` interface as well as methods for cryptographic proof generation.
 
-## Next {hide}
+# Next {hide}
 
 Learn about [encoding](./encoding.md) {hide}

internal/db/iterator_adapter.go

@@ -0,0 +1,24 @@
+package db
+
+import (
+	dbm "github.com/cosmos/cosmos-sdk/db"
+	storetypes "github.com/cosmos/cosmos-sdk/store/types"
+)
+
+var _ = (*storetypes.Iterator)(nil)
+
+type dbAsStoreIter struct {
+	dbm.Iterator
+	valid bool
+}
+
+// DBToStoreIterator returns an iterator wrapping the given iterator so that it satisfies the
+// (store/types).Iterator interface.
+func DBToStoreIterator(source dbm.Iterator) *dbAsStoreIter {
+	ret := &dbAsStoreIter{Iterator: source}
+	ret.Next()
+	return ret
+}
+
+func (it *dbAsStoreIter) Next()       { it.valid = it.Iterator.Next() }
+func (it *dbAsStoreIter) Valid() bool { return it.valid }

scripts/module-tests.sh

@@ -21,7 +21,7 @@ execute_mod_tests() {
 
     echo ">>> running $go_mod tests"
     cd $mod_dir;
-    go test -mod=readonly -timeout 30m -coverprofile=${root_dir}/${coverage_file}.tmp -covermode=atomic -tags='norace ledger test_ledger_mock'  ./...
+    go test -mod=readonly -timeout 30m -coverprofile=${root_dir}/${coverage_file}.tmp -covermode=atomic -tags='norace ledger test_ledger_mock rocksdb_build'  ./...
     local ret=$?
     echo "test return: " $ret;
     cd -;