From 4cc992fa55d7e7e38dce14b8d918c2e0b1c3f2a7 Mon Sep 17 00:00:00 2001 From: Sai Kumar Date: Tue, 18 Jan 2022 12:48:46 +0530 Subject: [PATCH 01/12] feat: add migration for adr40 State migration from iavl to smt --- store/rootmulti/store.go | 5 +++ store/v2/multi/migration.go | 70 +++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 store/v2/multi/migration.go diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index fd4bc84ac2a..dafe321222d 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -93,6 +93,11 @@ func (rs *Store) GetPruning() types.PruningOptions { return rs.pruningOpts } +// GetStores will return the stores +func (rs *Store) GetStores() map[types.StoreKey]types.CommitKVStore { + return rs.stores +} + // SetPruning sets the pruning strategy on the root store and all the sub-stores. // Note, calling SetPruning on the root store prior to LoadVersion or // LoadLatestVersion performs a no-op as the stores aren't mounted yet. diff --git a/store/v2/multi/migration.go b/store/v2/multi/migration.go new file mode 100644 index 00000000000..8e48882a95a --- /dev/null +++ b/store/v2/multi/migration.go @@ -0,0 +1,70 @@ +package root + +import ( + prefixdb "github.com/cosmos/cosmos-sdk/db/prefix" + "github.com/cosmos/cosmos-sdk/store/iavl" + "github.com/cosmos/cosmos-sdk/store/mem" + v1Store "github.com/cosmos/cosmos-sdk/store/rootmulti" + "github.com/cosmos/cosmos-sdk/store/transient" + "github.com/cosmos/cosmos-sdk/store/types" + sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" + "sort" + "strings" +) + +// MigrationFromIAVLStoreToSMTStore will migrate the complete state from iavl to smt +func MigrationFromIAVLStoreToSMTStore(rs *v1Store.Store, rootStore *Store) error { + // Collect stores to snapshot (only IAVL stores are supported) + type namedStore struct { + *iavl.Store + name string + } + var stores []namedStore + for key := range rs.GetStores() { + switch store := rs.GetCommitKVStore(key).(type) { + case *iavl.Store: + stores = append(stores, namedStore{name: key.Name(), Store: store}) + case *transient.Store, *mem.Store: + continue + default: + continue + } + } + + sort.Slice(stores, func(i, j int) bool { + return strings.Compare(stores[i].name, stores[j].name) == -1 + }) + + // make new smt store schema + if len(rootStore.schema) != 0 { + // schema is already exists + return sdkerrors.Wrapf(sdkerrors.ErrLogic, "smt store already have schema") + } + + // set the schema to smt store + schemaWriter := prefixdb.NewPrefixWriter(rootStore.stateTxn, schemaPrefix) + for _, store := range stores { + rootStore.schema[store.name] = types.StoreTypePersistent + err := schemaWriter.Set([]byte(store.name), []byte{byte(types.StoreTypePersistent)}) + if err != nil { + return sdkerrors.Wrap(err, "error at set the store schema key values") + } + } + + // iterate through all iavl stores + for _, store := range stores { + subStore, err := rootStore.getSubstore(store.name) + if err != nil { + return err + } + // iterate all iavl tree node key/values + iterator := store.Iterator(nil, nil) + for ; iterator.Valid(); iterator.Next() { + // set the iavl key,values into smt node + subStore.Set(iterator.Key(), iterator.Value()) + } + } + // commit the all key/values from iavl to smt tree (SMT Store) + rootStore.Commit() + return nil +} From df0dc4f682d8f85a4c2f22e492e8e64d6836dbe5 Mon Sep 17 00:00:00 2001 From: Sai Kumar Date: Wed, 19 Jan 2022 21:54:51 +0530 Subject: [PATCH 02/12] chore: removed sort from migration func --- store/v2/multi/migration.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/store/v2/multi/migration.go b/store/v2/multi/migration.go index 8e48882a95a..3bb0b2ef359 100644 --- a/store/v2/multi/migration.go +++ b/store/v2/multi/migration.go @@ -8,8 +8,6 @@ import ( "github.com/cosmos/cosmos-sdk/store/transient" "github.com/cosmos/cosmos-sdk/store/types" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" - "sort" - "strings" ) // MigrationFromIAVLStoreToSMTStore will migrate the complete state from iavl to smt @@ -31,10 +29,6 @@ func MigrationFromIAVLStoreToSMTStore(rs *v1Store.Store, rootStore *Store) error } } - sort.Slice(stores, func(i, j int) bool { - return strings.Compare(stores[i].name, stores[j].name) == -1 - }) - // make new smt store schema if len(rootStore.schema) != 0 { // schema is already exists From 79583ecb1c4202d38a44dd1e7bb13c39d0a1b774 Mon Sep 17 00:00:00 2001 From: Sai Kumar Date: Thu, 27 Jan 2022 10:57:04 +0530 Subject: [PATCH 03/12] chore: address the pr comments --- store/v2/multi/migration.go | 65 ++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 37 deletions(-) diff --git a/store/v2/multi/migration.go b/store/v2/multi/migration.go index 3bb0b2ef359..253104e47e9 100644 --- a/store/v2/multi/migration.go +++ b/store/v2/multi/migration.go @@ -10,18 +10,37 @@ import ( sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" ) -// MigrationFromIAVLStoreToSMTStore will migrate the complete state from iavl to smt -func MigrationFromIAVLStoreToSMTStore(rs *v1Store.Store, rootStore *Store) error { - // Collect stores to snapshot (only IAVL stores are supported) - type namedStore struct { - *iavl.Store - name string +// MigrateV2 will migrate the state from iavl to smt +func MigrateV2(rs *v1Store.Store, rootStore *Store) error { + // make new smt store schema + if len(rootStore.schema) != 0 { + // schema already exists + return sdkerrors.Wrapf(sdkerrors.ErrLogic, "smt store already have schema") } - var stores []namedStore + + // set the schema to smt store + schemaWriter := prefixdb.NewPrefixWriter(rootStore.stateTxn, schemaPrefix) + for key := range rs.GetStores() { switch store := rs.GetCommitKVStore(key).(type) { case *iavl.Store: - stores = append(stores, namedStore{name: key.Name(), Store: store}) + rootStore.schema[key.Name()] = types.StoreTypePersistent + err := schemaWriter.Set([]byte(key.Name()), []byte{byte(types.StoreTypePersistent)}) + if err != nil { + return sdkerrors.Wrap(err, "error at set the store schema key values") + } + + subStore, err := rootStore.getSubstore(key.Name()) + if err != nil { + return err + } + // iterate all iavl tree node key/values + iterator := store.Iterator(nil, nil) + for ; iterator.Valid(); iterator.Next() { + // set the iavl key,values into smt node + subStore.Set(iterator.Key(), iterator.Value()) + } + case *transient.Store, *mem.Store: continue default: @@ -29,36 +48,8 @@ func MigrationFromIAVLStoreToSMTStore(rs *v1Store.Store, rootStore *Store) error } } - // make new smt store schema - if len(rootStore.schema) != 0 { - // schema is already exists - return sdkerrors.Wrapf(sdkerrors.ErrLogic, "smt store already have schema") - } - - // set the schema to smt store - schemaWriter := prefixdb.NewPrefixWriter(rootStore.stateTxn, schemaPrefix) - for _, store := range stores { - rootStore.schema[store.name] = types.StoreTypePersistent - err := schemaWriter.Set([]byte(store.name), []byte{byte(types.StoreTypePersistent)}) - if err != nil { - return sdkerrors.Wrap(err, "error at set the store schema key values") - } - } - - // iterate through all iavl stores - for _, store := range stores { - subStore, err := rootStore.getSubstore(store.name) - if err != nil { - return err - } - // iterate all iavl tree node key/values - iterator := store.Iterator(nil, nil) - for ; iterator.Valid(); iterator.Next() { - // set the iavl key,values into smt node - subStore.Set(iterator.Key(), iterator.Value()) - } - } // commit the all key/values from iavl to smt tree (SMT Store) rootStore.Commit() + return nil } From 9f496e990acefd7e82e959c6acd5a2c0fee1fa00 Mon Sep 17 00:00:00 2001 From: Sai Kumar Date: Sat, 29 Jan 2022 14:09:49 +0530 Subject: [PATCH 04/12] test: add test for migrationv2 --- store/v2/multi/migration.go | 1 - store/v2/multi/migration_test.go | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 store/v2/multi/migration_test.go diff --git a/store/v2/multi/migration.go b/store/v2/multi/migration.go index 253104e47e9..b229fe0a81c 100644 --- a/store/v2/multi/migration.go +++ b/store/v2/multi/migration.go @@ -12,7 +12,6 @@ import ( // MigrateV2 will migrate the state from iavl to smt func MigrateV2(rs *v1Store.Store, rootStore *Store) error { - // make new smt store schema if len(rootStore.schema) != 0 { // schema already exists return sdkerrors.Wrapf(sdkerrors.ErrLogic, "smt store already have schema") diff --git a/store/v2/multi/migration_test.go b/store/v2/multi/migration_test.go new file mode 100644 index 00000000000..55f0021aa77 --- /dev/null +++ b/store/v2/multi/migration_test.go @@ -0,0 +1,38 @@ +package root + +import ( + "github.com/cosmos/cosmos-sdk/db/memdb" + "github.com/cosmos/cosmos-sdk/store/rootmulti" + "github.com/cosmos/cosmos-sdk/store/types" + "github.com/stretchr/testify/require" + dbm "github.com/tendermint/tm-db" + "testing" +) + +func TestMigrationV2(t *testing.T) { + storeName := "store1" + kvStoreKey := types.NewKVStoreKey(storeName) + + // setup a rootmulti store + db := dbm.NewMemDB() + v1Store := rootmulti.NewStore(db) + v1Store.MountStoreWithDB(kvStoreKey, types.StoreTypeIAVL, db) + err := v1Store.LoadLatestVersion() + require.Nil(t, err) + + // setup a temporary test data + v1StoreKVStore := v1Store.GetKVStore(kvStoreKey) + v1StoreKVStore.Set([]byte("temp_data"), []byte("one")) + + // setup a new root store of smt + db2 := memdb.NewDB() + v2Store, err := NewStore(db2, DefaultStoreConfig()) + require.Nil(t, err) + + // migrating the iavl store (v1) to smt store (v2) + err = MigrateV2(v1Store, v2Store) + require.NoError(t, err) + + v2StoreKVStore := v2Store.GetKVStore(kvStoreKey) + require.Equal(t, v2StoreKVStore.Get([]byte("temp_data")), []byte("one")) +} From b30528089a8adb0cd6d26861f6a254f57bbbf236 Mon Sep 17 00:00:00 2001 From: Sai Kumar Date: Mon, 31 Jan 2022 19:07:17 +0530 Subject: [PATCH 05/12] chore: change the `MigrationV2` MigrationV2 will return RootStore as output --- store/v2/multi/migration.go | 64 +++++++++++++++++--------------- store/v2/multi/migration_test.go | 7 ++-- 2 files changed, 38 insertions(+), 33 deletions(-) diff --git a/store/v2/multi/migration.go b/store/v2/multi/migration.go index b229fe0a81c..f98ee702e5d 100644 --- a/store/v2/multi/migration.go +++ b/store/v2/multi/migration.go @@ -1,7 +1,7 @@ package root import ( - prefixdb "github.com/cosmos/cosmos-sdk/db/prefix" + dbm "github.com/cosmos/cosmos-sdk/db" "github.com/cosmos/cosmos-sdk/store/iavl" "github.com/cosmos/cosmos-sdk/store/mem" v1Store "github.com/cosmos/cosmos-sdk/store/rootmulti" @@ -11,44 +11,50 @@ import ( ) // MigrateV2 will migrate the state from iavl to smt -func MigrateV2(rs *v1Store.Store, rootStore *Store) error { - if len(rootStore.schema) != 0 { - // schema already exists - return sdkerrors.Wrapf(sdkerrors.ErrLogic, "smt store already have schema") +func MigrateV2(rs *v1Store.Store, db dbm.DBConnection, storeConfig StoreConfig) (*Store, error) { + type namedStore struct { + *iavl.Store + name string } - - // set the schema to smt store - schemaWriter := prefixdb.NewPrefixWriter(rootStore.stateTxn, schemaPrefix) - + var stores []namedStore for key := range rs.GetStores() { switch store := rs.GetCommitKVStore(key).(type) { case *iavl.Store: - rootStore.schema[key.Name()] = types.StoreTypePersistent - err := schemaWriter.Set([]byte(key.Name()), []byte{byte(types.StoreTypePersistent)}) - if err != nil { - return sdkerrors.Wrap(err, "error at set the store schema key values") - } - - subStore, err := rootStore.getSubstore(key.Name()) - if err != nil { - return err - } - // iterate all iavl tree node key/values - iterator := store.Iterator(nil, nil) - for ; iterator.Valid(); iterator.Next() { - // set the iavl key,values into smt node - subStore.Set(iterator.Key(), iterator.Value()) - } - + storeConfig.prefixRegistry.StoreSchema[key.Name()] = types.StoreTypePersistent + stores = append(stores, namedStore{name: key.Name(), Store: store}) case *transient.Store, *mem.Store: + // Non-persisted stores shouldn't be snapshotted continue default: - continue + return nil, sdkerrors.Wrapf(sdkerrors.ErrLogic, "don't know how to snapshot store %q of type %T", key.Name(), store) + } + } + + // creating the new store of smt tree + rootStore, err := NewStore(db, storeConfig) + if err != nil { + return nil, err + } + + // iterate through the rootmulti stores and save the key/values into smt tree + for _, store := range stores { + subStore, err := rootStore.getSubstore(store.name) + if err != nil { + return nil, err + } + // iterate all iavl tree node key/values + iterator := store.Iterator(nil, nil) + for ; iterator.Valid(); iterator.Next() { + // set the iavl key,values into smt node + subStore.Set(iterator.Key(), iterator.Value()) } } // commit the all key/values from iavl to smt tree (SMT Store) - rootStore.Commit() + _, err = rootStore.commit(uint64(rs.LastCommitID().Version)) + if err != nil { + return nil, err + } - return nil + return rootStore, nil } diff --git a/store/v2/multi/migration_test.go b/store/v2/multi/migration_test.go index 55f0021aa77..9bca3e8419b 100644 --- a/store/v2/multi/migration_test.go +++ b/store/v2/multi/migration_test.go @@ -23,16 +23,15 @@ func TestMigrationV2(t *testing.T) { // setup a temporary test data v1StoreKVStore := v1Store.GetKVStore(kvStoreKey) v1StoreKVStore.Set([]byte("temp_data"), []byte("one")) + v1Store.Commit() // setup a new root store of smt db2 := memdb.NewDB() - v2Store, err := NewStore(db2, DefaultStoreConfig()) - require.Nil(t, err) - // migrating the iavl store (v1) to smt store (v2) - err = MigrateV2(v1Store, v2Store) + v2Store, err := MigrateV2(v1Store, db2, DefaultStoreConfig()) require.NoError(t, err) v2StoreKVStore := v2Store.GetKVStore(kvStoreKey) require.Equal(t, v2StoreKVStore.Get([]byte("temp_data")), []byte("one")) + require.Equal(t, v2Store.LastCommitID().Version,v1Store.LastCommitID().Version) } From 39d51cf3b1c88c5677662423bf459e06c0fe0c37 Mon Sep 17 00:00:00 2001 From: Sai Kumar Date: Tue, 1 Feb 2022 15:18:55 +0530 Subject: [PATCH 06/12] chore: refactored the migrationV2 --- store/v2/multi/migration.go | 23 ++++++++++------ store/v2/multi/migration_test.go | 45 +++++++++++++++++++++++++------- 2 files changed, 51 insertions(+), 17 deletions(-) diff --git a/store/v2/multi/migration.go b/store/v2/multi/migration.go index f98ee702e5d..2c24e03df4c 100644 --- a/store/v2/multi/migration.go +++ b/store/v2/multi/migration.go @@ -11,31 +11,38 @@ import ( ) // MigrateV2 will migrate the state from iavl to smt -func MigrateV2(rs *v1Store.Store, db dbm.DBConnection, storeConfig StoreConfig) (*Store, error) { +func MigrateV2(rootMultiStore *v1Store.Store, dbConnection dbm.DBConnection, storeConfig StoreConfig) (*Store, error) { type namedStore struct { *iavl.Store name string } var stores []namedStore - for key := range rs.GetStores() { - switch store := rs.GetCommitKVStore(key).(type) { + for key := range rootMultiStore.GetStores() { + switch store := rootMultiStore.GetCommitKVStore(key).(type) { case *iavl.Store: - storeConfig.prefixRegistry.StoreSchema[key.Name()] = types.StoreTypePersistent + err := storeConfig.RegisterSubstore(key.Name(), types.StoreTypePersistent) + if err != nil { + return nil, err + } stores = append(stores, namedStore{name: key.Name(), Store: store}) case *transient.Store, *mem.Store: - // Non-persisted stores shouldn't be snapshotted continue default: - return nil, sdkerrors.Wrapf(sdkerrors.ErrLogic, "don't know how to snapshot store %q of type %T", key.Name(), store) + return nil, sdkerrors.Wrapf(sdkerrors.ErrLogic, "don't know how to migrate store %q of type %T", key.Name(), store) } } // creating the new store of smt tree - rootStore, err := NewStore(db, storeConfig) + rootStore, err := NewStore(dbConnection, storeConfig) if err != nil { return nil, err } + // if version is 0 there is no state data to commit + if rootMultiStore.LastCommitID().Version == 0 { + return rootStore, nil + } + // iterate through the rootmulti stores and save the key/values into smt tree for _, store := range stores { subStore, err := rootStore.getSubstore(store.name) @@ -51,7 +58,7 @@ func MigrateV2(rs *v1Store.Store, db dbm.DBConnection, storeConfig StoreConfig) } // commit the all key/values from iavl to smt tree (SMT Store) - _, err = rootStore.commit(uint64(rs.LastCommitID().Version)) + _, err = rootStore.commit(uint64(rootMultiStore.LastCommitID().Version)) if err != nil { return nil, err } diff --git a/store/v2/multi/migration_test.go b/store/v2/multi/migration_test.go index 9bca3e8419b..add6885344f 100644 --- a/store/v2/multi/migration_test.go +++ b/store/v2/multi/migration_test.go @@ -23,15 +23,42 @@ func TestMigrationV2(t *testing.T) { // setup a temporary test data v1StoreKVStore := v1Store.GetKVStore(kvStoreKey) v1StoreKVStore.Set([]byte("temp_data"), []byte("one")) - v1Store.Commit() - // setup a new root store of smt - db2 := memdb.NewDB() - // migrating the iavl store (v1) to smt store (v2) - v2Store, err := MigrateV2(v1Store, db2, DefaultStoreConfig()) - require.NoError(t, err) + testCases := []struct{ + testName string + emptyStore bool + }{ + { + "Migration With Empty Store", + true, + }, + { + "Migration From Root Multi Store (IAVL) to SMT ", + false, + }, + } - v2StoreKVStore := v2Store.GetKVStore(kvStoreKey) - require.Equal(t, v2StoreKVStore.Get([]byte("temp_data")), []byte("one")) - require.Equal(t, v2Store.LastCommitID().Version,v1Store.LastCommitID().Version) + for _, testCase := range testCases{ + if !testCase.emptyStore { + v1Store.Commit() + } + + // setup a new root store of smt + db2 := memdb.NewDB() + storeConfig := DefaultStoreConfig() + // migrating the iavl store (v1) to smt store (v2) + v2Store, err := MigrateV2(v1Store, db2, storeConfig) + require.NoError(t, err) + + v2StoreKVStore := v2Store.GetKVStore(kvStoreKey) + + if testCase.emptyStore { + // check the empty store + require.Nil(t, v2StoreKVStore.Get([]byte("temp_data"))) + require.Equal(t, v2Store.LastCommitID().Version, v1Store.LastCommitID().Version) + }else{ + require.Equal(t, v2StoreKVStore.Get([]byte("temp_data")), []byte("one")) + require.Equal(t, v2Store.LastCommitID().Version, v1Store.LastCommitID().Version) + } + } } From 38527f7d98c44366e3d6e88f363abcb1085e3541 Mon Sep 17 00:00:00 2001 From: Sai Kumar Date: Tue, 15 Feb 2022 11:02:22 +0530 Subject: [PATCH 07/12] chore: address the pr review comments Added more test data and more stores --- store/v2/multi/migration.go | 4 +- store/v2/multi/migration_test.go | 63 +++++++++++++++++++++++--------- 2 files changed, 47 insertions(+), 20 deletions(-) diff --git a/store/v2/multi/migration.go b/store/v2/multi/migration.go index 2c24e03df4c..f91084269eb 100644 --- a/store/v2/multi/migration.go +++ b/store/v2/multi/migration.go @@ -11,7 +11,7 @@ import ( ) // MigrateV2 will migrate the state from iavl to smt -func MigrateV2(rootMultiStore *v1Store.Store, dbConnection dbm.DBConnection, storeConfig StoreConfig) (*Store, error) { +func MigrateV2(rootMultiStore *v1Store.Store, store2db dbm.DBConnection, storeConfig StoreConfig) (*Store, error) { type namedStore struct { *iavl.Store name string @@ -33,7 +33,7 @@ func MigrateV2(rootMultiStore *v1Store.Store, dbConnection dbm.DBConnection, sto } // creating the new store of smt tree - rootStore, err := NewStore(dbConnection, storeConfig) + rootStore, err := NewStore(store2db, storeConfig) if err != nil { return nil, err } diff --git a/store/v2/multi/migration_test.go b/store/v2/multi/migration_test.go index add6885344f..d3e38855a66 100644 --- a/store/v2/multi/migration_test.go +++ b/store/v2/multi/migration_test.go @@ -1,31 +1,55 @@ package root import ( + "encoding/binary" + "fmt" "github.com/cosmos/cosmos-sdk/db/memdb" + "github.com/cosmos/cosmos-sdk/store/iavl" "github.com/cosmos/cosmos-sdk/store/rootmulti" "github.com/cosmos/cosmos-sdk/store/types" "github.com/stretchr/testify/require" dbm "github.com/tendermint/tm-db" + "math/rand" "testing" ) func TestMigrationV2(t *testing.T) { - storeName := "store1" - kvStoreKey := types.NewKVStoreKey(storeName) + r := rand.New(rand.NewSource(49872768940)) // setup a rootmulti store db := dbm.NewMemDB() v1Store := rootmulti.NewStore(db) - v1Store.MountStoreWithDB(kvStoreKey, types.StoreTypeIAVL, db) + + // mount the kvStores + var keys []*types.KVStoreKey + for i := uint8(0); i < 10; i++ { + key := types.NewKVStoreKey(fmt.Sprintf("store%v", i)) + v1Store.MountStoreWithDB(key, types.StoreTypeIAVL, nil) + keys = append(keys, key) + } + err := v1Store.LoadLatestVersion() require.Nil(t, err) - // setup a temporary test data - v1StoreKVStore := v1Store.GetKVStore(kvStoreKey) - v1StoreKVStore.Set([]byte("temp_data"), []byte("one")) + // setup a random test data + for _, key := range keys { + store := v1Store.GetStore(key).(*iavl.Store) + store.Set([]byte("temp_data"), []byte("one")) + + for i := 0; i < len(keys); i++ { + k := make([]byte, 8) + v := make([]byte, 1024) + binary.BigEndian.PutUint64(k, uint64(i)) + _, err := r.Read(v) + if err != nil { + panic(err) + } + store.Set(k, v) + } + } - testCases := []struct{ - testName string + testCases := []struct { + testName string emptyStore bool }{ { @@ -38,7 +62,7 @@ func TestMigrationV2(t *testing.T) { }, } - for _, testCase := range testCases{ + for _, testCase := range testCases { if !testCase.emptyStore { v1Store.Commit() } @@ -50,15 +74,18 @@ func TestMigrationV2(t *testing.T) { v2Store, err := MigrateV2(v1Store, db2, storeConfig) require.NoError(t, err) - v2StoreKVStore := v2Store.GetKVStore(kvStoreKey) - - if testCase.emptyStore { - // check the empty store - require.Nil(t, v2StoreKVStore.Get([]byte("temp_data"))) - require.Equal(t, v2Store.LastCommitID().Version, v1Store.LastCommitID().Version) - }else{ - require.Equal(t, v2StoreKVStore.Get([]byte("temp_data")), []byte("one")) - require.Equal(t, v2Store.LastCommitID().Version, v1Store.LastCommitID().Version) + for _, key := range keys { + v2StoreKVStore := v2Store.GetKVStore(key) + if testCase.emptyStore { + // check the empty store + require.Nil(t, v2StoreKVStore.Get([]byte("temp_data"))) + require.Equal(t, v2Store.LastCommitID().Version, v1Store.LastCommitID().Version) + } else { + require.Equal(t, v2StoreKVStore.Get([]byte("temp_data")), []byte("one")) + require.Equal(t, v2Store.LastCommitID().Version, v1Store.LastCommitID().Version) + } } + err = v2Store.Close() + require.NoError(t, err) } } From 469536599c2fb60a1abefeabd58f45e4a0d85451 Mon Sep 17 00:00:00 2001 From: Sai Kumar Date: Fri, 25 Feb 2022 17:29:23 +0530 Subject: [PATCH 08/12] fix: fix the build issue --- store/rootmulti/store.go | 5 ----- store/v2/multi/migration.go | 2 +- store/v2/multi/migration_test.go | 2 +- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index 683289e40bf..5858f86636b 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -87,11 +87,6 @@ func (rs *Store) GetPruning() types.PruningOptions { return rs.pruningOpts } -// GetStores will return the stores -func (rs *Store) GetStores() map[types.StoreKey]types.CommitKVStore { - return rs.stores -} - // SetPruning sets the pruning strategy on the root store and all the sub-stores. // Note, calling SetPruning on the root store prior to LoadVersion or // LoadLatestVersion performs a no-op as the stores aren't mounted yet. diff --git a/store/v2/multi/migration.go b/store/v2/multi/migration.go index f91084269eb..8cdd73c1e7d 100644 --- a/store/v2/multi/migration.go +++ b/store/v2/multi/migration.go @@ -1,4 +1,4 @@ -package root +package multi import ( dbm "github.com/cosmos/cosmos-sdk/db" diff --git a/store/v2/multi/migration_test.go b/store/v2/multi/migration_test.go index d3e38855a66..765bba397f0 100644 --- a/store/v2/multi/migration_test.go +++ b/store/v2/multi/migration_test.go @@ -1,4 +1,4 @@ -package root +package multi import ( "encoding/binary" From 677639e6bf347df2c6d40242347ef2c06df9c94f Mon Sep 17 00:00:00 2001 From: Sai Kumar Date: Fri, 4 Mar 2022 12:04:08 +0530 Subject: [PATCH 09/12] chore: address the pr comments --- store/rootmulti/snapshot_test.go | 7 ++++--- store/rootmulti/store.go | 6 +++--- store/v2/multi/migration.go | 11 ++++++----- store/v2/multi/migration_test.go | 17 +++++++++++++++-- 4 files changed, 28 insertions(+), 13 deletions(-) diff --git a/store/rootmulti/snapshot_test.go b/store/rootmulti/snapshot_test.go index cbdb481a05f..0a1c0a695ea 100644 --- a/store/rootmulti/snapshot_test.go +++ b/store/rootmulti/snapshot_test.go @@ -210,7 +210,8 @@ func TestMultistoreSnapshotRestore(t *testing.T) { require.Equal(t, *dummyExtensionItem.GetExtension(), *nextItem.GetExtension()) assert.Equal(t, source.LastCommitID(), target.LastCommitID()) - for key, sourceStore := range source.GetStores() { + for _, key := range source.GetStoreNames() { + sourceStore := source.GetStoreByName(key.Name()).(types.CommitKVStore) targetStore := target.GetStoreByName(key.Name()).(types.CommitKVStore) switch sourceStore.GetStoreType() { case types.StoreTypeTransient: @@ -234,7 +235,7 @@ func benchmarkMultistoreSnapshot(b *testing.B, stores uint8, storeKeys uint64) { for i := 0; i < b.N; i++ { target := rootmulti.NewStore(dbm.NewMemDB()) - for key := range source.GetStores() { + for _, key := range source.GetStoreNames() { target.MountStoreWithDB(key, types.StoreTypeIAVL, nil) } err := target.LoadLatestVersion() @@ -269,7 +270,7 @@ func benchmarkMultistoreSnapshotRestore(b *testing.B, stores uint8, storeKeys ui for i := 0; i < b.N; i++ { target := rootmulti.NewStore(dbm.NewMemDB()) - for key := range source.GetStores() { + for _, key := range source.GetStoreNames() { target.MountStoreWithDB(key, types.StoreTypeIAVL, nil) } err := target.LoadLatestVersion() diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index b264352ff21..46573ba6265 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -148,9 +148,9 @@ func (rs *Store) GetCommitKVStore(key types.StoreKey) types.CommitKVStore { return rs.stores[key] } -// GetStores returns mounted stores -func (rs *Store) GetStores() map[types.StoreKey]types.CommitKVStore { - return rs.stores +// GetStoreNames returns the storeNames and StoreKeys +func (rs *Store) GetStoreNames() map[string]types.StoreKey { + return rs.keysByName } // LoadLatestVersionAndUpgrade implements CommitMultiStore diff --git a/store/v2/multi/migration.go b/store/v2/multi/migration.go index 8cdd73c1e7d..438c46ae5b6 100644 --- a/store/v2/multi/migration.go +++ b/store/v2/multi/migration.go @@ -17,18 +17,19 @@ func MigrateV2(rootMultiStore *v1Store.Store, store2db dbm.DBConnection, storeCo name string } var stores []namedStore - for key := range rootMultiStore.GetStores() { - switch store := rootMultiStore.GetCommitKVStore(key).(type) { + for _, storeKey := range rootMultiStore.GetStoreNames() { + keyName := storeKey.Name() + switch store := rootMultiStore.GetStoreByName(keyName).(type) { case *iavl.Store: - err := storeConfig.RegisterSubstore(key.Name(), types.StoreTypePersistent) + err := storeConfig.RegisterSubstore(keyName, types.StoreTypePersistent) if err != nil { return nil, err } - stores = append(stores, namedStore{name: key.Name(), Store: store}) + stores = append(stores, namedStore{name: keyName, Store: store}) case *transient.Store, *mem.Store: continue default: - return nil, sdkerrors.Wrapf(sdkerrors.ErrLogic, "don't know how to migrate store %q of type %T", key.Name(), store) + return nil, sdkerrors.Wrapf(sdkerrors.ErrLogic, "don't know how to migrate store %q of type %T", keyName, store) } } diff --git a/store/v2/multi/migration_test.go b/store/v2/multi/migration_test.go index 765bba397f0..d90f2ea81da 100644 --- a/store/v2/multi/migration_test.go +++ b/store/v2/multi/migration_test.go @@ -79,13 +79,26 @@ func TestMigrationV2(t *testing.T) { if testCase.emptyStore { // check the empty store require.Nil(t, v2StoreKVStore.Get([]byte("temp_data"))) - require.Equal(t, v2Store.LastCommitID().Version, v1Store.LastCommitID().Version) } else { require.Equal(t, v2StoreKVStore.Get([]byte("temp_data")), []byte("one")) - require.Equal(t, v2Store.LastCommitID().Version, v1Store.LastCommitID().Version) } + require.Equal(t, v2Store.LastCommitID().Version, v1Store.LastCommitID().Version) } err = v2Store.Close() require.NoError(t, err) } } + +func TestMigrateV2ForEmptyStore(t *testing.T) { + // setup a rootmulti store + db := dbm.NewMemDB() + v1Store := rootmulti.NewStore(db) + err := v1Store.LoadLatestVersion() + require.Nil(t, err) + db2 := memdb.NewDB() + storeConfig := DefaultStoreConfig() + // migrating the iavl store (v1) to smt store (v2) + v2Store, err := MigrateV2(v1Store, db2, storeConfig) + require.NoError(t, err) + require.Equal(t, v2Store.LastCommitID(), v1Store.LastCommitID()) +} From a97ed5a2584167bc24b91478488b607fd8bdbf6b Mon Sep 17 00:00:00 2001 From: Sai Kumar Date: Fri, 11 Mar 2022 19:33:20 +0530 Subject: [PATCH 10/12] address the pr comments --- store/rootmulti/store.go | 4 ++-- store/v2/multi/migration.go | 6 +++--- store/v2/multi/migration_test.go | 10 ++++++---- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/store/rootmulti/store.go b/store/rootmulti/store.go index 46573ba6265..bd9502187cd 100644 --- a/store/rootmulti/store.go +++ b/store/rootmulti/store.go @@ -148,8 +148,8 @@ func (rs *Store) GetCommitKVStore(key types.StoreKey) types.CommitKVStore { return rs.stores[key] } -// GetStoreNames returns the storeNames and StoreKeys -func (rs *Store) GetStoreNames() map[string]types.StoreKey { +// StoreKeysByName returns mapping storeNames -> StoreKeys +func (rs *Store) StoreKeysByName() map[string]types.StoreKey { return rs.keysByName } diff --git a/store/v2/multi/migration.go b/store/v2/multi/migration.go index 438c46ae5b6..7481b421ac3 100644 --- a/store/v2/multi/migration.go +++ b/store/v2/multi/migration.go @@ -10,14 +10,14 @@ import ( sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" ) -// MigrateV2 will migrate the state from iavl to smt -func MigrateV2(rootMultiStore *v1Store.Store, store2db dbm.DBConnection, storeConfig StoreConfig) (*Store, error) { +// MigrateFromV1 will migrate the state from iavl to smt +func MigrateFromV1(rootMultiStore *v1Store.Store, store2db dbm.DBConnection, storeConfig StoreConfig) (*Store, error) { type namedStore struct { *iavl.Store name string } var stores []namedStore - for _, storeKey := range rootMultiStore.GetStoreNames() { + for _, storeKey := range rootMultiStore.StoreKeysByName() { keyName := storeKey.Name() switch store := rootMultiStore.GetStoreByName(keyName).(type) { case *iavl.Store: diff --git a/store/v2/multi/migration_test.go b/store/v2/multi/migration_test.go index d90f2ea81da..09f1f74b5a1 100644 --- a/store/v2/multi/migration_test.go +++ b/store/v2/multi/migration_test.go @@ -3,14 +3,15 @@ package multi import ( "encoding/binary" "fmt" + "math/rand" + "testing" + "github.com/cosmos/cosmos-sdk/db/memdb" "github.com/cosmos/cosmos-sdk/store/iavl" "github.com/cosmos/cosmos-sdk/store/rootmulti" "github.com/cosmos/cosmos-sdk/store/types" "github.com/stretchr/testify/require" dbm "github.com/tendermint/tm-db" - "math/rand" - "testing" ) func TestMigrationV2(t *testing.T) { @@ -71,7 +72,7 @@ func TestMigrationV2(t *testing.T) { db2 := memdb.NewDB() storeConfig := DefaultStoreConfig() // migrating the iavl store (v1) to smt store (v2) - v2Store, err := MigrateV2(v1Store, db2, storeConfig) + v2Store, err := MigrateFromV1(v1Store, db2, storeConfig) require.NoError(t, err) for _, key := range keys { @@ -89,6 +90,7 @@ func TestMigrationV2(t *testing.T) { } } +// TestMigrateV2ForEmptyStore checking empty store migration func TestMigrateV2ForEmptyStore(t *testing.T) { // setup a rootmulti store db := dbm.NewMemDB() @@ -98,7 +100,7 @@ func TestMigrateV2ForEmptyStore(t *testing.T) { db2 := memdb.NewDB() storeConfig := DefaultStoreConfig() // migrating the iavl store (v1) to smt store (v2) - v2Store, err := MigrateV2(v1Store, db2, storeConfig) + v2Store, err := MigrateFromV1(v1Store, db2, storeConfig) require.NoError(t, err) require.Equal(t, v2Store.LastCommitID(), v1Store.LastCommitID()) } From bb61b5dd9a4c16e05d9d643724fb7f8e75e1cd77 Mon Sep 17 00:00:00 2001 From: Sai Kumar Date: Tue, 15 Mar 2022 17:48:21 +0530 Subject: [PATCH 11/12] address the pr comments --- store/rootmulti/snapshot_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/store/rootmulti/snapshot_test.go b/store/rootmulti/snapshot_test.go index 0a1c0a695ea..92e1311fdd8 100644 --- a/store/rootmulti/snapshot_test.go +++ b/store/rootmulti/snapshot_test.go @@ -210,7 +210,7 @@ func TestMultistoreSnapshotRestore(t *testing.T) { require.Equal(t, *dummyExtensionItem.GetExtension(), *nextItem.GetExtension()) assert.Equal(t, source.LastCommitID(), target.LastCommitID()) - for _, key := range source.GetStoreNames() { + for _, key := range source.StoreKeysByName() { sourceStore := source.GetStoreByName(key.Name()).(types.CommitKVStore) targetStore := target.GetStoreByName(key.Name()).(types.CommitKVStore) switch sourceStore.GetStoreType() { @@ -235,7 +235,7 @@ func benchmarkMultistoreSnapshot(b *testing.B, stores uint8, storeKeys uint64) { for i := 0; i < b.N; i++ { target := rootmulti.NewStore(dbm.NewMemDB()) - for _, key := range source.GetStoreNames() { + for _, key := range source.StoreKeysByName() { target.MountStoreWithDB(key, types.StoreTypeIAVL, nil) } err := target.LoadLatestVersion() @@ -270,7 +270,7 @@ func benchmarkMultistoreSnapshotRestore(b *testing.B, stores uint8, storeKeys ui for i := 0; i < b.N; i++ { target := rootmulti.NewStore(dbm.NewMemDB()) - for _, key := range source.GetStoreNames() { + for _, key := range source.StoreKeysByName() { target.MountStoreWithDB(key, types.StoreTypeIAVL, nil) } err := target.LoadLatestVersion() From 1af9b3b374de403391e1ca74ed287dd7e8d83267 Mon Sep 17 00:00:00 2001 From: Sai Kumar Date: Thu, 17 Mar 2022 15:43:30 +0530 Subject: [PATCH 12/12] chore: update changelog --- CHANGELOG.md | 1 + store/{v2 => v2alpha1}/multi/migration.go | 0 store/{v2 => v2alpha1}/multi/migration_test.go | 0 3 files changed, 1 insertion(+) rename store/{v2 => v2alpha1}/multi/migration.go (100%) rename store/{v2 => v2alpha1}/multi/migration_test.go (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 40bcadd7642..9d7d45778ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * [#11179](https://github.com/cosmos/cosmos-sdk/pull/11179) Add state rollback command. * [\#10794](https://github.com/cosmos/cosmos-sdk/pull/10794) ADR-040: Add State Sync to V2 Store * [\#11234](https://github.com/cosmos/cosmos-sdk/pull/11234) Add `GRPCClient` field to Client Context. If `GRPCClient` field is set to nil, the `Invoke` method would use ABCI query, otherwise use gprc. +* [\#10962](https://github.com/cosmos/cosmos-sdk/pull/10962) ADR-040: Add state migration from iavl (v1Store) to smt (v2Store) ### API Breaking Changes diff --git a/store/v2/multi/migration.go b/store/v2alpha1/multi/migration.go similarity index 100% rename from store/v2/multi/migration.go rename to store/v2alpha1/multi/migration.go diff --git a/store/v2/multi/migration_test.go b/store/v2alpha1/multi/migration_test.go similarity index 100% rename from store/v2/multi/migration_test.go rename to store/v2alpha1/multi/migration_test.go