feat: Add API TraverseStateChanges to extract state changes from ia…

…vl versions (backport: #654)  (#796)

Co-authored-by: yihuang <[email protected]>
Co-authored-by: Marko <[email protected]>

.golangci.yml

@@ -7,7 +7,6 @@ linters:
   disable-all: true
   enable:
     - bodyclose
-    - depguard
     - dogsled
     - errcheck
     - exportloopref

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Improvements
 
+- [#654](https://github.com/cosmos/iavl/pull/654) Add API `TraverseStateChanges` to extract state changes from iavl versions.
 - [#726](https://github.com/cosmos/iavl/pull/726) Make `KVPair` and `ChangeSet` serializable with protobuf.
 
 ## 0.20.0 (March 14, 2023)

basic_test.go

@@ -341,7 +341,7 @@ func TestIntegration(t *testing.T) {
 	}
 
 	for i, x := range records {
-		if val, removed, err := tree.Remove([]byte(x.key)); err != nil { //nolint:gocritic
+		if val, removed, err := tree.Remove([]byte(x.key)); err != nil {
 			require.NoError(t, err)
 		} else if !removed {
 			t.Error("Wasn't removed")

diff_test.go

@@ -21,16 +21,25 @@ func TestDiffRoundTrip(t *testing.T) {
 	db := db.NewMemDB()
 	tree, err := NewMutableTree(db, 0, true)
 	require.NoError(t, err)
-	for i := range changeSets {
-		v, err := tree.SaveChangeSet(changeSets[i])
+	for _, cs := range changeSets {
+		for _, pair := range cs.Pairs {
+			if pair.Delete {
+				_, removed, err := tree.Remove(pair.Key)
+				require.True(t, removed)
+				require.NoError(t, err)
+			} else {
+				_, err := tree.Set(pair.Key, pair.Value)
+				require.NoError(t, err)
+			}
+		}
+		_, _, err := tree.SaveVersion()
 		require.NoError(t, err)
-		require.Equal(t, int64(i+1), v)
 	}
 
 	// extract change sets from db
 	var extractChangeSets []*ChangeSet
 	tree2 := NewImmutableTree(db, 0, true)
-	err = tree2.ndb.traverseStateChanges(0, math.MaxInt64, func(version int64, changeSet *ChangeSet) error {
+	err = tree2.TraverseStateChanges(0, math.MaxInt64, func(version int64, changeSet *ChangeSet) error {
 		extractChangeSets = append(extractChangeSets, changeSet)
 		return nil
 	})
@@ -56,9 +65,9 @@ func genChangeSets(r *rand.Rand, n int) []*ChangeSet {
 		}
 		if len(changeSets) > 0 {
 			// pick some random keys to delete from the last version
-			lastChangeSet := changeSets[len(changeSets)-1]
+			pairs := changeSets[len(changeSets)-1].Pairs
 			count = r.Int63n(10)
-			for _, pair := range lastChangeSet.Pairs {
+			for _, pair := range pairs {
 				if count <= 0 {
 					break
 				}
@@ -73,9 +82,9 @@ func genChangeSets(r *rand.Rand, n int) []*ChangeSet {
 			}
 
 			// Special case, set to identical value
-			if len(lastChangeSet.Pairs) > 0 {
-				i := r.Int63n(int64(len(lastChangeSet.Pairs)))
-				pair := lastChangeSet.Pairs[i]
+			if len(pairs) > 0 {
+				i := r.Int63n(int64(len(pairs)))
+				pair := pairs[i]
 				if !pair.Delete {
 					items[string(pair.Key)] = &KVPair{
 						Key:   pair.Key,
@@ -93,7 +102,8 @@ func genChangeSets(r *rand.Rand, n int) []*ChangeSet {
 
 		var cs ChangeSet
 		for _, key := range keys {
-			cs.Pairs = append(cs.Pairs, items[key])
+			p := items[key]
+			cs.Pairs = append(cs.Pairs, p)
 		}
 
 		changeSets = append(changeSets, &cs)

immutable_tree.go

@@ -67,7 +67,7 @@ func (t *ImmutableTree) RenderShape(indent string, encoder NodeEncoder) ([]strin
 type NodeEncoder func(id []byte, depth int, isLeaf bool) string
 
 // defaultNodeEncoder can encode any node unless the client overrides it
-func defaultNodeEncoder(id []byte, depth int, isLeaf bool) string {
+func defaultNodeEncoder(id []byte, _ int, isLeaf bool) string {
 	prefix := "- "
 	if isLeaf {
 		prefix = "* "
@@ -331,3 +331,9 @@ func (t *ImmutableTree) nodeSize() int {
 	})
 	return size
 }
+
+// TraverseStateChanges iterate the range of versions, compare each version to it's predecessor to extract the state changes of it.
+// endVersion is exclusive.
+func (t *ImmutableTree) TraverseStateChanges(startVersion, endVersion int64, fn func(version int64, changeSet *ChangeSet) error) error {
+	return t.ndb.traverseStateChanges(startVersion, endVersion, fn)
+}

nodedb.go

@@ -1118,49 +1118,34 @@ var (
 // traverseStateChanges iterate the range of versions, compare each version to it's predecessor to extract the state changes of it.
 // endVersion is exclusive, set to `math.MaxInt64` to cover the latest version.
 func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func(version int64, changeSet *ChangeSet) error) error {
-	firstVersion, err := ndb.getFirstVersion()
+	predecessor, err := ndb.getPreviousVersion(startVersion)
 	if err != nil {
 		return err
 	}
-	if startVersion < firstVersion {
-		startVersion = firstVersion
-	}
-	latestVersion, err := ndb.getLatestVersion()
+	prevRoot, err := ndb.getRoot(predecessor)
 	if err != nil {
 		return err
 	}
-	if endVersion > latestVersion {
-		endVersion = latestVersion
-	}
-
-	prevVersion := startVersion - 1
-	prevRoot, err := ndb.getRoot(prevVersion)
-	if err != nil && err != ErrVersionDoesNotExist {
-		return err
-	}
-
-	for version := startVersion; version <= endVersion; version++ {
-		root, err := ndb.getRoot(version)
-		if err != nil {
-			return err
-		}
+	return ndb.traverseRange(rootKeyFormat.Key(startVersion), rootKeyFormat.Key(endVersion), func(k, hash []byte) error {
+		var version int64
+		rootKeyFormat.Scan(k, &version)
 
 		var changeSet ChangeSet
 		receiveKVPair := func(pair *KVPair) error {
 			changeSet.Pairs = append(changeSet.Pairs, pair)
 			return nil
 		}
 
-		if err := ndb.extractStateChanges(prevVersion, prevRoot, root, receiveKVPair); err != nil {
+		if err := ndb.extractStateChanges(predecessor, prevRoot, hash, receiveKVPair); err != nil {
 			return err
 		}
 
 		if err := fn(version, &changeSet); err != nil {
 			return err
 		}
-		prevVersion = version
-		prevRoot = root
-	}
 
-	return nil
+		predecessor = version
+		prevRoot = hash
+		return nil
+	})
 }

tree_random_test.go

@@ -377,7 +377,7 @@ func assertOrphans(t *testing.T, tree *MutableTree, expected int) {
 }
 
 // Checks that a version is the maximum mirrored version.
-func assertMaxVersion(t *testing.T, tree *MutableTree, version int64, mirrors map[int64]map[string]string) { //nolint:unparam
+func assertMaxVersion(t *testing.T, _ *MutableTree, version int64, mirrors map[int64]map[string]string) {
 	max := int64(0)
 	for v := range mirrors {
 		if v > max {