all: implement path-based state scheme

cmd/geth/snapshot.go

@@ -165,6 +165,8 @@ block is used.
 	}
 )
 
+// Deprecation: this command should be deprecated once the hash-based
+// scheme is deprecated.
 func pruneState(ctx *cli.Context) error {
 	stack, config := makeConfigNode(ctx)
 	defer stack.Close()
@@ -399,7 +401,7 @@ func traverseRawState(ctx *cli.Context) error {
 		// Check the present for non-empty hash node(embedded node doesn't
 		// have their own hash).
 		if node != (common.Hash{}) {
-			blob := rawdb.ReadTrieNode(chaindb, node)
+			blob := rawdb.ReadLegacyTrieNode(chaindb, node)
 			if len(blob) == 0 {
 				log.Error("Missing trie node(account)", "hash", node)
 				return errors.New("missing account")
@@ -433,10 +435,10 @@ func traverseRawState(ctx *cli.Context) error {
 					nodes += 1
 					node := storageIter.Hash()
 
-					// Check the present for non-empty hash node(embedded node doesn't
+					// Check the presence for non-empty hash node(embedded node doesn't
 					// have their own hash).
 					if node != (common.Hash{}) {
-						blob := rawdb.ReadTrieNode(chaindb, node)
+						blob := rawdb.ReadLegacyTrieNode(chaindb, node)
 						if len(blob) == 0 {
 							log.Error("Missing trie node(storage)", "hash", node)
 							return errors.New("missing storage")

core/blockchain.go

@@ -910,14 +910,14 @@ func (bc *BlockChain) Stop() {
 				recent := bc.GetBlockByNumber(number - offset)
 
 				log.Info("Writing cached state to disk", "block", recent.Number(), "hash", recent.Hash(), "root", recent.Root())
-				if err := triedb.Commit(recent.Root(), true, nil); err != nil {
+				if err := triedb.Commit(recent.Root(), true); err != nil {
 					log.Error("Failed to commit recent state trie", "err", err)
 				}
 			}
 		}
 		if snapBase != (common.Hash{}) {
 			log.Info("Writing snapshot state to disk", "root", snapBase)
-			if err := triedb.Commit(snapBase, true, nil); err != nil {
+			if err := triedb.Commit(snapBase, true); err != nil {
 				log.Error("Failed to commit recent state trie", "err", err)
 			}
 		}
@@ -929,8 +929,8 @@ func (bc *BlockChain) Stop() {
 		}
 	}
 	// Flush the collected preimages to disk
-	if err := bc.stateCache.TrieDB().CommitPreimages(); err != nil {
-		log.Error("Failed to commit trie preimages", "err", err)
+	if err := bc.stateCache.TrieDB().Close(); err != nil {
+		log.Error("Failed to close trie db", "err", err)
 	}
 	// Ensure all live cached entries be saved into disk, so that we can skip
 	// cache warmup when node restarts.
@@ -1310,7 +1310,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 	}
 	// If we're running an archive node, always flush
 	if bc.cacheConfig.TrieDirtyDisabled {
-		return bc.triedb.Commit(root, false, nil)
+		return bc.triedb.Commit(root, false)
 	} else {
 		// Full but not archive node, do proper garbage collection
 		bc.triedb.Reference(root, common.Hash{}) // metadata reference to keep trie alive
@@ -1342,7 +1342,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
 						log.Info("State in memory for too long, committing", "time", bc.gcproc, "allowance", bc.cacheConfig.TrieTimeLimit, "optimum", float64(chosen-lastWrite)/TriesInMemory)
 					}
 					// Flush an entire trie and restart the counters
-					bc.triedb.Commit(header.Root, true, nil)
+					bc.triedb.Commit(header.Root, true)
 					lastWrite = chosen
 					bc.gcproc = 0
 				}

core/blockchain_repair_test.go

@@ -1803,7 +1803,7 @@ func testRepair(t *testing.T, tt *rewindTest, snapshots bool) {
 		t.Fatalf("Failed to import canonical chain start: %v", err)
 	}
 	if tt.commitBlock > 0 {
-		chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), true, nil)
+		chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), false)
 		if snapshots {
 			if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
 				t.Fatalf("Failed to flatten snapshots: %v", err)
@@ -1918,7 +1918,7 @@ func TestIssue23496(t *testing.T) {
 	if _, err := chain.InsertChain(blocks[:1]); err != nil {
 		t.Fatalf("Failed to import canonical chain start: %v", err)
 	}
-	chain.stateCache.TrieDB().Commit(blocks[0].Root(), true, nil)
+	chain.stateCache.TrieDB().Commit(blocks[0].Root(), false)
 
 	// Insert block B2 and commit the snapshot into disk
 	if _, err := chain.InsertChain(blocks[1:2]); err != nil {
@@ -1932,7 +1932,7 @@ func TestIssue23496(t *testing.T) {
 	if _, err := chain.InsertChain(blocks[2:3]); err != nil {
 		t.Fatalf("Failed to import canonical chain start: %v", err)
 	}
-	chain.stateCache.TrieDB().Commit(blocks[2].Root(), true, nil)
+	chain.stateCache.TrieDB().Commit(blocks[2].Root(), false)
 
 	// Insert the remaining blocks
 	if _, err := chain.InsertChain(blocks[3:]); err != nil {

core/blockchain_sethead_test.go

@@ -2004,7 +2004,7 @@ func testSetHead(t *testing.T, tt *rewindTest, snapshots bool) {
 		t.Fatalf("Failed to import canonical chain start: %v", err)
 	}
 	if tt.commitBlock > 0 {
-		chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), true, nil)
+		chain.stateCache.TrieDB().Commit(canonblocks[tt.commitBlock-1].Root(), false)
 		if snapshots {
 			if err := chain.snaps.Cap(canonblocks[tt.commitBlock-1].Root(), 0); err != nil {
 				t.Fatalf("Failed to flatten snapshots: %v", err)

core/blockchain_snapshot_test.go

@@ -99,7 +99,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
 		startPoint = point
 
 		if basic.commitBlock > 0 && basic.commitBlock == point {
-			chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), true, nil)
+			chain.stateCache.TrieDB().Commit(blocks[point-1].Root(), false)
 		}
 		if basic.snapshotBlock > 0 && basic.snapshotBlock == point {
 			// Flushing the entire snap tree into the disk, the

core/blockchain_test.go

@@ -1701,7 +1701,7 @@ func TestTrieForkGC(t *testing.T) {
 		chain.stateCache.TrieDB().Dereference(blocks[len(blocks)-1-i].Root())
 		chain.stateCache.TrieDB().Dereference(forks[len(blocks)-1-i].Root())
 	}
-	if len(chain.stateCache.TrieDB().Nodes()) > 0 {
+	if nodes, _ := chain.TrieDB().Size(); nodes > 0 {
 		t.Fatalf("stale tries still alive after garbase collection")
 	}
 }

core/chain_makers.go

@@ -284,7 +284,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse
 			if err != nil {
 				panic(fmt.Sprintf("state write error: %v", err))
 			}
-			if err := statedb.Database().TrieDB().Commit(root, false, nil); err != nil {
+			if err := statedb.Database().TrieDB().Commit(root, false); err != nil {
 				panic(fmt.Sprintf("trie write error: %v", err))
 			}
 			return block, b.receipts

core/dao_test.go

@@ -83,7 +83,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		if _, err := bc.InsertChain(blocks); err != nil {
 			t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
 		}
-		if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil {
+		if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
 			t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
 		}
 		bc.Stop()
@@ -106,7 +106,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 		if _, err := bc.InsertChain(blocks); err != nil {
 			t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
 		}
-		if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil {
+		if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
 			t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
 		}
 		bc.Stop()
@@ -131,7 +131,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	if _, err := bc.InsertChain(blocks); err != nil {
 		t.Fatalf("failed to import contra-fork chain for expansion: %v", err)
 	}
-	if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil {
+	if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
 		t.Fatalf("failed to commit contra-fork head for expansion: %v", err)
 	}
 	blocks, _ = GenerateChain(&proConf, conBc.CurrentBlock(), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {})
@@ -149,7 +149,7 @@ func TestDAOForkRangeExtradata(t *testing.T) {
 	if _, err := bc.InsertChain(blocks); err != nil {
 		t.Fatalf("failed to import pro-fork chain for expansion: %v", err)
 	}
-	if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, true, nil); err != nil {
+	if err := bc.stateCache.TrieDB().Commit(bc.CurrentHeader().Root, false); err != nil {
 		t.Fatalf("failed to commit pro-fork head for expansion: %v", err)
 	}
 	blocks, _ = GenerateChain(&conConf, proBc.CurrentBlock(), ethash.NewFaker(), genDb, 1, func(i int, gen *BlockGen) {})

core/genesis.go

@@ -157,7 +157,7 @@ func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database) error {
 	}
 	// Commit newly generated states into disk if it's not empty.
 	if root != types.EmptyRootHash {
-		if err := triedb.Commit(root, true, nil); err != nil {
+		if err := triedb.Commit(root, true); err != nil {
 			return err
 		}
 	}

core/rawdb/accessors_state.go

@@ -17,6 +17,8 @@
 package rawdb
 
 import (
+	"encoding/binary"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
@@ -28,6 +30,17 @@ func ReadPreimage(db ethdb.KeyValueReader, hash common.Hash) []byte {
 	return data
 }
 
+// WritePreimages writes the provided set of preimages to the database.
+func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) {
+	for hash, preimage := range preimages {
+		if err := db.Put(preimageKey(hash), preimage); err != nil {
+			log.Crit("Failed to store trie preimage", "err", err)
+		}
+	}
+	preimageCounter.Inc(int64(len(preimages)))
+	preimageHitCounter.Inc(int64(len(preimages)))
+}
+
 // ReadCode retrieves the contract code of the provided code hash.
 func ReadCode(db ethdb.KeyValueReader, hash common.Hash) []byte {
 	// Try with the prefixed code scheme first, if not then try with legacy
@@ -48,12 +61,6 @@ func ReadCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) []byte {
 	return data
 }
 
-// ReadTrieNode retrieves the trie node of the provided hash.
-func ReadTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte {
-	data, _ := db.Get(hash.Bytes())
-	return data
-}
-
 // HasCode checks if the contract code corresponding to the
 // provided code hash is present in the db.
 func HasCode(db ethdb.KeyValueReader, hash common.Hash) bool {
@@ -74,47 +81,117 @@ func HasCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) bool {
 	return ok
 }
 
-// HasTrieNode checks if the trie node with the provided hash is present in db.
-func HasTrieNode(db ethdb.KeyValueReader, hash common.Hash) bool {
-	ok, _ := db.Has(hash.Bytes())
-	return ok
-}
-
-// WritePreimages writes the provided set of preimages to the database.
-func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) {
-	for hash, preimage := range preimages {
-		if err := db.Put(preimageKey(hash), preimage); err != nil {
-			log.Crit("Failed to store trie preimage", "err", err)
-		}
-	}
-	preimageCounter.Inc(int64(len(preimages)))
-	preimageHitCounter.Inc(int64(len(preimages)))
-}
-
 // WriteCode writes the provided contract code database.
 func WriteCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) {
 	if err := db.Put(codeKey(hash), code); err != nil {
 		log.Crit("Failed to store contract code", "err", err)
 	}
 }
 
-// WriteTrieNode writes the provided trie node database.
-func WriteTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) {
-	if err := db.Put(hash.Bytes(), node); err != nil {
-		log.Crit("Failed to store trie node", "err", err)
-	}
-}
-
 // DeleteCode deletes the specified contract code from the database.
 func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) {
 	if err := db.Delete(codeKey(hash)); err != nil {
 		log.Crit("Failed to delete contract code", "err", err)
 	}
 }
 
-// DeleteTrieNode deletes the specified trie node from the database.
-func DeleteTrieNode(db ethdb.KeyValueWriter, hash common.Hash) {
-	if err := db.Delete(hash.Bytes()); err != nil {
-		log.Crit("Failed to delete trie node", "err", err)
+// ReadReverseDiff retrieves the state reverse diff with the given associated
+// identifier. Calculate the real position of reverse diff in freezer by minus
+// one since the first reverse diff is started from one(zero for empty state).
+func ReadReverseDiff(db ethdb.AncientReaderOp, id uint64) []byte {
+	blob, err := db.Ancient(freezerReverseDiffTable, id-1)
+	if err != nil {
+		return nil
+	}
+	return blob
+}
+
+// ReadReverseDiffHash retrieves the state root corresponding to the specified
+// reverse diff. Calculate the real position of reverse diff in freezer by minus
+// one since the first reverse diff is started from one(zero for empty state).
+func ReadReverseDiffHash(db ethdb.AncientReaderOp, id uint64) common.Hash {
+	blob, err := db.Ancient(freezerReverseDiffHashTable, id-1)
+	if err != nil {
+		return common.Hash{}
+	}
+	return common.BytesToHash(blob)
+}
+
+// WriteReverseDiff writes the provided reverse diff to database. Calculate the
+// real position of reverse diff in freezer by minus one since the first reverse
+// diff is started from one(zero for empty state).
+func WriteReverseDiff(db ethdb.AncientWriter, id uint64, blob []byte, state common.Hash) {
+	db.ModifyAncients(func(op ethdb.AncientWriteOp) error {
+		op.AppendRaw(freezerReverseDiffTable, id-1, blob)
+		op.AppendRaw(freezerReverseDiffHashTable, id-1, state.Bytes())
+		return nil
+	})
+}
+
+// ReadReverseDiffLookup retrieves the reverse diff id with the given associated
+// state root. Return nil if it's not existent.
+func ReadReverseDiffLookup(db ethdb.KeyValueReader, root common.Hash) *uint64 {
+	data, err := db.Get(reverseDiffLookupKey(root))
+	if err != nil || len(data) == 0 {
+		return nil
+	}
+	id := binary.BigEndian.Uint64(data)
+	return &id
+}
+
+// WriteReverseDiffLookup writes the provided reverse diff lookup to database.
+func WriteReverseDiffLookup(db ethdb.KeyValueWriter, root common.Hash, id uint64) {
+	var buff [8]byte
+	binary.BigEndian.PutUint64(buff[:], id)
+	if err := db.Put(reverseDiffLookupKey(root), buff[:]); err != nil {
+		log.Crit("Failed to store reverse diff lookup", "err", err)
+	}
+}
+
+// DeleteReverseDiffLookup deletes the specified reverse diff lookup from the database.
+func DeleteReverseDiffLookup(db ethdb.KeyValueWriter, root common.Hash) {
+	if err := db.Delete(reverseDiffLookupKey(root)); err != nil {
+		log.Crit("Failed to delete reverse diff lookup", "err", err)
+	}
+}
+
+// ReadReverseDiffHead retrieves the number of the latest reverse diff from
+// the database.
+func ReadReverseDiffHead(db ethdb.KeyValueReader) uint64 {
+	data, _ := db.Get(reverseDiffHeadKey)
+	if len(data) != 8 {
+		return 0
+	}
+	return binary.BigEndian.Uint64(data)
+}
+
+// WriteReverseDiffHead stores the number of the latest reverse diff id
+// into database.
+func WriteReverseDiffHead(db ethdb.KeyValueWriter, number uint64) {
+	if err := db.Put(reverseDiffHeadKey, encodeBlockNumber(number)); err != nil {
+		log.Crit("Failed to store the head reverse diff id", "err", err)
+	}
+}
+
+// ReadTrieJournal retrieves the serialized in-memory trie node diff layers saved at
+// the last shutdown. The blob is expected to be max a few 10s of megabytes.
+func ReadTrieJournal(db ethdb.KeyValueReader) []byte {
+	data, _ := db.Get(triesJournalKey)
+	return data
+}
+
+// WriteTrieJournal stores the serialized in-memory trie node diff layers to save at
+// shutdown. The blob is expected to be max a few 10s of megabytes.
+func WriteTrieJournal(db ethdb.KeyValueWriter, journal []byte) {
+	if err := db.Put(triesJournalKey, journal); err != nil {
+		log.Crit("Failed to store tries journal", "err", err)
+	}
+}
+
+// DeleteTrieJournal deletes the serialized in-memory trie node diff layers saved at
+// the last shutdown
+func DeleteTrieJournal(db ethdb.KeyValueWriter) {
+	if err := db.Delete(triesJournalKey); err != nil {
+		log.Crit("Failed to remove tries journal", "err", err)
 	}
 }