feat(dot/state): Implement online pruning of state tries.

cmd/gossamer/config.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ChainSafe/gossamer/dot"
 	ctoml "github.com/ChainSafe/gossamer/dot/config/toml"
 	"github.com/ChainSafe/gossamer/dot/state"
+	"github.com/ChainSafe/gossamer/dot/state/pruner"
 	"github.com/ChainSafe/gossamer/dot/types"
 	"github.com/ChainSafe/gossamer/lib/common"
 	"github.com/ChainSafe/gossamer/lib/genesis"
@@ -443,6 +444,9 @@ func setDotGlobalConfigFromToml(tomlCfg *ctoml.Config, cfg *dot.GlobalConfig) {
 		}
 
 		cfg.MetricsPort = tomlCfg.Global.MetricsPort
+
+		cfg.RetainBlocks = tomlCfg.Global.RetainBlocks
+		cfg.Pruning = pruner.Mode(tomlCfg.Global.Pruning)
 	}
 }
 
@@ -472,6 +476,8 @@ func setDotGlobalConfigFromFlags(ctx *cli.Context, cfg *dot.GlobalConfig) {
 		cfg.MetricsPort = uint32(metricsPort)
 	}
 
+	cfg.RetainBlocks = ctx.GlobalInt64(RetainBlockNumberFlag.Name)
+	cfg.Pruning = pruner.Mode(ctx.GlobalString(Pruning.Name))
 	cfg.NoTelemetry = ctx.Bool("no-telemetry")
 }
 

cmd/gossamer/export.go

@@ -79,11 +79,13 @@ func dotConfigToToml(dcfg *dot.Config) *ctoml.Config {
 	cfg := &ctoml.Config{}
 
 	cfg.Global = ctoml.GlobalConfig{
-		Name:        dcfg.Global.Name,
-		ID:          dcfg.Global.ID,
-		BasePath:    dcfg.Global.BasePath,
-		LogLvl:      dcfg.Global.LogLvl.String(),
-		MetricsPort: dcfg.Global.MetricsPort,
+		Name:         dcfg.Global.Name,
+		ID:           dcfg.Global.ID,
+		BasePath:     dcfg.Global.BasePath,
+		LogLvl:       dcfg.Global.LogLvl.String(),
+		MetricsPort:  dcfg.Global.MetricsPort,
+		RetainBlocks: dcfg.Global.RetainBlocks,
+		Pruning:      string(dcfg.Global.Pruning),
 	}
 
 	cfg.Log = ctoml.LogConfig{

cmd/gossamer/flags.go

@@ -285,6 +285,14 @@ var (
 		Usage: "Retain number of block from latest block while pruning",
 		Value: 256,
 	}
+
+	// Pruning triggers the online pruning of historical state tries. It's either full or archive. To enable pruning the value
+	// should be set to `full`.
+	Pruning = cli.StringFlag{
+		Name:  "pruning",
+		Usage: `State trie online pruning ("full", "archive")`,
+		Value: "full",
+	}
 )
 
 // flag sets that are shared by multiple commands
@@ -302,6 +310,7 @@ var (
 		DBPathFlag,
 		BloomFilterSizeFlag,
 		RetainBlockNumberFlag,
+		Pruning,
 	}
 
 	// StartupFlags are flags that are valid for use with the root command and the export subcommand

cmd/gossamer/main.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/ChainSafe/gossamer/dot"
 	"github.com/ChainSafe/gossamer/dot/state"
+	"github.com/ChainSafe/gossamer/dot/state/pruner"
 	"github.com/ChainSafe/gossamer/lib/keystore"
 	"github.com/ChainSafe/gossamer/lib/utils"
 	log "github.com/ChainSafe/log15"
@@ -37,6 +38,7 @@ const (
 	importRuntimeCommandName = "import-runtime"
 	importStateCommandName   = "import-state"
 	pruningStateCommandName  = "prune-state"
+	defaultRetainBlocks      = 256
 )
 
 // app is the cli application
@@ -234,6 +236,14 @@ func gossamerAction(ctx *cli.Context) error {
 		return err
 	}
 
+	if cfg.Global.RetainBlocks < defaultRetainBlocks {
+		return fmt.Errorf("--%s cannot be less than %d", RetainBlockNumberFlag.Name, defaultRetainBlocks)
+	}
+
+	if !cfg.Global.Pruning.IsValid() {
+		return fmt.Errorf("--%s must be either %s or %s", pruner.Full, pruner.Archive, Pruning.Name)
+	}
+
 	cfg.Global.LogLvl = lvl
 
 	// expand data directory and update node configuration (performed separately
@@ -448,7 +458,7 @@ func pruneState(ctx *cli.Context) error {
 		return err
 	}
 
-	logger.Info("Pruner initialised")
+	logger.Info("Offline pruner initialised")
 
 	err = pruner.SetBloomFilter()
 	if err != nil {

dot/build_spec.go

@@ -107,7 +107,11 @@ func BuildFromDB(path string) (*BuildSpec, error) {
 	tmpGen.Genesis.Raw = make(map[string]map[string]string)
 	tmpGen.Genesis.Runtime = make(map[string]map[string]interface{})
 
-	stateSrvc := state.NewService(path, log.LvlCrit)
+	config := state.Config{
+		Path:     path,
+		LogLevel: log.LvlInfo,
+	}
+	stateSrvc := state.NewService(config)
 
 	// start state service (initialise state database)
 	err := stateSrvc.Start()

dot/config.go

@@ -23,6 +23,7 @@ import (
 	"github.com/ChainSafe/gossamer/chain/gssmr"
 	"github.com/ChainSafe/gossamer/chain/kusama"
 	"github.com/ChainSafe/gossamer/chain/polkadot"
+	"github.com/ChainSafe/gossamer/dot/state/pruner"
 	"github.com/ChainSafe/gossamer/dot/types"
 	log "github.com/ChainSafe/log15"
 )
@@ -52,6 +53,8 @@ type GlobalConfig struct {
 	PublishMetrics bool
 	MetricsPort    uint32
 	NoTelemetry    bool
+	RetainBlocks   int64
+	Pruning        pruner.Mode
 }
 
 // LogConfig represents the log levels for individual packages

dot/config/toml/config.go

@@ -29,11 +29,13 @@ type Config struct {
 
 // GlobalConfig is to marshal/unmarshal toml global config vars
 type GlobalConfig struct {
-	Name        string `toml:"name,omitempty"`
-	ID          string `toml:"id,omitempty"`
-	BasePath    string `toml:"basepath,omitempty"`
-	LogLvl      string `toml:"log,omitempty"`
-	MetricsPort uint32 `toml:"metrics-port,omitempty"`
+	Name         string `toml:"name,omitempty"`
+	ID           string `toml:"id,omitempty"`
+	BasePath     string `toml:"basepath,omitempty"`
+	LogLvl       string `toml:"log,omitempty"`
+	MetricsPort  uint32 `toml:"metrics-port,omitempty"`
+	RetainBlocks int64  `toml:"retain-blocks,omitempty"`
+	Pruning      string `toml:"pruning,omitempty"`
 }
 
 // LogConfig represents the log levels for individual packages

dot/core/digest_test.go

@@ -35,7 +35,12 @@ import (
 func newTestDigestHandler(t *testing.T, withBABE, withGrandpa bool) *DigestHandler { //nolint
 	testDatadirPath, err := ioutil.TempDir("/tmp", "test-datadir-*")
 	require.NoError(t, err)
-	stateSrvc := state.NewService(testDatadirPath, log.LvlInfo)
+
+	config := state.Config{
+		Path:     testDatadirPath,
+		LogLevel: log.LvlInfo,
+	}
+	stateSrvc := state.NewService(config)
 	stateSrvc.UseMemDB()
 
 	gen, genTrie, genHeader := newTestGenesisWithTrieAndHeader(t)

dot/core/test_helpers.go

@@ -130,7 +130,11 @@ func NewTestService(t *testing.T, cfg *Config) *Service {
 	gen, genTrie, genHeader := newTestGenesisWithTrieAndHeader(t)
 
 	if cfg.BlockState == nil || cfg.StorageState == nil || cfg.TransactionState == nil || cfg.EpochState == nil {
-		stateSrvc = state.NewService(testDatadirPath, log.LvlInfo)
+		config := state.Config{
+			Path:     testDatadirPath,
+			LogLevel: log.LvlInfo,
+		}
+		stateSrvc = state.NewService(config)
 		stateSrvc.UseMemDB()
 
 		err = stateSrvc.Initialise(gen, genHeader, genTrie)

dot/import.go

@@ -45,7 +45,11 @@ func ImportState(basepath, stateFP, headerFP string, firstSlot uint64) error {
 
 	log.Info("ImportState", "header", header)
 
-	srv := state.NewService(basepath, log.LvlInfo)
+	config := state.Config{
+		Path:     basepath,
+		LogLevel: log.LvlInfo,
+	}
+	srv := state.NewService(config)
 	return srv.Import(header, tr, firstSlot)
 }
 

dot/node.go

@@ -90,8 +90,13 @@ func InitNode(cfg *Config) error {
 		return fmt.Errorf("failed to create genesis block from trie: %w", err)
 	}
 
+	config := state.Config{
+		Path:     cfg.Global.BasePath,
+		LogLevel: cfg.Global.LogLvl,
+	}
+
 	// create new state service
-	stateSrvc := state.NewService(cfg.Global.BasePath, cfg.Global.LogLvl)
+	stateSrvc := state.NewService(config)
 
 	// initialise state service with genesis data, block, and trie
 	err = stateSrvc.Initialise(gen, header, t)

dot/node_test.go

@@ -212,7 +212,11 @@ func TestInitNode_LoadGenesisData(t *testing.T) {
 	err := InitNode(cfg)
 	require.NoError(t, err)
 
-	stateSrvc := state.NewService(cfg.Global.BasePath, log.LvlTrace)
+	config := state.Config{
+		Path:     cfg.Global.BasePath,
+		LogLevel: log.LvlInfo,
+	}
+	stateSrvc := state.NewService(config)
 
 	gen, err := genesis.NewGenesisFromJSONRaw(genPath)
 	require.NoError(t, err)

dot/rpc/modules/chain_test.go

@@ -280,7 +280,12 @@ var gen, genTrie, genesisHeader = newTestGenesisWithTrieAndHeader()
 func newTestStateService(t *testing.T) *state.Service {
 	testDatadirPath, err := ioutil.TempDir("/tmp", "test-datadir-*")
 	require.NoError(t, err)
-	stateSrvc := state.NewService(testDatadirPath, log.LvlInfo)
+
+	config := state.Config{
+		Path:     testDatadirPath,
+		LogLevel: log.LvlInfo,
+	}
+	stateSrvc := state.NewService(config)
 	stateSrvc.UseMemDB()
 
 	err = stateSrvc.Initialise(gen, genesisHeader, genTrie)

dot/rpc/modules/state_test.go

@@ -429,7 +429,7 @@ func setupStateModule(t *testing.T) (*StateModule, *common.Hash, *common.Hash) {
 
 	sr1, err := ts.Root()
 	require.NoError(t, err)
-	err = chain.Storage.StoreTrie(ts)
+	err = chain.Storage.StoreTrie(ts, nil)
 	require.NoError(t, err)
 
 	err = chain.Block.AddBlock(&types.Block{

dot/rpc/modules/system_test.go

@@ -359,7 +359,7 @@ func setupSystemModule(t *testing.T) *SystemModule {
 	require.NoError(t, err)
 	ts.Set(aliceAcctStoKey, aliceAcctEncoded)
 
-	err = chain.Storage.StoreTrie(ts)
+	err = chain.Storage.StoreTrie(ts, nil)
 	require.NoError(t, err)
 	err = chain.Block.AddBlock(&types.Block{
 		Header: &types.Header{

dot/services.go

@@ -22,6 +22,7 @@ import (
 	"path/filepath"
 
 	"github.com/ChainSafe/chaindb"
+	"github.com/ChainSafe/gossamer/dot/state/pruner"
 
 	"github.com/ChainSafe/gossamer/dot/core"
 	"github.com/ChainSafe/gossamer/dot/network"
@@ -53,7 +54,20 @@ func newInMemoryDB(path string) (chaindb.Database, error) {
 // createStateService creates the state service and initialise state database
 func createStateService(cfg *Config) (*state.Service, error) {
 	logger.Debug("creating state service...")
-	stateSrvc := state.NewService(cfg.Global.BasePath, cfg.Log.StateLvl)
+
+	config := state.Config{
+		Path:     cfg.Global.BasePath,
+		LogLevel: cfg.Log.StateLvl,
+		Pruning: struct {
+			Mode              pruner.Mode
+			NumRetainedBlocks int64
+		}{
+			Mode:              cfg.Global.Pruning,
+			NumRetainedBlocks: cfg.Global.RetainBlocks,
+		},
+	}
+
+	stateSrvc := state.NewService(config)
 
 	// start state service (initialise state database)
 	err := stateSrvc.Start()

dot/state/initialize.go

@@ -89,7 +89,7 @@ func (s *Service) Initialise(gen *genesis.Genesis, header *types.Header, t *trie
 	}
 
 	// create storage state from genesis trie
-	storageState, err := NewStorageState(db, blockState, t)
+	storageState, err := NewStorageState(db, blockState, t, "", 0)
 	if err != nil {
 		return fmt.Errorf("failed to create storage state from trie: %s", err)
 	}

dot/state/prune.go → dot/state/offline_pruner.go

@@ -14,11 +14,11 @@ import (
 	"github.com/dgraph-io/badger/v2/pb"
 )
 
-// Pruner is an offline tool to prune the stale state with the help of
-// bloom filter, The workflow of pruner is very simple:
+// OfflinePruner is a tool to prune the stale state with the help of
+// bloom filter, The workflow of Pruner is very simple:
 // - iterate the storage state, reconstruct the relevant state tries
 // - iterate the database, stream all the targeted keys to new DB
-type Pruner struct {
+type OfflinePruner struct {
 	inputDB        *chaindb.BadgerDB
 	storageState   *StorageState
 	blockState     *BlockState
@@ -30,8 +30,8 @@ type Pruner struct {
 	prunedDBPath string
 }
 
-// NewPruner creates an instance of Pruner.
-func NewPruner(inputDBPath, prunedDBPath string, bloomSize uint64, retainBlockNum int64) (*Pruner, error) {
+// NewPruner creates an instance of OfflinePruner.
+func NewPruner(inputDBPath, prunedDBPath string, bloomSize uint64, retainBlockNum int64) (*OfflinePruner, error) {
 	db, err := utils.LoadChainDB(inputDBPath)
 	if err != nil {
 		return nil, fmt.Errorf("failed to load DB %w", err)
@@ -62,12 +62,12 @@ func NewPruner(inputDBPath, prunedDBPath string, bloomSize uint64, retainBlockNu
 	}
 
 	// load storage state
-	storageState, err := NewStorageState(db, blockState, trie.NewEmptyTrie())
+	storageState, err := NewStorageState(db, blockState, trie.NewEmptyTrie(), "", 0)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create new storage state %w", err)
 	}
 
-	return &Pruner{
+	return &OfflinePruner{
 		inputDB:        db,
 		storageState:   storageState,
 		blockState:     blockState,
@@ -80,7 +80,7 @@ func NewPruner(inputDBPath, prunedDBPath string, bloomSize uint64, retainBlockNu
 }
 
 // SetBloomFilter loads keys with storage prefix of last `retainBlockNum` blocks into the bloom filter
-func (p *Pruner) SetBloomFilter() error {
+func (p *OfflinePruner) SetBloomFilter() error {
 	defer p.inputDB.Close() // nolint: errcheck
 	finalisedHash, err := p.blockState.GetFinalizedHash(0, 0)
 	if err != nil {
@@ -134,7 +134,7 @@ func (p *Pruner) SetBloomFilter() error {
 }
 
 // Prune starts streaming the data from input db to the pruned db.
-func (p *Pruner) Prune() error {
+func (p *OfflinePruner) Prune() error {
 	inputDB, err := utils.LoadBadgerDB(p.inputDBPath)
 	if err != nil {
 		return fmt.Errorf("failed to load DB %w", err)