feat: add daily refetch module

CHANGELOG.md

@@ -3,6 +3,10 @@
 #### Staking Module
 - ([\#443](https://github.com/forbole/bdjuno/pull/443)) Remove tombstone status from staking module(already stored in slashing module)
 
+#### Daily refetch
+- ([\#454](https://github.com/forbole/bdjuno/pull/454)) Added `daily refetch` module to refetch missing blocks every day
+
+
 ## Version v3.2.0
 ### Changes
 #### Mint module

database/daily_refetch.go

@@ -0,0 +1,24 @@
+package database
+
+// GetTotalBlocks implements database.Database
+func (db *Db) GetTotalBlocks() (int64, error) {
+	var blockCount int64
+	err := db.Sql.QueryRow(`SELECT count(*) FROM block;`).Scan(&blockCount)
+	return blockCount, err
+}
+
+// GetMissingBlocks returns an array of missing blocks from one day ago
+func (db *Db) GetMissingBlocks(startHeight, endHeight int64) []int64 {
+	var result []int64
+	stmt := `SELECT generate_series($1::int,$2::int) EXCEPT SELECT height FROM block ORDER BY 1;`
+	err := db.Sqlx.Select(&result, stmt, startHeight, endHeight)
+	if err != nil {
+		return nil
+	}
+
+	if len(result) == 0 {
+		return nil
+	}
+
+	return result
+}

modules/daily_refetch/handle_periodic_operations.go

@@ -0,0 +1,71 @@
+package daily_refetch
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/forbole/juno/v3/parser"
+	"github.com/forbole/juno/v3/types/config"
+
+	"github.com/go-co-op/gocron"
+	"github.com/rs/zerolog/log"
+
+	parsecmdtypes "github.com/forbole/juno/v3/cmd/parse/types"
+)
+
+func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error {
+	log.Debug().Str("module", "daily refetch").Msg("setting up periodic tasks")
+
+	// Setup a cron job to run every midnight
+	if _, err := scheduler.Every(1).Day().At("00:00").Do(func() {
+		m.refetchMissingBlocks()
+	}); err != nil {
+		return fmt.Errorf("error while setting up daily refetch periodic operation: %s", err)
+	}
+
+	return nil
+}
+
+// refetchMissingBlocks checks for missing blocks from one day ago and refetches them
+func (m *Module) refetchMissingBlocks() error {
+	log.Trace().Str("module", "daily refetch").Str("refetching", "blocks").
+		Msg("refetching missing blocks")
+
+	latestBlock, err := m.node.LatestHeight()
+	if err != nil {
+		return fmt.Errorf("error while getting latest block: %s", err)
+	}
+
+	blockHeightDayAgo, err := m.database.GetBlockHeightTimeDayAgo(time.Now())
+	if err != nil {
+		return fmt.Errorf("error while getting block height from a day ago: %s", err)
+	}
+	var startHeight int64 = blockHeightDayAgo.Height
+
+	missingBlocks := m.database.GetMissingBlocks(startHeight, latestBlock)
+
+	// return if no blocks are missing
+	if len(missingBlocks) == 0 {
+		return nil
+	}
+
+	parseCtx, err := parsecmdtypes.GetParserContext(config.Cfg, parsecmdtypes.NewConfig())
+	if err != nil {
+		return err
+	}
+
+	workerCtx := parser.NewContext(parseCtx.EncodingConfig, parseCtx.Node, parseCtx.Database, parseCtx.Logger, parseCtx.Modules)
+	worker := parser.NewWorker(workerCtx, nil, 0)
+
+	log.Info().Int64("start height", startHeight).Int64("end height", latestBlock).
+		Msg("getting missing blocks and transactions from a day ago")
+	for _, block := range missingBlocks {
+		err = worker.Process(block)
+		if err != nil {
+			return fmt.Errorf("error while re-fetching block %d: %s", block, err)
+		}
+	}
+
+	return nil
+
+}

modules/daily_refetch/module.go

@@ -0,0 +1,35 @@
+package daily_refetch
+
+import (
+	"github.com/forbole/juno/v3/node"
+
+	bdjunodb "github.com/forbole/bdjuno/v3/database"
+
+	"github.com/forbole/juno/v3/modules"
+)
+
+var (
+	_ modules.Module                   = &Module{}
+	_ modules.PeriodicOperationsModule = &Module{}
+)
+
+type Module struct {
+	node     node.Node
+	database *bdjunodb.Db
+}
+
+// NewModule builds a new Module instance
+func NewModule(
+	node node.Node,
+	database *bdjunodb.Db,
+) *Module {
+	return &Module{
+		node:     node,
+		database: database,
+	}
+}
+
+// Name implements modules.Module
+func (m *Module) Name() string {
+	return "daily refetch"
+}

modules/registrar.go

@@ -24,6 +24,7 @@ import (
 	"github.com/forbole/bdjuno/v3/modules/distribution"
 	"github.com/forbole/bdjuno/v3/modules/feegrant"
 
+	dailyrefetch "github.com/forbole/bdjuno/v3/modules/daily_refetch"
 	"github.com/forbole/bdjuno/v3/modules/gov"
 	"github.com/forbole/bdjuno/v3/modules/mint"
 	"github.com/forbole/bdjuno/v3/modules/modules"
@@ -75,6 +76,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules {
 	authModule := auth.NewModule(r.parser, cdc, db)
 	bankModule := bank.NewModule(r.parser, sources.BankSource, cdc, db)
 	consensusModule := consensus.NewModule(db)
+	dailyRefetchModule := dailyrefetch.NewModule(ctx.Proxy, db)
 	distrModule := distribution.NewModule(sources.DistrSource, cdc, db)
 	feegrantModule := feegrant.NewModule(cdc, db)
 	mintModule := mint.NewModule(sources.MintSource, cdc, db)
@@ -91,6 +93,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules {
 		authModule,
 		bankModule,
 		consensusModule,
+		dailyRefetchModule,
 		distrModule,
 		feegrantModule,
 		govModule,