feat: add daily refetch module

CHANGELOG.md

@@ -3,6 +3,10 @@
 #### Staking Module
 - ([\#443](https://github.com/forbole/bdjuno/pull/443)) Remove tombstone status from staking module(already stored in slashing module)
 
+#### Daily refetch
+- ([\#454](https://github.com/forbole/bdjuno/pull/454)) Added `daily refetch` module to refetch missing blocks every day
+
+
 ## Version v3.2.0
 ### Changes
 #### Mint module

database/daily_refetch.go

@@ -0,0 +1,8 @@
+package database
+
+// GetTotalBlocks implements database.Database
+func (db *Db) GetTotalBlocks() (int64, error) {
+	var blockCount int64
+	err := db.Sql.QueryRow(`SELECT count(*) FROM block;`).Scan(&blockCount)
+	return blockCount, err
+}

modules/daily_refetch/handle_periodic_operations.go

@@ -0,0 +1,72 @@
+package daily_refetch
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/forbole/juno/v3/parser"
+	"github.com/forbole/juno/v3/types/config"
+
+	"github.com/go-co-op/gocron"
+	"github.com/rs/zerolog/log"
+
+	parsecmdtypes "github.com/forbole/juno/v3/cmd/parse/types"
+)
+
+func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error {
+	log.Debug().Str("module", "daily refetch").Msg("setting up periodic tasks")
+
+	// Setup a cron job to run every midnight
+	if _, err := scheduler.Every(1).Day().At("00:00").Do(func() {
+		m.refetchMissingBlocks()
+	}); err != nil {
+		return fmt.Errorf("error while setting up daily refetch periodic operation: %s", err)
+	}
+
+	return nil
+}
+
+// refetchMissingBlocks checks for missing blocks from one day ago and refetches them
+func (m *Module) refetchMissingBlocks() error {
+	log.Trace().Str("module", "daily refetch").Str("refetching", "blocks").
+		Msg("refetching missing blocks")
+
+	latestBlock, err := m.node.LatestHeight()
+	if err != nil {
+		return fmt.Errorf("error while getting latest block: %s", err)
+	}
+
+	blockCount, err := m.database.GetTotalBlocks()
+	if err != nil {
+		return fmt.Errorf("error while getting block count: %s", err)
+	}
+
+	blockHeightDayAgo, err := m.database.GetBlockHeightTimeDayAgo(time.Now())
+	if err != nil {
+		return fmt.Errorf("error while getting block height from a day ago: %s", err)
+	}
+	var startHeight int64 = blockHeightDayAgo.Height
+
+	if blockCount != latestBlock {
+		parseCtx, err := parsecmdtypes.GetParserContext(config.Cfg, parsecmdtypes.NewConfig())
+		if err != nil {
+			return err
+		}
+
+		workerCtx := parser.NewContext(parseCtx.EncodingConfig, parseCtx.Node, parseCtx.Database, parseCtx.Logger, parseCtx.Modules)
+		worker := parser.NewWorker(workerCtx, nil, 0)
+
+		log.Info().Int64("start height", startHeight).Int64("end height", latestBlock).
+			Msg("getting missing blocks and transactions from a day ago")
+		for k := startHeight; k <= latestBlock; k++ {
+			err = worker.ProcessIfNotExists(k)
+			if err != nil {
+				return fmt.Errorf("error while re-fetching block %d: %s", k, err)
+			}
+		}
+
+	}
+
+	return nil
+
+}

modules/daily_refetch/module.go

@@ -0,0 +1,35 @@
+package daily_refetch
+
+import (
+	"github.com/forbole/juno/v3/node"
+
+	bdjunodb "github.com/forbole/bdjuno/v3/database"
+
+	"github.com/forbole/juno/v3/modules"
+)
+
+var (
+	_ modules.Module                   = &Module{}
+	_ modules.PeriodicOperationsModule = &Module{}
+)
+
+type Module struct {
+	node     node.Node
+	database *bdjunodb.Db
+}
+
+// NewModule builds a new Module instance
+func NewModule(
+	node node.Node,
+	database *bdjunodb.Db,
+) *Module {
+	return &Module{
+		node:     node,
+		database: database,
+	}
+}
+
+// Name implements modules.Module
+func (m *Module) Name() string {
+	return "daily refetch"
+}

modules/registrar.go

@@ -24,6 +24,7 @@ import (
 	"github.com/forbole/bdjuno/v3/modules/distribution"
 	"github.com/forbole/bdjuno/v3/modules/feegrant"
 
+	dailyrefetch "github.com/forbole/bdjuno/v3/modules/daily_refetch"
 	"github.com/forbole/bdjuno/v3/modules/gov"
 	"github.com/forbole/bdjuno/v3/modules/mint"
 	"github.com/forbole/bdjuno/v3/modules/modules"
@@ -75,6 +76,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules {
 	authModule := auth.NewModule(r.parser, cdc, db)
 	bankModule := bank.NewModule(r.parser, sources.BankSource, cdc, db)
 	consensusModule := consensus.NewModule(db)
+	dailyRefetchModule := dailyrefetch.NewModule(ctx.Proxy, db)
 	distrModule := distribution.NewModule(sources.DistrSource, cdc, db)
 	feegrantModule := feegrant.NewModule(cdc, db)
 	mintModule := mint.NewModule(sources.MintSource, cdc, db)
@@ -91,6 +93,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules {
 		authModule,
 		bankModule,
 		consensusModule,
+		dailyRefetchModule,
 		distrModule,
 		feegrantModule,
 		govModule,