feat: add daily refetch module (forbole#454)

## Description

Closes: BDU-479



---

### Author Checklist

*All items are required. Please add a note to the item if the item is not applicable and
please add links to any relevant follow up issues.*

I have...

- [x] included the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [ ] added `!` to the type prefix if API or client breaking change
- [x] targeted the correct branch
- [x] provided a link to the relevant issue or specification
- [x] added a changelog entry to `CHANGELOG.md`
- [x] included comments for [documenting Go code](https://blog.golang.org/godoc)
- [x] updated the relevant documentation or specification
- [x] reviewed "Files changed" and left comments if necessary
- [x] confirmed all CI checks have passed

### Reviewers Checklist

*All items are required. Please add a note if the item is not applicable and please add
your handle next to the items reviewed if you only reviewed selected items.*

I have...

- [ ] confirmed the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title
- [ ] confirmed `!` in the type prefix if API or client breaking change
- [ ] confirmed all author checklist items have been addressed
- [ ] reviewed API design and naming
- [ ] reviewed documentation is accurate
- [ ] reviewed tests and test coverage
- [ ] manually tested (if applicable)

CHANGELOG.md

@@ -5,6 +5,10 @@
 - ([\#455](https://github.com/forbole/bdjuno/pull/455)) Added `unbonding_tokens` and `staked_not_bonded_tokens` values to staking pool table
 
 
+#### Daily refetch
+- ([\#454](https://github.com/forbole/bdjuno/pull/454)) Added `daily refetch` module to refetch missing blocks every day
+
+
 ## Version v3.2.0
 ### Changes
 #### Mint module

database/daily_refetch.go

@@ -0,0 +1,24 @@
+package database
+
+// GetTotalBlocks implements database.Database
+func (db *Db) GetTotalBlocks() (int64, error) {
+	var blockCount int64
+	err := db.Sql.QueryRow(`SELECT count(*) FROM block;`).Scan(&blockCount)
+	return blockCount, err
+}
+
+// GetMissingBlocks returns an array of missing blocks from one day ago
+func (db *Db) GetMissingBlocks(startHeight, endHeight int64) []int64 {
+	var result []int64
+	stmt := `SELECT generate_series($1::int,$2::int) EXCEPT SELECT height FROM block ORDER BY 1;`
+	err := db.Sqlx.Select(&result, stmt, startHeight, endHeight)
+	if err != nil {
+		return nil
+	}
+
+	if len(result) == 0 {
+		return nil
+	}
+
+	return result
+}

modules/daily_refetch/handle_periodic_operations.go

@@ -0,0 +1,71 @@
+package daily_refetch
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/forbole/juno/v3/parser"
+	"github.com/forbole/juno/v3/types/config"
+
+	"github.com/go-co-op/gocron"
+	"github.com/rs/zerolog/log"
+
+	parsecmdtypes "github.com/forbole/juno/v3/cmd/parse/types"
+)
+
+func (m *Module) RegisterPeriodicOperations(scheduler *gocron.Scheduler) error {
+	log.Debug().Str("module", "daily refetch").Msg("setting up periodic tasks")
+
+	// Setup a cron job to run every midnight
+	if _, err := scheduler.Every(1).Day().At("00:00").Do(func() {
+		m.refetchMissingBlocks()
+	}); err != nil {
+		return fmt.Errorf("error while setting up daily refetch periodic operation: %s", err)
+	}
+
+	return nil
+}
+
+// refetchMissingBlocks checks for missing blocks from one day ago and refetches them
+func (m *Module) refetchMissingBlocks() error {
+	log.Trace().Str("module", "daily refetch").Str("refetching", "blocks").
+		Msg("refetching missing blocks")
+
+	latestBlock, err := m.node.LatestHeight()
+	if err != nil {
+		return fmt.Errorf("error while getting latest block: %s", err)
+	}
+
+	blockHeightDayAgo, err := m.database.GetBlockHeightTimeDayAgo(time.Now())
+	if err != nil {
+		return fmt.Errorf("error while getting block height from a day ago: %s", err)
+	}
+	var startHeight int64 = blockHeightDayAgo.Height
+
+	missingBlocks := m.database.GetMissingBlocks(startHeight, latestBlock)
+
+	// return if no blocks are missing
+	if len(missingBlocks) == 0 {
+		return nil
+	}
+
+	parseCtx, err := parsecmdtypes.GetParserContext(config.Cfg, parsecmdtypes.NewConfig())
+	if err != nil {
+		return err
+	}
+
+	workerCtx := parser.NewContext(parseCtx.EncodingConfig, parseCtx.Node, parseCtx.Database, parseCtx.Logger, parseCtx.Modules)
+	worker := parser.NewWorker(workerCtx, nil, 0)
+
+	log.Info().Int64("start height", startHeight).Int64("end height", latestBlock).
+		Msg("getting missing blocks and transactions from a day ago")
+	for _, block := range missingBlocks {
+		err = worker.Process(block)
+		if err != nil {
+			return fmt.Errorf("error while re-fetching block %d: %s", block, err)
+		}
+	}
+
+	return nil
+
+}

modules/daily_refetch/module.go

@@ -0,0 +1,35 @@
+package daily_refetch
+
+import (
+	"github.com/forbole/juno/v3/node"
+
+	bdjunodb "github.com/forbole/bdjuno/v3/database"
+
+	"github.com/forbole/juno/v3/modules"
+)
+
+var (
+	_ modules.Module                   = &Module{}
+	_ modules.PeriodicOperationsModule = &Module{}
+)
+
+type Module struct {
+	node     node.Node
+	database *bdjunodb.Db
+}
+
+// NewModule builds a new Module instance
+func NewModule(
+	node node.Node,
+	database *bdjunodb.Db,
+) *Module {
+	return &Module{
+		node:     node,
+		database: database,
+	}
+}
+
+// Name implements modules.Module
+func (m *Module) Name() string {
+	return "daily refetch"
+}

modules/registrar.go

@@ -24,6 +24,7 @@ import (
 	"github.com/forbole/bdjuno/v3/modules/distribution"
 	"github.com/forbole/bdjuno/v3/modules/feegrant"
 
+	dailyrefetch "github.com/forbole/bdjuno/v3/modules/daily_refetch"
 	"github.com/forbole/bdjuno/v3/modules/gov"
 	"github.com/forbole/bdjuno/v3/modules/mint"
 	"github.com/forbole/bdjuno/v3/modules/modules"
@@ -75,6 +76,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules {
 	authModule := auth.NewModule(r.parser, cdc, db)
 	bankModule := bank.NewModule(r.parser, sources.BankSource, cdc, db)
 	consensusModule := consensus.NewModule(db)
+	dailyRefetchModule := dailyrefetch.NewModule(ctx.Proxy, db)
 	distrModule := distribution.NewModule(sources.DistrSource, cdc, db)
 	feegrantModule := feegrant.NewModule(cdc, db)
 	mintModule := mint.NewModule(sources.MintSource, cdc, db)
@@ -91,6 +93,7 @@ func (r *Registrar) BuildModules(ctx registrar.Context) jmodules.Modules {
 		authModule,
 		bankModule,
 		consensusModule,
+		dailyRefetchModule,
 		distrModule,
 		feegrantModule,
 		govModule,