Add debug mode + fix bug (#26)

* Add debug mode for a given epoch

* Handle orphaned/missed blocks

* Add log verbosity to cli

README.md

@@ -58,6 +58,8 @@ This project requires:
 
 ### consensus-client
 
+Remember to enable `--enable-debug-rpc-endpoints`
+
 ### execution-client
 
 ### chaind
@@ -102,6 +104,8 @@ $ ./eth-pools-metrics --help
 Usage of ./eth-pools-metrics:
   -beacon-rpc-endpoint string
     	Address:Port of a eth2 beacon node endpoint (default "localhost:4000")
+  -epoch-debug string
+    	Calculates the stats for a given epoch and exits, useful for debugging
   -eth1address string
     	Ethereum 1 http endpoint. To be used by rocket pool
   -eth2address string
@@ -114,6 +118,8 @@ Usage of ./eth-pools-metrics:
     	Postgres db endpoit: postgresql://user:password@netloc:port/dbname (optional)
   -prometheus-port int
     	Prometheus port to listen to (default 9500)
+  -verbosity string
+    	Logging verbosity (trace, debug, info=default, warn, error, fatal, panic) (default "info")
   -version
     	Prints the release version and exits
   -withdrawal-credentials value

config/config.go

@@ -21,6 +21,8 @@ type Config struct {
 	Postgres              string
 	Eth1Address           string
 	Eth2Address           string
+	EpochDebug            string
+	Verbosity             string
 }
 
 // custom implementation to allow providing the same flag multiple times
@@ -53,6 +55,8 @@ func NewCliConfig() (*Config, error) {
 	var postgres = flag.String("postgres", "", "Postgres db endpoit: postgresql://user:password@netloc:port/dbname (optional)")
 	var eth1Address = flag.String("eth1address", "", "Ethereum 1 http endpoint. To be used by rocket pool")
 	var eth2Address = flag.String("eth2address", "", "Ethereum 2 http endpoint")
+	var epochDebug = flag.String("epoch-debug", "", "Calculates the stats for a given epoch and exits, useful for debugging")
+	var verbosity = flag.String("verbosity", "info", "Logging verbosity (trace, debug, info=default, warn, error, fatal, panic)")
 	flag.Parse()
 
 	if *version {
@@ -86,6 +90,8 @@ func NewCliConfig() (*Config, error) {
 		Postgres:              *postgres,
 		Eth1Address:           *eth1Address,
 		Eth2Address:           *eth2Address,
+		EpochDebug:            *epochDebug,
+		Verbosity:             *verbosity,
 	}
 	logConfig(conf)
 	return conf, nil
@@ -102,5 +108,6 @@ func logConfig(cfg *Config) {
 		"Postgres":       cfg.Postgres,
 		"Eth1Address":    cfg.Eth1Address,
 		"Eth2Address":    cfg.Eth2Address,
+		"EpochDebug":     cfg.EpochDebug,
 	}).Info("Cli Config:")
 }

main.go

@@ -2,14 +2,15 @@ package main
 
 import (
 	"context"
+	"os"
+	"os/signal"
+	"syscall"
+
 	"github.com/alrevuelta/eth-pools-metrics/config"
 	"github.com/alrevuelta/eth-pools-metrics/metrics"
 	"github.com/alrevuelta/eth-pools-metrics/price"
 	"github.com/alrevuelta/eth-pools-metrics/prometheus"
 	log "github.com/sirupsen/logrus"
-	"os"
-	"os/signal"
-	"syscall"
 )
 
 func main() {
@@ -18,6 +19,12 @@ func main() {
 		log.Fatal(err)
 	}
 
+	logLevel, err := log.ParseLevel(config.Verbosity)
+	if err != nil {
+		log.Fatal(err)
+	}
+	log.SetLevel(logLevel)
+
 	prometheus.Run(config.PrometheusPort)
 
 	metrics, err := metrics.NewMetrics(

metrics/metrics.go

@@ -2,7 +2,9 @@ package metrics
 
 import (
 	"context"
+	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -51,7 +53,8 @@ type Metrics struct {
 	Epoch uint64
 	Slot  uint64
 
-	PoolNames []string
+	PoolNames  []string
+	epochDebug string
 }
 
 func NewMetrics(
@@ -115,6 +118,7 @@ func NewMetrics(
 		postgresql:  pg,
 		PoolNames:   config.PoolNames,
 		httpClient:  httpClient,
+		epochDebug:  config.EpochDebug,
 	}, nil
 }
 
@@ -171,6 +175,16 @@ func (a *Metrics) Loop() {
 
 		currentEpoch := uint64(headSlot.HeadSlot)/uint64(32) - 3
 
+		// If a debug epoch is set, overwrite the slot. Will compute just metrics for that epoch
+		if a.epochDebug != "" {
+			epochDebugUint64, err := strconv.ParseUint(a.epochDebug, 10, 64)
+			if err != nil {
+				log.Fatal(err)
+			}
+			log.Warn("Debugging mode, calculating metrics for epoch: ", a.epochDebug)
+			currentEpoch = epochDebugUint64
+		}
+
 		if prevEpoch >= currentEpoch {
 			// do nothing
 			time.Sleep(5 * time.Second)
@@ -236,6 +250,11 @@ func (a *Metrics) Loop() {
 
 		prevBeaconState = currentBeaconState
 		prevEpoch = currentEpoch
+
+		if a.epochDebug != "" {
+			log.Warn("Running in debug mode, exiting ok.")
+			os.Exit(0)
+		}
 	}
 }
 

metrics/proposalduties.go

@@ -3,6 +3,7 @@ package metrics
 import (
 	"context"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/alrevuelta/eth-pools-metrics/prometheus"
@@ -67,6 +68,8 @@ func (p *ProposalDuties) RunProposalMetrics(
 }
 
 func (p *ProposalDuties) GetProposalDuties(epoch uint64) ([]*api.ProposerDuty, error) {
+	log.Info("Fetching proposal duties for epoch: ", epoch)
+
 	// Empty indexes to force fetching all duties
 	indexes := make([]phase0.ValidatorIndex, 0)
 
@@ -83,20 +86,26 @@ func (p *ProposalDuties) GetProposalDuties(epoch uint64) ([]*api.ProposerDuty, e
 }
 
 func (p *ProposalDuties) GetProposedBlocks(epoch uint64) ([]*api.BeaconBlockHeader, error) {
+	log.Info("Fetching proposed blocks for epoch: ", epoch)
 
 	epochBlockHeaders := make([]*api.BeaconBlockHeader, 0)
 	slotsInEpoch := uint64(32)
 
-	slotWithinEpoch := uint64(0)
-	for slotWithinEpoch < slotsInEpoch {
-		epochStr := strconv.FormatUint(epoch*slotsInEpoch+slotWithinEpoch, 10)
+	for i := uint64(0); i < slotsInEpoch; i++ {
+		slot := epoch*slotsInEpoch + uint64(i)
+		slotStr := strconv.FormatUint(slot, 10)
+		log.Debug("Fetching block for slot:" + slotStr)
 
-		blockHeader, err := p.httpClient.BeaconBlockHeader(context.Background(), epochStr)
+		blockHeader, err := p.httpClient.BeaconBlockHeader(context.Background(), slotStr)
 		if err != nil {
-			return epochBlockHeaders, err
+			// This error is expected in skipped or orphaned blocks
+			if !strings.Contains(err.Error(), "Could not find requested block") {
+				return epochBlockHeaders, errors.Wrap(err, "error getting beacon block header")
+			}
+			log.Warn("Block at slot " + slotStr + " was not found")
+			continue
 		}
 		epochBlockHeaders = append(epochBlockHeaders, blockHeader)
-		slotWithinEpoch++
 	}
 
 	return epochBlockHeaders, nil
@@ -114,7 +123,8 @@ func (p *ProposalDuties) GetProposalMetrics(
 	}
 
 	if len(proposalDuties) != len(proposedBlocks) {
-		return proposalMetrics, errors.New("duties and blocks have different sizes")
+		log.Warn("Duties and blocks have different sizes, ok if n blocks were missed/orphaned")
+		//return proposalMetrics, errors.New("duties and blocks have different sizes")
 	}
 
 	if proposalDuties == nil || proposedBlocks == nil {