tools: update catchpointdump 'database' and 'database check' commands…

… to handle staging tables and KVs (#4802)

* update catchpointdump 'database' and 'database check' commands to handle staging tables and KVs

* address CR comments

cmd/catchpointdump/database.go

@@ -30,13 +30,16 @@ import (
 )
 
 var ledgerTrackerFilename string
+var ledgerTrackerStaging bool
 
 func init() {
 	databaseCmd.Flags().StringVarP(&ledgerTrackerFilename, "tracker", "t", "", "Specify the ledger tracker file name ( i.e. ./ledger.tracker.sqlite )")
 	databaseCmd.Flags().StringVarP(&outFileName, "output", "o", "", "Specify an outfile for the dump ( i.e. ledger.dump.txt )")
+	databaseCmd.Flags().BoolVarP(&ledgerTrackerStaging, "staging", "s", false, "Specify whether to look in the catchpoint staging or regular tables. (default false)")
 	databaseCmd.AddCommand(checkCmd)
 
 	checkCmd.Flags().StringVarP(&ledgerTrackerFilename, "tracker", "t", "", "Specify the ledger tracker file name ( i.e. ./ledger.tracker.sqlite )")
+	checkCmd.Flags().BoolVarP(&ledgerTrackerStaging, "staging", "s", false, "Specify whether to look in the catchpoint staging or regular tables. (default false)")
 }
 
 var databaseCmd = &cobra.Command{
@@ -58,10 +61,14 @@ var databaseCmd = &cobra.Command{
 			}
 			defer outFile.Close()
 		}
-		err = printAccountsDatabase(ledgerTrackerFilename, ledger.CatchpointFileHeader{}, outFile, nil)
+		err = printAccountsDatabase(ledgerTrackerFilename, ledgerTrackerStaging, ledger.CatchpointFileHeader{}, outFile, nil)
 		if err != nil {
 			reportErrorf("Unable to print account database : %v", err)
 		}
+		err = printKeyValueStore(ledgerTrackerFilename, ledgerTrackerStaging, outFile)
+		if err != nil {
+			reportErrorf("Unable to print key value store : %v", err)
+		}
 	},
 }
 
@@ -99,14 +106,19 @@ func checkDatabase(databaseName string, outFile *os.File) error {
 
 	var stats merkletrie.Stats
 	err = dbAccessor.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) {
-		committer, err := ledger.MakeMerkleCommitter(tx, false)
+		committer, err := ledger.MakeMerkleCommitter(tx, ledgerTrackerStaging)
 		if err != nil {
 			return err
 		}
 		trie, err := merkletrie.MakeTrie(committer, ledger.TrieMemoryConfig)
 		if err != nil {
 			return err
 		}
+		root, err := trie.RootHash()
+		if err != nil {
+			return err
+		}
+		fmt.Fprintf(outFile, " Root: %s\n", root)
 		stats, err = trie.GetStats()
 		if err != nil {
 			return err

cmd/catchpointdump/file.go

@@ -127,11 +127,11 @@ var fileCmd = &cobra.Command{
 				defer outFile.Close()
 			}
 
-			err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile, excludedFields.GetSlice())
+			err = printAccountsDatabase("./ledger.tracker.sqlite", true, fileHeader, outFile, excludedFields.GetSlice())
 			if err != nil {
 				reportErrorf("Unable to print account database : %v", err)
 			}
-			err = printKeyValueStore("./ledger.tracker.sqlite", outFile)
+			err = printKeyValueStore("./ledger.tracker.sqlite", true, outFile)
 			if err != nil {
 				reportErrorf("Unable to print key value store : %v", err)
 			}
@@ -246,7 +246,7 @@ func printDumpingCatchpointProgressLine(progress int, barLength int, dld int64)
 	fmt.Printf(escapeCursorUp + escapeDeleteLine + outString + "\n")
 }
 
-func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFileHeader, outFile *os.File, excludeFields []string) error {
+func printAccountsDatabase(databaseName string, stagingTables bool, fileHeader ledger.CatchpointFileHeader, outFile *os.File, excludeFields []string) error {
 	lastProgressUpdate := time.Now()
 	progress := uint64(0)
 	defer printDumpingCatchpointProgressLine(0, 0, 0)
@@ -324,6 +324,9 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile
 		if fileHeader.Version == 0 {
 			var totals ledgercore.AccountTotals
 			id := ""
+			if stagingTables {
+				id = "catchpointStaging"
+			}
 			row := tx.QueryRow("SELECT online, onlinerewardunits, offline, offlinerewardunits, notparticipating, notparticipatingrewardunits, rewardslevel FROM accounttotals WHERE id=?", id)
 			err = row.Scan(&totals.Online.Money.Raw, &totals.Online.RewardUnits,
 				&totals.Offline.Money.Raw, &totals.Offline.RewardUnits,
@@ -341,7 +344,7 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile
 
 		balancesTable := "accountbase"
 		resourcesTable := "resources"
-		if fileHeader.Version != 0 {
+		if stagingTables {
 			balancesTable = "catchpointbalances"
 			resourcesTable = "catchpointresources"
 		}
@@ -367,7 +370,7 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile
 			return nil
 		}
 
-		if fileHeader.Version < ledger.CatchpointFileVersionV6 {
+		if fileHeader.Version != 0 && fileHeader.Version < ledger.CatchpointFileVersionV6 {
 			var rows *sql.Rows
 			rows, err = tx.Query(fmt.Sprintf("SELECT address, data FROM %s order by address", balancesTable))
 			if err != nil {
@@ -422,7 +425,6 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile
 				return fmt.Errorf("expected %d accounts but got only %d", rowsCount, acctCount)
 			}
 		}
-
 		// increase the deadline warning to disable the warning message.
 		_, _ = db.ResetTransactionWarnDeadline(ctx, tx, time.Now().Add(5*time.Second))
 		return err
@@ -441,7 +443,7 @@ func printKeyValue(writer *bufio.Writer, key, value []byte) {
 	fmt.Fprintf(writer, "%s : %v\n", pretty, base64.StdEncoding.EncodeToString(value))
 }
 
-func printKeyValueStore(databaseName string, outFile *os.File) error {
+func printKeyValueStore(databaseName string, stagingTables bool, outFile *os.File) error {
 	fmt.Printf("\n")
 	printDumpingCatchpointProgressLine(0, 50, 0)
 	lastProgressUpdate := time.Now()
@@ -456,15 +458,20 @@ func printKeyValueStore(databaseName string, outFile *os.File) error {
 		return err
 	}
 
+	kvTable := "kvstore"
+	if stagingTables {
+		kvTable = "catchpointkvstore"
+	}
+
 	return dbAccessor.Atomic(func(ctx context.Context, tx *sql.Tx) error {
 		var rowsCount int64
-		err := tx.QueryRow("SELECT count(*) from catchpointkvstore").Scan(&rowsCount)
+		err := tx.QueryRow(fmt.Sprintf("SELECT count(*) from %s", kvTable)).Scan(&rowsCount)
 		if err != nil {
 			return err
 		}
 
 		// ordered to make dumps more "diffable"
-		rows, err := tx.Query("SELECT key, value FROM catchpointkvstore order by key")
+		rows, err := tx.Query(fmt.Sprintf("SELECT key, value FROM %s order by key", kvTable))
 		if err != nil {
 			return err
 		}

cmd/catchpointdump/net.go

@@ -353,11 +353,11 @@ func loadAndDump(addr string, tarFile string, genesisInitState ledgercore.InitSt
 			return err
 		}
 		defer outFile.Close()
-		err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile, excludedFields.GetSlice())
+		err = printAccountsDatabase("./ledger.tracker.sqlite", true, fileHeader, outFile, excludedFields.GetSlice())
 		if err != nil {
 			return err
 		}
-		err = printKeyValueStore("./ledger.tracker.sqlite", outFile)
+		err = printKeyValueStore("./ledger.tracker.sqlite", true, outFile)
 		if err != nil {
 			return err
 		}