From 60cdd7d70dca766a0abc489e6f736155ff2d8a50 Mon Sep 17 00:00:00 2001 From: cce <51567+cce@users.noreply.github.com> Date: Fri, 18 Nov 2022 17:11:44 -0500 Subject: [PATCH] tools: update catchpointdump 'database' and 'database check' commands to handle staging tables and KVs (#4802) * update catchpointdump 'database' and 'database check' commands to handle staging tables and KVs * address CR comments --- cmd/catchpointdump/database.go | 16 ++++++++++++++-- cmd/catchpointdump/file.go | 25 ++++++++++++++++--------- cmd/catchpointdump/net.go | 4 ++-- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/cmd/catchpointdump/database.go b/cmd/catchpointdump/database.go index c1ec554e2f..dfbe7bdf5a 100644 --- a/cmd/catchpointdump/database.go +++ b/cmd/catchpointdump/database.go @@ -30,13 +30,16 @@ import ( ) var ledgerTrackerFilename string +var ledgerTrackerStaging bool func init() { databaseCmd.Flags().StringVarP(&ledgerTrackerFilename, "tracker", "t", "", "Specify the ledger tracker file name ( i.e. ./ledger.tracker.sqlite )") databaseCmd.Flags().StringVarP(&outFileName, "output", "o", "", "Specify an outfile for the dump ( i.e. ledger.dump.txt )") + databaseCmd.Flags().BoolVarP(&ledgerTrackerStaging, "staging", "s", false, "Specify whether to look in the catchpoint staging or regular tables. (default false)") databaseCmd.AddCommand(checkCmd) checkCmd.Flags().StringVarP(&ledgerTrackerFilename, "tracker", "t", "", "Specify the ledger tracker file name ( i.e. ./ledger.tracker.sqlite )") + checkCmd.Flags().BoolVarP(&ledgerTrackerStaging, "staging", "s", false, "Specify whether to look in the catchpoint staging or regular tables. (default false)") } var databaseCmd = &cobra.Command{ @@ -58,10 +61,14 @@ var databaseCmd = &cobra.Command{ } defer outFile.Close() } - err = printAccountsDatabase(ledgerTrackerFilename, ledger.CatchpointFileHeader{}, outFile, nil) + err = printAccountsDatabase(ledgerTrackerFilename, ledgerTrackerStaging, ledger.CatchpointFileHeader{}, outFile, nil) if err != nil { reportErrorf("Unable to print account database : %v", err) } + err = printKeyValueStore(ledgerTrackerFilename, ledgerTrackerStaging, outFile) + if err != nil { + reportErrorf("Unable to print key value store : %v", err) + } }, } @@ -99,7 +106,7 @@ func checkDatabase(databaseName string, outFile *os.File) error { var stats merkletrie.Stats err = dbAccessor.Atomic(func(ctx context.Context, tx *sql.Tx) (err error) { - committer, err := ledger.MakeMerkleCommitter(tx, false) + committer, err := ledger.MakeMerkleCommitter(tx, ledgerTrackerStaging) if err != nil { return err } @@ -107,6 +114,11 @@ func checkDatabase(databaseName string, outFile *os.File) error { if err != nil { return err } + root, err := trie.RootHash() + if err != nil { + return err + } + fmt.Fprintf(outFile, " Root: %s\n", root) stats, err = trie.GetStats() if err != nil { return err diff --git a/cmd/catchpointdump/file.go b/cmd/catchpointdump/file.go index 1735726411..aed2dfc1fc 100644 --- a/cmd/catchpointdump/file.go +++ b/cmd/catchpointdump/file.go @@ -127,11 +127,11 @@ var fileCmd = &cobra.Command{ defer outFile.Close() } - err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile, excludedFields.GetSlice()) + err = printAccountsDatabase("./ledger.tracker.sqlite", true, fileHeader, outFile, excludedFields.GetSlice()) if err != nil { reportErrorf("Unable to print account database : %v", err) } - err = printKeyValueStore("./ledger.tracker.sqlite", outFile) + err = printKeyValueStore("./ledger.tracker.sqlite", true, outFile) if err != nil { reportErrorf("Unable to print key value store : %v", err) } @@ -246,7 +246,7 @@ func printDumpingCatchpointProgressLine(progress int, barLength int, dld int64) fmt.Printf(escapeCursorUp + escapeDeleteLine + outString + "\n") } -func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFileHeader, outFile *os.File, excludeFields []string) error { +func printAccountsDatabase(databaseName string, stagingTables bool, fileHeader ledger.CatchpointFileHeader, outFile *os.File, excludeFields []string) error { lastProgressUpdate := time.Now() progress := uint64(0) defer printDumpingCatchpointProgressLine(0, 0, 0) @@ -324,6 +324,9 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile if fileHeader.Version == 0 { var totals ledgercore.AccountTotals id := "" + if stagingTables { + id = "catchpointStaging" + } row := tx.QueryRow("SELECT online, onlinerewardunits, offline, offlinerewardunits, notparticipating, notparticipatingrewardunits, rewardslevel FROM accounttotals WHERE id=?", id) err = row.Scan(&totals.Online.Money.Raw, &totals.Online.RewardUnits, &totals.Offline.Money.Raw, &totals.Offline.RewardUnits, @@ -341,7 +344,7 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile balancesTable := "accountbase" resourcesTable := "resources" - if fileHeader.Version != 0 { + if stagingTables { balancesTable = "catchpointbalances" resourcesTable = "catchpointresources" } @@ -367,7 +370,7 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile return nil } - if fileHeader.Version < ledger.CatchpointFileVersionV6 { + if fileHeader.Version != 0 && fileHeader.Version < ledger.CatchpointFileVersionV6 { var rows *sql.Rows rows, err = tx.Query(fmt.Sprintf("SELECT address, data FROM %s order by address", balancesTable)) if err != nil { @@ -422,7 +425,6 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile return fmt.Errorf("expected %d accounts but got only %d", rowsCount, acctCount) } } - // increase the deadline warning to disable the warning message. _, _ = db.ResetTransactionWarnDeadline(ctx, tx, time.Now().Add(5*time.Second)) return err @@ -441,7 +443,7 @@ func printKeyValue(writer *bufio.Writer, key, value []byte) { fmt.Fprintf(writer, "%s : %v\n", pretty, base64.StdEncoding.EncodeToString(value)) } -func printKeyValueStore(databaseName string, outFile *os.File) error { +func printKeyValueStore(databaseName string, stagingTables bool, outFile *os.File) error { fmt.Printf("\n") printDumpingCatchpointProgressLine(0, 50, 0) lastProgressUpdate := time.Now() @@ -456,15 +458,20 @@ func printKeyValueStore(databaseName string, outFile *os.File) error { return err } + kvTable := "kvstore" + if stagingTables { + kvTable = "catchpointkvstore" + } + return dbAccessor.Atomic(func(ctx context.Context, tx *sql.Tx) error { var rowsCount int64 - err := tx.QueryRow("SELECT count(*) from catchpointkvstore").Scan(&rowsCount) + err := tx.QueryRow(fmt.Sprintf("SELECT count(*) from %s", kvTable)).Scan(&rowsCount) if err != nil { return err } // ordered to make dumps more "diffable" - rows, err := tx.Query("SELECT key, value FROM catchpointkvstore order by key") + rows, err := tx.Query(fmt.Sprintf("SELECT key, value FROM %s order by key", kvTable)) if err != nil { return err } diff --git a/cmd/catchpointdump/net.go b/cmd/catchpointdump/net.go index 8c1b056af3..e63afb1871 100644 --- a/cmd/catchpointdump/net.go +++ b/cmd/catchpointdump/net.go @@ -353,11 +353,11 @@ func loadAndDump(addr string, tarFile string, genesisInitState ledgercore.InitSt return err } defer outFile.Close() - err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile, excludedFields.GetSlice()) + err = printAccountsDatabase("./ledger.tracker.sqlite", true, fileHeader, outFile, excludedFields.GetSlice()) if err != nil { return err } - err = printKeyValueStore("./ledger.tracker.sqlite", outFile) + err = printKeyValueStore("./ledger.tracker.sqlite", true, outFile) if err != nil { return err }