Skip to content

Commit

Permalink
catchpointdump: allow print filters (#3566)
Browse files Browse the repository at this point in the history 
## Summary

Added a new option `--exclude-fields` to `file` subcommand.
This simplifies large dumps comparison by excluding all exclude some header fields.
Also moved some shared cmd code into a new cmdutil package.

## Test Plan

Tested manually
  • Loading branch information
@algorandskiy
algorandskiy authored Feb 4, 2022
1 parent c765903 commit bc9cebd
Show file tree
Hide file tree
Showing 6 changed files with 193 additions and 89 deletions.
2 changes: 1 addition & 1 deletion cmd/catchpointdump/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ var databaseCmd = &cobra.Command{
}
defer outFile.Close()
}
err = printAccountsDatabase(ledgerTrackerFilename, ledger.CatchpointFileHeader{}, outFile)
err = printAccountsDatabase(ledgerTrackerFilename, ledger.CatchpointFileHeader{}, outFile, nil)
if err != nil {
reportErrorf("Unable to print account database : %v", err)
}
Expand Down
51 changes: 47 additions & 4 deletions cmd/catchpointdump/file.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (

"github.com/spf13/cobra"

cmdutil "github.com/algorand/go-algorand/cmd/util"
"github.com/algorand/go-algorand/config"
"github.com/algorand/go-algorand/data/basics"
"github.com/algorand/go-algorand/ledger"
Expand All @@ -41,10 +42,12 @@ import (

var tarFile string
var outFileName string
var excludedFields *cmdutil.CobraStringSliceValue = cmdutil.MakeCobraStringSliceValue(nil, []string{"version", "catchpoint"})

func init() {
fileCmd.Flags().StringVarP(&tarFile, "tar", "t", "", "Specify the tar file to process")
fileCmd.Flags().StringVarP(&outFileName, "output", "o", "", "Specify an outfile for the dump ( i.e. tracker.dump.txt )")
fileCmd.Flags().VarP(excludedFields, "exclude-fields", "e", "List of fields to exclude from the dump: ["+excludedFields.AllowedString()+"]")
}

var fileCmd = &cobra.Command{
Expand Down Expand Up @@ -108,7 +111,7 @@ var fileCmd = &cobra.Command{
defer outFile.Close()
}

err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile)
err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile, excludedFields.GetSlice())
if err != nil {
reportErrorf("Unable to print account database : %v", err)
}
Expand Down Expand Up @@ -187,7 +190,7 @@ func printDumpingCatchpointProgressLine(progress int, barLength int, dld int64)
fmt.Printf(escapeCursorUp + escapeDeleteLine + outString + "\n")
}

func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFileHeader, outFile *os.File) error {
func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFileHeader, outFile *os.File, excludeFields []string) error {
lastProgressUpdate := time.Now()
progress := uint64(0)
defer printDumpingCatchpointProgressLine(0, 0, 0)
Expand All @@ -200,14 +203,54 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile
return err
}
if fileHeader.Version != 0 {
fmt.Fprintf(fileWriter, "Version: %d\nBalances Round: %d\nBlock Round: %d\nBlock Header Digest: %s\nCatchpoint: %s\nTotal Accounts: %d\nTotal Chunks: %d\n",
var headerFields = []string{
"Version: %d",
"Balances Round: %d",
"Block Round: %d",
"Block Header Digest: %s",
"Catchpoint: %s",
"Total Accounts: %d",
"Total Chunks: %d",
}
var headerValues = []interface{}{
fileHeader.Version,
fileHeader.BalancesRound,
fileHeader.BlocksRound,
fileHeader.BlockHeaderDigest.String(),
fileHeader.Catchpoint,
fileHeader.TotalAccounts,
fileHeader.TotalChunks)
fileHeader.TotalChunks,
}
// safety check
if len(headerFields) != len(headerValues) {
return fmt.Errorf("printing failed: header formatting mismatch")
}

var actualFields []string
var actualValues []interface{}
if len(excludeFields) == 0 {
actualFields = headerFields
actualValues = headerValues
} else {
actualFields = make([]string, 0, len(headerFields)-len(excludeFields))
actualValues = make([]interface{}, 0, len(headerFields)-len(excludeFields))
for i, field := range headerFields {
lower := strings.ToLower(field)
excluded := false
for _, filter := range excludeFields {
if strings.HasPrefix(lower, filter) {
excluded = true
break
}
}
if !excluded {
actualFields = append(actualFields, field)
actualValues = append(actualValues, headerValues[i])
}
}
}

fmt.Fprintf(fileWriter, strings.Join(actualFields, "\n")+"\n", actualValues...)

totals := fileHeader.Totals
fmt.Fprintf(fileWriter, "AccountTotals - Online Money: %d\nAccountTotals - Online RewardUnits : %d\nAccountTotals - Offline Money: %d\nAccountTotals - Offline RewardUnits : %d\nAccountTotals - Not Participating Money: %d\nAccountTotals - Not Participating Money RewardUnits: %d\nAccountTotals - Rewards Level: %d\n",
Expand Down
2 changes: 1 addition & 1 deletion cmd/catchpointdump/net.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ func makeFileDump(addr string, tarFile string) error {
if err != nil {
return err
}
err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile)
err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile, nil)
if err != nil {
return err
}
Expand Down
43 changes: 3 additions & 40 deletions cmd/goal/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,9 @@
package main

import (
"fmt"
"strings"

"github.com/spf13/cobra"

cmdutil "github.com/algorand/go-algorand/cmd/util"
)

const (
Expand Down Expand Up @@ -52,7 +51,7 @@ var (
dumpForDryrunAccts []string
)

var dumpForDryrunFormat cobraStringValue = *makeCobraStringValue("json", []string{"msgp"})
var dumpForDryrunFormat cmdutil.CobraStringValue = *cmdutil.MakeCobraStringValue("json", []string{"msgp"})

func addTxnFlags(cmd *cobra.Command) {
cmd.Flags().Uint64Var(&fee, "fee", 0, "The transaction fee (automatically determined by default), in microAlgos")
Expand All @@ -69,39 +68,3 @@ func addTxnFlags(cmd *cobra.Command) {
cmd.Flags().Var(&dumpForDryrunFormat, "dryrun-dump-format", "Dryrun dump format: "+dumpForDryrunFormat.AllowedString())
cmd.Flags().StringSliceVar(&dumpForDryrunAccts, "dryrun-accounts", nil, "additional accounts to include into dryrun request obj")
}

type cobraStringValue struct {
value string
allowed []string
isSet bool
}

func makeCobraStringValue(value string, others []string) *cobraStringValue {
c := new(cobraStringValue)
c.value = value
c.allowed = make([]string, 0, len(others)+1)
c.allowed = append(c.allowed, value)
for _, s := range others {
c.allowed = append(c.allowed, s)
}
return c
}

func (c *cobraStringValue) String() string { return c.value }
func (c *cobraStringValue) Type() string { return "string" }
func (c *cobraStringValue) IsSet() bool { return c.isSet }

func (c *cobraStringValue) Set(other string) error {
for _, s := range c.allowed {
if other == s {
c.value = other
c.isSet = true
return nil
}
}
return fmt.Errorf("value %s not allowed", other)
}

func (c *cobraStringValue) AllowedString() string {
return strings.Join(c.allowed, ", ")
}
53 changes: 10 additions & 43 deletions cmd/tealdbg/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,14 @@
package main

import (
"fmt"
"io/ioutil"
"log"
"os"
"strings"

"github.com/gorilla/mux"
"github.com/spf13/cobra"

cmdutil "github.com/algorand/go-algorand/cmd/util"
)

func main() {
Expand Down Expand Up @@ -65,49 +65,16 @@ var remoteCmd = &cobra.Command{
},
}

// cobraStringValue is a cobra's string flag with restricted values
type cobraStringValue struct {
value string
allowed []string
isSet bool
}

func makeCobraStringValue(value string, others []string) *cobraStringValue {
c := new(cobraStringValue)
c.value = value
c.allowed = make([]string, 0, len(others)+1)
c.allowed = append(c.allowed, value)
for _, s := range others {
c.allowed = append(c.allowed, s)
}
return c
}

func (c *cobraStringValue) String() string { return c.value }
func (c *cobraStringValue) Type() string { return "string" }
func (c *cobraStringValue) IsSet() bool { return c.isSet }

func (c *cobraStringValue) Set(other string) error {
for _, s := range c.allowed {
if other == s {
c.value = other
c.isSet = true
return nil
}
}
return fmt.Errorf("value %s not allowed", other)
}

func (c *cobraStringValue) AllowedString() string {
return strings.Join(c.allowed, ", ")
type frontendValue struct {
*cmdutil.CobraStringValue
}

type frontendValue struct {
*cobraStringValue
func (f *frontendValue) value() string {
return f.CobraStringValue.String()
}

func (f *frontendValue) Make(router *mux.Router, appAddress string) (da DebugAdapter) {
switch f.value {
switch f.value() {
case "web":
wa := MakeWebPageFrontend(&WebPageFrontendParams{router, appAddress})
return wa
Expand All @@ -120,10 +87,10 @@ func (f *frontendValue) Make(router *mux.Router, appAddress string) (da DebugAda
}

type runModeValue struct {
*cobraStringValue
*cmdutil.CobraStringValue
}

var frontend frontendValue = frontendValue{makeCobraStringValue("cdt", []string{"web"})}
var frontend frontendValue = frontendValue{cmdutil.MakeCobraStringValue("cdt", []string{"web"})}
var proto string
var txnFile string
var groupIndex int
Expand All @@ -133,7 +100,7 @@ var indexerURL string
var indexerToken string
var roundNumber uint64
var timestamp int64
var runMode runModeValue = runModeValue{makeCobraStringValue("auto", []string{"signature", "application"})}
var runMode runModeValue = runModeValue{cmdutil.MakeCobraStringValue("auto", []string{"signature", "application"})}
var port int
var iface string
var noFirstRun bool
Expand Down
Loading

0 comments on commit bc9cebd

Please sign in to comment.