diff --git a/cmd/catchpointdump/bench.go b/cmd/catchpointdump/bench.go new file mode 100644 index 0000000000..1cfdd306cd --- /dev/null +++ b/cmd/catchpointdump/bench.go @@ -0,0 +1,184 @@ +// Copyright (C) 2019-2024 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package main + +import ( + "context" + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/algorand/go-algorand/config" + "github.com/algorand/go-algorand/crypto" + "github.com/algorand/go-algorand/data/bookkeeping" + "github.com/algorand/go-algorand/ledger" + "github.com/algorand/go-algorand/ledger/ledgercore" + "github.com/algorand/go-algorand/logging" + "github.com/algorand/go-algorand/protocol" + tools "github.com/algorand/go-algorand/tools/network" +) + +var reportJsonPath string + +func init() { + benchCmd.Flags().StringVarP(&networkName, "net", "n", "", "Specify the network name ( i.e. mainnet.algorand.network )") + benchCmd.Flags().IntVarP(&round, "round", "r", 0, "Specify the round number ( i.e. 7700000 )") + benchCmd.Flags().StringVarP(&relayAddress, "relay", "p", "", "Relay address to use ( i.e. r-ru.algorand-mainnet.network:4160 )") + benchCmd.Flags().StringVarP(&catchpointFile, "tar", "t", "", "Specify the catchpoint file (either .tar or .tar.gz) to process") + benchCmd.Flags().StringVarP(&reportJsonPath, "report", "j", "", "Specify the file to save the Json formatted report to") +} + +var benchCmd = &cobra.Command{ + Use: "bench", + Short: "Benchmark a catchpoint restore", + Long: "Benchmark a catchpoint restore", + Args: validateNoPosArgsFn, + RunE: func(cmd *cobra.Command, args []string) (err error) { + + // Either source the file locally or require a network name to download + if catchpointFile == "" && networkName == "" { + return fmt.Errorf("provide either catchpoint file or network name") + } + loadOnly = true + benchmark := makeBenchmarkReport() + + if catchpointFile == "" { + if round == 0 { + return fmt.Errorf("round not set") + } + stage := benchmark.startStage("network") + catchpointFile, err = downloadCatchpointFromAnyRelay(networkName, round, relayAddress) + if err != nil { + return fmt.Errorf("failed to download catchpoint : %v", err) + } + stage.completeStage() + } + stats, err := os.Stat(catchpointFile) + if err != nil { + return fmt.Errorf("unable to stat '%s' : %v", catchpointFile, err) + } + + catchpointSize := stats.Size() + if catchpointSize == 0 { + return fmt.Errorf("empty file '%s' : %v", catchpointFile, err) + } + + genesisInitState := ledgercore.InitState{ + Block: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{ + UpgradeState: bookkeeping.UpgradeState{ + CurrentProtocol: protocol.ConsensusCurrentVersion, + }, + }}, + } + cfg := config.GetDefaultLocal() + l, err := ledger.OpenLedger(logging.Base(), "./ledger", false, genesisInitState, cfg) + if err != nil { + return fmt.Errorf("unable to open ledger : %v", err) + } + + defer os.Remove("./ledger.block.sqlite") + defer os.Remove("./ledger.block.sqlite-shm") + defer os.Remove("./ledger.block.sqlite-wal") + defer os.Remove("./ledger.tracker.sqlite") + defer os.Remove("./ledger.tracker.sqlite-shm") + defer os.Remove("./ledger.tracker.sqlite-wal") + defer l.Close() + + catchupAccessor := ledger.MakeCatchpointCatchupAccessor(l, logging.Base()) + err = catchupAccessor.ResetStagingBalances(context.Background(), true) + if err != nil { + return fmt.Errorf("unable to initialize catchup database : %v", err) + } + + reader, err := os.Open(catchpointFile) + if err != nil { + return fmt.Errorf("unable to read '%s' : %v", catchpointFile, err) + } + defer reader.Close() + + printDigests = false + stage := benchmark.startStage("database") + + _, err = loadCatchpointIntoDatabase(context.Background(), catchupAccessor, reader, catchpointSize) + if err != nil { + return fmt.Errorf("unable to load catchpoint file into in-memory database : %v", err) + } + stage.completeStage() + + stage = benchmark.startStage("digest") + + err = buildMerkleTrie(context.Background(), catchupAccessor) + if err != nil { + return fmt.Errorf("unable to build Merkle tree : %v", err) + } + stage.completeStage() + + benchmark.printReport() + if reportJsonPath != "" { + if err := benchmark.saveReport(reportJsonPath); err != nil { + fmt.Printf("error writing report to %s: %v\n", reportJsonPath, err) + } + } + + return err + }, +} + +func downloadCatchpointFromAnyRelay(network string, round int, relayAddress string) (string, error) { + var addrs []string + if relayAddress != "" { + addrs = []string{relayAddress} + } else { + //append relays + dnsaddrs, err := tools.ReadFromSRV(context.Background(), "algobootstrap", "tcp", networkName, "", false) + if err != nil || len(dnsaddrs) == 0 { + return "", fmt.Errorf("unable to bootstrap records for '%s' : %v", networkName, err) + } + addrs = append(addrs, dnsaddrs...) + // append archivers + dnsaddrs, err = tools.ReadFromSRV(context.Background(), "archive", "tcp", networkName, "", false) + if err == nil && len(dnsaddrs) > 0 { + addrs = append(addrs, dnsaddrs...) + } + } + + for _, addr := range addrs { + tarName, err := downloadCatchpoint(addr, round) + if err != nil { + reportInfof("failed to download catchpoint from '%s' : %v", addr, err) + continue + } + return tarName, nil + } + return "", fmt.Errorf("catchpoint for round %d on network %s could not be downloaded from any relay", round, network) +} + +func buildMerkleTrie(ctx context.Context, catchupAccessor ledger.CatchpointCatchupAccessor) (err error) { + err = catchupAccessor.BuildMerkleTrie(ctx, func(uint64, uint64) {}) + if err != nil { + return err + } + fmt.Printf("\n Building Merkle Trie, this will take a few minutes...") + var balanceHash, spverHash crypto.Digest + balanceHash, spverHash, _, err = catchupAccessor.GetVerifyData(ctx) + if err != nil { + return err + } + fmt.Printf("done. \naccounts digest=%s, spver digest=%s\n\n", balanceHash, spverHash) + return nil +} diff --git a/cmd/catchpointdump/bench_report.go b/cmd/catchpointdump/bench_report.go new file mode 100644 index 0000000000..87219f2e25 --- /dev/null +++ b/cmd/catchpointdump/bench_report.go @@ -0,0 +1,146 @@ +package main + +import ( + "crypto/sha256" + "encoding/json" + "fmt" + "os" + "runtime" + "syscall" + "time" + + "github.com/google/uuid" + . "github.com/klauspost/cpuid/v2" +) + +type benchStage struct { + stage string + start time.Time + duration time.Duration + cpuTimeNS int64 + completed bool +} + +type hostInfo struct { + CpuCoreCnt int `json:"cores"` + CpuLogicalCnt int `json:"log_cores"` + CpuBaseMHz int64 `json:"base_mhz"` + CpuMaxMHz int64 `json:"max_mhz"` + CpuName string `json:"cpu_name"` + CpuVendor string `json:"cpu_vendor"` + MemMB int `json:"mem_mb"` + OS string `json:"os"` + ID uuid.UUID `json:"uuid"` +} + +type benchReport struct { + ReportID uuid.UUID `json:"report"` + Stages []*benchStage `json:"stages"` + HostInfo *hostInfo `json:"host"` + // TODO: query cpu cores, bogomips and stuff (windows/mac compatible) +} + +func (s *benchStage) MarshalJSON() ([]byte, error) { + return json.Marshal(&struct { + Stage string `json:"stage"` + Duration int64 `json:"duration_sec"` + CpuTime int64 `json:"cpu_time_sec"` + }{ + Stage: s.stage, + Duration: int64(s.duration.Seconds()), + CpuTime: s.cpuTimeNS / 1000000000, + }) +} + +func (bs *benchStage) String() string { + return fmt.Sprintf(">> stage:%s duration_sec:%.1f duration_min:%.1f cpu_sec:%d", bs.stage, bs.duration.Seconds(), bs.duration.Minutes(), bs.cpuTimeNS/1000000000) +} + +func maybeGetTotalMemory() uint64 { + switch runtime.GOOS { + case "linux": + // Use sysinfo on Linux + var si syscall.Sysinfo_t + err := syscall.Sysinfo(&si) + if err != nil { + return 0 + } + return si.Totalram + default: + return 0 + } +} + +func gatherHostInfo() *hostInfo { + nid := sha256.Sum256(uuid.NodeID()) + uuid, _ := uuid.FromBytes(nid[0:16]) + + ni := &hostInfo{ + CpuCoreCnt: CPU.PhysicalCores, + CpuLogicalCnt: CPU.LogicalCores, + CpuName: CPU.BrandName, + CpuVendor: CPU.VendorID.String(), + CpuMaxMHz: CPU.BoostFreq / 1_000_000, + CpuBaseMHz: CPU.Hz / 1_000_000, + MemMB: int(maybeGetTotalMemory()) / 1024 / 1024, + ID: uuid, + OS: runtime.GOOS, + } + + return ni +} + +func makeBenchmarkReport() *benchReport { + uuid, _ := uuid.NewV7() + return &benchReport{ + Stages: make([]*benchStage, 0), + HostInfo: gatherHostInfo(), + ReportID: uuid, + } +} + +func GetCPU() int64 { + usage := new(syscall.Rusage) + syscall.Getrusage(syscall.RUSAGE_SELF, usage) + return usage.Utime.Nano() + usage.Stime.Nano() +} + +func (br *benchReport) startStage(stage string) *benchStage { + bs := &benchStage{ + stage: stage, + start: time.Now(), + duration: 0, + cpuTimeNS: GetCPU(), + completed: false, + } + br.Stages = append(br.Stages, bs) + return bs +} + +func (bs *benchStage) completeStage() { + bs.duration = time.Since(bs.start) + bs.completed = true + bs.cpuTimeNS = GetCPU() - bs.cpuTimeNS +} + +func (br *benchReport) printReport() { + fmt.Print("\nBenchmark report:\n") + for i := range br.Stages { + fmt.Println(br.Stages[i].String()) + } +} + +func (br *benchReport) saveReport(filename string) error { + jsonData, err := json.MarshalIndent(br, "", " ") + if err != nil { + return err + } + + // Write to file with permissions set to 0644 + err = os.WriteFile(filename, jsonData, 0644) + if err != nil { + return err + } + + return nil +} diff --git a/cmd/catchpointdump/commands.go b/cmd/catchpointdump/commands.go index f9dae40186..2165da0a3b 100644 --- a/cmd/catchpointdump/commands.go +++ b/cmd/catchpointdump/commands.go @@ -43,6 +43,7 @@ var versionCheck bool func init() { rootCmd.AddCommand(fileCmd) rootCmd.AddCommand(netCmd) + rootCmd.AddCommand(benchCmd) rootCmd.AddCommand(databaseCmd) }