From 5a9e273c6dfa548590693b41088524051e5d11c5 Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 20 Nov 2024 10:35:37 +0100 Subject: [PATCH 1/8] save --- erigon-lib/commitment/hex_patricia_hashed.go | 28 ++++++++++++++++++-- erigon-lib/state/domain_shared.go | 11 ++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/erigon-lib/commitment/hex_patricia_hashed.go b/erigon-lib/commitment/hex_patricia_hashed.go index 7f3632b078a..9282561756f 100644 --- a/erigon-lib/commitment/hex_patricia_hashed.go +++ b/erigon-lib/commitment/hex_patricia_hashed.go @@ -40,11 +40,11 @@ import ( "golang.org/x/crypto/sha3" - "github.com/erigontech/erigon-lib/common/hexutility" - "github.com/erigontech/erigon-lib/common" + "github.com/erigontech/erigon-lib/common/hexutility" "github.com/erigontech/erigon-lib/common/length" "github.com/erigontech/erigon-lib/rlp" + state2 "github.com/erigontech/erigon-lib/state" ) // keccakState wraps sha3.state. In addition to the usual hash methods, it also supports @@ -975,10 +975,12 @@ func (hph *HexPatriciaHashed) needUnfolding(hashedKey []byte) int { // unfoldBranchNode returns true if unfolding has been done func (hph *HexPatriciaHashed) unfoldBranchNode(row, depth int, deleted bool) (bool, error) { key := hexToCompact(hph.currentKey[:hph.currentKeyLen]) + timeSpentReadingBranchA := time.Now() branchData, fileEndTxNum, err := hph.ctx.Branch(key) if err != nil { return false, err } + timeSpentReadingBranch += time.Since(timeSpentReadingBranchA) hph.depthsToTxNum[depth] = fileEndTxNum if len(branchData) >= 2 { branchData = branchData[2:] // skip touch map and keep the rest @@ -1517,6 +1519,18 @@ func (hph *HexPatriciaHashed) RootHash() ([]byte, error) { return rootHash[1:], nil // first byte is 128+hash_len=160 } +var ( + timeSpentFold time.Duration + timeSpentUnf time.Duration + timeSpentReadingAccount time.Duration + timeSpentReadingStorage time.Duration + timeSpentReadingBranch time.Duration +) + +func printShit() { + +} + func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, logPrefix string) (rootHash []byte, err error) { var ( m runtime.MemStats @@ -1546,29 +1560,37 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log } // Keep folding until the currentKey is the prefix of the key we modify for hph.needFolding(hashedKey) { + start := time.Now() if err := hph.fold(); err != nil { return fmt.Errorf("fold: %w", err) } + timeSpentFold += time.Since(start) } // Now unfold until we step on an empty cell for unfolding := hph.needUnfolding(hashedKey); unfolding > 0; unfolding = hph.needUnfolding(hashedKey) { + start := time.Now() if err := hph.unfold(hashedKey, unfolding); err != nil { return fmt.Errorf("unfold: %w", err) } + timeSpentUnf += time.Since(start) } if stateUpdate == nil { // Update the cell if len(plainKey) == hph.accountKeyLen { + a := time.Now() update, err = hph.ctx.Account(plainKey) if err != nil { return fmt.Errorf("GetAccount for key %x failed: %w", plainKey, err) } + timeSpentReadingAccount += time.Since(a) } else { + a := time.Now() update, err = hph.ctx.Storage(plainKey) if err != nil { return fmt.Errorf("GetStorage for key %x failed: %w", plainKey, err) } + timeSpentReadingStorage += time.Since(a) } } else { if update == nil { @@ -1587,6 +1609,8 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log if err != nil { return nil, fmt.Errorf("hash sort failed: %w", err) } + fmt.Println("totalTimeProcessing", time.Since(start), "timeSpentFold", timeSpentFold, "timeSpentUnf", timeSpentUnf, "timeSpentReadingCommitmentDB", state2.ReadFromDB, "timeSpentReadingCommitmentFiles", state2.ReadFromFiles, "timeSpentReplacingAccountAndStorage", state2.ReplacedKeys) + timeSpentFold, timeSpentUnf, state2.ReadFromDB, state2.ReadFromFiles, state2.ReplacedKeys = 0, 0, 0, 0, 0 // Folding everything up to the root for hph.activeRows > 0 { diff --git a/erigon-lib/state/domain_shared.go b/erigon-lib/state/domain_shared.go index 5a214bbfbb1..6b0f210cb93 100644 --- a/erigon-lib/state/domain_shared.go +++ b/erigon-lib/state/domain_shared.go @@ -432,7 +432,14 @@ func (sd *SharedDomains) SizeEstimate() uint64 { return uint64(sd.estSize) * 4 } +var ( + ReadFromDB time.Duration + ReadFromFiles time.Duration + ReplacedKeys time.Duration +) + func (sd *SharedDomains) LatestCommitment(prefix []byte) ([]byte, uint64, error) { + a := time.Now() if v, prevStep, ok := sd.get(kv.CommitmentDomain, prefix); ok { // sd cache values as is (without transformation) so safe to return return v, prevStep, nil @@ -442,6 +449,7 @@ func (sd *SharedDomains) LatestCommitment(prefix []byte) ([]byte, uint64, error) return nil, 0, fmt.Errorf("commitment prefix %x read error: %w", prefix, err) } if found { + ReadFromDB += time.Since(a) // db store values as is (without transformation) so safe to return return v, step, nil } @@ -454,6 +462,7 @@ func (sd *SharedDomains) LatestCommitment(prefix []byte) ([]byte, uint64, error) } if !sd.aggTx.a.commitmentValuesTransform || bytes.Equal(prefix, keyCommitmentState) { + ReadFromFiles += time.Since(a) return v, endTx / sd.aggTx.a.StepSize(), nil } @@ -462,6 +471,8 @@ func (sd *SharedDomains) LatestCommitment(prefix []byte) ([]byte, uint64, error) if err != nil { return nil, 0, err } + ReplacedKeys += time.Since(a) + return rv, endTx / sd.aggTx.a.StepSize(), nil } From 75f161fa2e0587a2e5c9d5dbecec6d5f7836d748 Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 20 Nov 2024 10:37:54 +0100 Subject: [PATCH 2/8] save --- eth/stagedsync/exec3.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 5be4d62c25d..664df929aff 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -509,6 +509,7 @@ Loop: skipPostEvaluation := false var usedGas uint64 var txTasks []*state.TxTask + start := time.Now() for txIndex := -1; txIndex <= len(txs); txIndex++ { // Do not oversend, wait for the result heap to go under certain size txTask := &state.TxTask{ @@ -606,7 +607,7 @@ Loop: } mxExecBlocks.Add(1) - + fmt.Println("block exec", time.Since(start)) if shouldGenerateChangesets { aggTx := executor.tx().(state2.HasAggTx).AggTx().(*state2.AggregatorRoTx) aggTx.RestrictSubsetFileDeletions(true) @@ -614,6 +615,7 @@ Loop: if _, err := executor.domains().ComputeCommitment(ctx, true, blockNum, execStage.LogPrefix()); err != nil { return err } + fmt.Println("commitment", time.Since(start)) ts += time.Since(start) aggTx.RestrictSubsetFileDeletions(false) executor.domains().SavePastChangesetAccumulator(b.Hash(), blockNum, changeset) From 23552da16c6d29387ebe93740a0c2ee8d112f396 Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 20 Nov 2024 10:40:05 +0100 Subject: [PATCH 3/8] save --- erigon-lib/commitment/hex_patricia_hashed.go | 5 ++--- erigon-lib/common/cli.go | 7 +++++++ erigon-lib/state/domain_shared.go | 12 +++--------- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/erigon-lib/commitment/hex_patricia_hashed.go b/erigon-lib/commitment/hex_patricia_hashed.go index 9282561756f..ea0c6ea2de8 100644 --- a/erigon-lib/commitment/hex_patricia_hashed.go +++ b/erigon-lib/commitment/hex_patricia_hashed.go @@ -44,7 +44,6 @@ import ( "github.com/erigontech/erigon-lib/common/hexutility" "github.com/erigontech/erigon-lib/common/length" "github.com/erigontech/erigon-lib/rlp" - state2 "github.com/erigontech/erigon-lib/state" ) // keccakState wraps sha3.state. In addition to the usual hash methods, it also supports @@ -1609,8 +1608,8 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log if err != nil { return nil, fmt.Errorf("hash sort failed: %w", err) } - fmt.Println("totalTimeProcessing", time.Since(start), "timeSpentFold", timeSpentFold, "timeSpentUnf", timeSpentUnf, "timeSpentReadingCommitmentDB", state2.ReadFromDB, "timeSpentReadingCommitmentFiles", state2.ReadFromFiles, "timeSpentReplacingAccountAndStorage", state2.ReplacedKeys) - timeSpentFold, timeSpentUnf, state2.ReadFromDB, state2.ReadFromFiles, state2.ReplacedKeys = 0, 0, 0, 0, 0 + fmt.Println("totalTimeProcessing", time.Since(start), "timeSpentFold", timeSpentFold, "timeSpentUnf", timeSpentUnf, "timeSpentReadingCommitmentDB", common.ReadFromDB, "timeSpentReadingCommitmentFiles", common.ReadFromFiles, "timeSpentReplacingAccountAndStorage", common.ReplacedKeys) + timeSpentFold, timeSpentUnf, common.ReadFromDB, common.ReadFromFiles, common.ReplacedKeys = 0, 0, 0, 0, 0 // Folding everything up to the root for hph.activeRows > 0 { diff --git a/erigon-lib/common/cli.go b/erigon-lib/common/cli.go index 21411d53b77..dba1e13f43a 100644 --- a/erigon-lib/common/cli.go +++ b/erigon-lib/common/cli.go @@ -22,10 +22,17 @@ import ( "os/signal" "strings" "syscall" + "time" "github.com/erigontech/erigon-lib/log/v3" ) +var ( + ReadFromDB time.Duration + ReadFromFiles time.Duration + ReplacedKeys time.Duration +) + func RootContext() (context.Context, context.CancelFunc) { ctx, cancel := context.WithCancel(context.Background()) go func() { diff --git a/erigon-lib/state/domain_shared.go b/erigon-lib/state/domain_shared.go index 6b0f210cb93..c6692be3815 100644 --- a/erigon-lib/state/domain_shared.go +++ b/erigon-lib/state/domain_shared.go @@ -432,12 +432,6 @@ func (sd *SharedDomains) SizeEstimate() uint64 { return uint64(sd.estSize) * 4 } -var ( - ReadFromDB time.Duration - ReadFromFiles time.Duration - ReplacedKeys time.Duration -) - func (sd *SharedDomains) LatestCommitment(prefix []byte) ([]byte, uint64, error) { a := time.Now() if v, prevStep, ok := sd.get(kv.CommitmentDomain, prefix); ok { @@ -449,7 +443,7 @@ func (sd *SharedDomains) LatestCommitment(prefix []byte) ([]byte, uint64, error) return nil, 0, fmt.Errorf("commitment prefix %x read error: %w", prefix, err) } if found { - ReadFromDB += time.Since(a) + common.ReadFromDB += time.Since(a) // db store values as is (without transformation) so safe to return return v, step, nil } @@ -462,7 +456,7 @@ func (sd *SharedDomains) LatestCommitment(prefix []byte) ([]byte, uint64, error) } if !sd.aggTx.a.commitmentValuesTransform || bytes.Equal(prefix, keyCommitmentState) { - ReadFromFiles += time.Since(a) + common.ReadFromFiles += time.Since(a) return v, endTx / sd.aggTx.a.StepSize(), nil } @@ -471,7 +465,7 @@ func (sd *SharedDomains) LatestCommitment(prefix []byte) ([]byte, uint64, error) if err != nil { return nil, 0, err } - ReplacedKeys += time.Since(a) + common.ReplacedKeys += time.Since(a) return rv, endTx / sd.aggTx.a.StepSize(), nil } From 80adba23784bf98dad0c530b32973a889d8aaa5d Mon Sep 17 00:00:00 2001 From: Giulio Date: Wed, 20 Nov 2024 10:54:56 +0100 Subject: [PATCH 4/8] save --- erigon-lib/commitment/hex_patricia_hashed.go | 4 ++-- erigon-lib/common/cli.go | 1 + erigon-lib/state/domain_shared.go | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/erigon-lib/commitment/hex_patricia_hashed.go b/erigon-lib/commitment/hex_patricia_hashed.go index ea0c6ea2de8..09308e04e58 100644 --- a/erigon-lib/commitment/hex_patricia_hashed.go +++ b/erigon-lib/commitment/hex_patricia_hashed.go @@ -1608,8 +1608,8 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log if err != nil { return nil, fmt.Errorf("hash sort failed: %w", err) } - fmt.Println("totalTimeProcessing", time.Since(start), "timeSpentFold", timeSpentFold, "timeSpentUnf", timeSpentUnf, "timeSpentReadingCommitmentDB", common.ReadFromDB, "timeSpentReadingCommitmentFiles", common.ReadFromFiles, "timeSpentReplacingAccountAndStorage", common.ReplacedKeys) - timeSpentFold, timeSpentUnf, common.ReadFromDB, common.ReadFromFiles, common.ReplacedKeys = 0, 0, 0, 0, 0 + fmt.Println("totalTimeProcessing", time.Since(start), "timeSpentFold", timeSpentFold, "timeSpentUnf", timeSpentUnf, "timeSpentReadingCommitmentDB", common.ReadFromDB, "timeSpentReadingCommitmentFiles", common.ReadFromFiles, "timeSpentReplacingAccountAndStorage", common.ReplacedKeys, "timeSpentReplacingAccountAndStorageOnly", common.ReplacedKeys2) + timeSpentFold, timeSpentUnf, common.ReadFromDB, common.ReadFromFiles, common.ReplacedKeys, common.ReplacedKeys2 = 0, 0, 0, 0, 0, 0 // Folding everything up to the root for hph.activeRows > 0 { diff --git a/erigon-lib/common/cli.go b/erigon-lib/common/cli.go index dba1e13f43a..187690d6a28 100644 --- a/erigon-lib/common/cli.go +++ b/erigon-lib/common/cli.go @@ -31,6 +31,7 @@ var ( ReadFromDB time.Duration ReadFromFiles time.Duration ReplacedKeys time.Duration + ReplacedKeys2 time.Duration ) func RootContext() (context.Context, context.CancelFunc) { diff --git a/erigon-lib/state/domain_shared.go b/erigon-lib/state/domain_shared.go index c6692be3815..c6b6ba441b9 100644 --- a/erigon-lib/state/domain_shared.go +++ b/erigon-lib/state/domain_shared.go @@ -460,12 +460,14 @@ func (sd *SharedDomains) LatestCommitment(prefix []byte) ([]byte, uint64, error) return v, endTx / sd.aggTx.a.StepSize(), nil } + xenoverse := time.Now() // replace shortened keys in the branch with full keys to allow HPH work seamlessly rv, err := sd.replaceShortenedKeysInBranch(prefix, commitment.BranchData(v), startTx, endTx) if err != nil { return nil, 0, err } common.ReplacedKeys += time.Since(a) + common.ReplacedKeys2 += time.Since(xenoverse) return rv, endTx / sd.aggTx.a.StepSize(), nil } From 9fe09225fe36e5be78d8232f8b81c91b2d0118b8 Mon Sep 17 00:00:00 2001 From: Alex Sharp Date: Wed, 20 Nov 2024 19:42:03 +0000 Subject: [PATCH 5/8] Add cumulative metrics, calculate percentages of total --- erigon-lib/commitment/commitment.go | 5 ++ erigon-lib/commitment/hex_patricia_hashed.go | 55 ++++++++++++++------ erigon-lib/common/cli.go | 8 --- erigon-lib/state/domain_shared.go | 53 +++++++++++++++++-- 4 files changed, 92 insertions(+), 29 deletions(-) diff --git a/erigon-lib/commitment/commitment.go b/erigon-lib/commitment/commitment.go index 2d7e6c316c5..00755facefd 100644 --- a/erigon-lib/commitment/commitment.go +++ b/erigon-lib/commitment/commitment.go @@ -25,6 +25,7 @@ import ( "math/bits" "sort" "strings" + "time" "github.com/holiman/uint256" @@ -113,6 +114,10 @@ type PatriciaContext interface { Account(plainKey []byte) (*Update, error) // fetch storage with given plain key Storage(plainKey []byte) (*Update, error) + // Reset performance counters, but not cumulative counters + ResetPerfCounters() + // Obtain performance counters + PerfCounters() map[string]time.Duration } type TrieVariant string diff --git a/erigon-lib/commitment/hex_patricia_hashed.go b/erigon-lib/commitment/hex_patricia_hashed.go index 09308e04e58..fb3052a8bc1 100644 --- a/erigon-lib/commitment/hex_patricia_hashed.go +++ b/erigon-lib/commitment/hex_patricia_hashed.go @@ -974,12 +974,10 @@ func (hph *HexPatriciaHashed) needUnfolding(hashedKey []byte) int { // unfoldBranchNode returns true if unfolding has been done func (hph *HexPatriciaHashed) unfoldBranchNode(row, depth int, deleted bool) (bool, error) { key := hexToCompact(hph.currentKey[:hph.currentKeyLen]) - timeSpentReadingBranchA := time.Now() branchData, fileEndTxNum, err := hph.ctx.Branch(key) if err != nil { return false, err } - timeSpentReadingBranch += time.Since(timeSpentReadingBranchA) hph.depthsToTxNum[depth] = fileEndTxNum if len(branchData) >= 2 { branchData = branchData[2:] // skip touch map and keep the rest @@ -1519,11 +1517,10 @@ func (hph *HexPatriciaHashed) RootHash() ([]byte, error) { } var ( - timeSpentFold time.Duration - timeSpentUnf time.Duration - timeSpentReadingAccount time.Duration - timeSpentReadingStorage time.Duration - timeSpentReadingBranch time.Duration + timeTotalCum time.Duration + timeSpentFold time.Duration + timeSpentUnf time.Duration + perfCountersCum map[string]time.Duration ) func printShit() { @@ -1563,7 +1560,8 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log if err := hph.fold(); err != nil { return fmt.Errorf("fold: %w", err) } - timeSpentFold += time.Since(start) + t := time.Since(start) + timeSpentFold += t } // Now unfold until we step on an empty cell for unfolding := hph.needUnfolding(hashedKey); unfolding > 0; unfolding = hph.needUnfolding(hashedKey) { @@ -1571,25 +1569,22 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log if err := hph.unfold(hashedKey, unfolding); err != nil { return fmt.Errorf("unfold: %w", err) } - timeSpentUnf += time.Since(start) + t := time.Since(start) + timeSpentUnf += t } if stateUpdate == nil { // Update the cell if len(plainKey) == hph.accountKeyLen { - a := time.Now() update, err = hph.ctx.Account(plainKey) if err != nil { return fmt.Errorf("GetAccount for key %x failed: %w", plainKey, err) } - timeSpentReadingAccount += time.Since(a) } else { - a := time.Now() update, err = hph.ctx.Storage(plainKey) if err != nil { return fmt.Errorf("GetStorage for key %x failed: %w", plainKey, err) } - timeSpentReadingStorage += time.Since(a) } } else { if update == nil { @@ -1608,8 +1603,6 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log if err != nil { return nil, fmt.Errorf("hash sort failed: %w", err) } - fmt.Println("totalTimeProcessing", time.Since(start), "timeSpentFold", timeSpentFold, "timeSpentUnf", timeSpentUnf, "timeSpentReadingCommitmentDB", common.ReadFromDB, "timeSpentReadingCommitmentFiles", common.ReadFromFiles, "timeSpentReplacingAccountAndStorage", common.ReplacedKeys, "timeSpentReplacingAccountAndStorageOnly", common.ReplacedKeys2) - timeSpentFold, timeSpentUnf, common.ReadFromDB, common.ReadFromFiles, common.ReplacedKeys, common.ReplacedKeys2 = 0, 0, 0, 0, 0, 0 // Folding everything up to the root for hph.activeRows > 0 { @@ -1617,6 +1610,37 @@ func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, log return nil, fmt.Errorf("final fold: %w", err) } } + total := time.Since(start) + timeTotalCum += total + perfCounters := hph.ctx.PerfCounters() + if perfCountersCum == nil { + perfCountersCum = map[string]time.Duration{} + } + for k, v := range perfCounters { + perfCountersCum[k+"_cum"] += v + } + perfCountersCum["fold_cum"] += timeSpentFold + perfCountersCum["unfold_cum"] += timeSpentUnf + fmt.Println( + "total", total, "total_cum", timeTotalCum, + "\nfold", timeSpentFold, + "unfold", timeSpentUnf, + "\n", perfCounters, + "\n", perfCountersCum) + timeSpentFold, timeSpentUnf = 0, 0 + hph.ctx.ResetPerfCounters() + cum_keys := make([]string, 0, len(perfCountersCum)) + for k := range perfCountersCum { + cum_keys = append(cum_keys, k) + } + sort.Strings(cum_keys) + perfCountersPercents := map[string]float64{} + for k, v := range perfCountersCum { + perfCountersPercents[k] = 100.0 * float64(v) / float64(timeTotalCum) + } + for _, k := range cum_keys { + fmt.Printf("%6.2f%% %s\n", perfCountersPercents[k], k) + } rootHash, err = hph.RootHash() if err != nil { @@ -1982,7 +2006,6 @@ func (hph *HexPatriciaHashed) SetState(buf []byte) error { if hph.ctx == nil { panic("nil ctx") } - update, err := hph.ctx.Account(hph.root.accountAddr[:hph.root.accountAddrLen]) if err != nil { return err diff --git a/erigon-lib/common/cli.go b/erigon-lib/common/cli.go index 187690d6a28..21411d53b77 100644 --- a/erigon-lib/common/cli.go +++ b/erigon-lib/common/cli.go @@ -22,18 +22,10 @@ import ( "os/signal" "strings" "syscall" - "time" "github.com/erigontech/erigon-lib/log/v3" ) -var ( - ReadFromDB time.Duration - ReadFromFiles time.Duration - ReplacedKeys time.Duration - ReplacedKeys2 time.Duration -) - func RootContext() (context.Context, context.CancelFunc) { ctx, cancel := context.WithCancel(context.Background()) go func() { diff --git a/erigon-lib/state/domain_shared.go b/erigon-lib/state/domain_shared.go index c6b6ba441b9..c2b136ed852 100644 --- a/erigon-lib/state/domain_shared.go +++ b/erigon-lib/state/domain_shared.go @@ -434,16 +434,24 @@ func (sd *SharedDomains) SizeEstimate() uint64 { func (sd *SharedDomains) LatestCommitment(prefix []byte) ([]byte, uint64, error) { a := time.Now() + var b time.Time + var db_time, files_time time.Duration + defer func() { + sd.sdCtx.commReadTime += time.Since(a) + sd.sdCtx.commDbTime += db_time + sd.sdCtx.commFilesTime += files_time + }() if v, prevStep, ok := sd.get(kv.CommitmentDomain, prefix); ok { // sd cache values as is (without transformation) so safe to return return v, prevStep, nil } v, step, found, err := sd.aggTx.d[kv.CommitmentDomain].getLatestFromDb(prefix, sd.roTx) + db_time = time.Since(a) if err != nil { return nil, 0, fmt.Errorf("commitment prefix %x read error: %w", prefix, err) } + b = time.Now() if found { - common.ReadFromDB += time.Since(a) // db store values as is (without transformation) so safe to return return v, step, nil } @@ -451,29 +459,30 @@ func (sd *SharedDomains) LatestCommitment(prefix []byte) ([]byte, uint64, error) // getFromFiles doesn't provide same semantics as getLatestFromDB - it returns start/end tx // of file where the value is stored (not exact step when kv has been set) v, _, startTx, endTx, err := sd.aggTx.d[kv.CommitmentDomain].getFromFiles(prefix, 0) + files_time = time.Since(b) if err != nil { return nil, 0, fmt.Errorf("commitment prefix %x read error: %w", prefix, err) } if !sd.aggTx.a.commitmentValuesTransform || bytes.Equal(prefix, keyCommitmentState) { - common.ReadFromFiles += time.Since(a) return v, endTx / sd.aggTx.a.StepSize(), nil } - xenoverse := time.Now() // replace shortened keys in the branch with full keys to allow HPH work seamlessly rv, err := sd.replaceShortenedKeysInBranch(prefix, commitment.BranchData(v), startTx, endTx) if err != nil { return nil, 0, err } - common.ReplacedKeys += time.Since(a) - common.ReplacedKeys2 += time.Since(xenoverse) return rv, endTx / sd.aggTx.a.StepSize(), nil } // replaceShortenedKeysInBranch replaces shortened keys in the branch with full keys func (sd *SharedDomains) replaceShortenedKeysInBranch(prefix []byte, branch commitment.BranchData, fStartTxNum uint64, fEndTxNum uint64) (commitment.BranchData, error) { + a := time.Now() + defer func() { + sd.sdCtx.replaceTime += time.Since(a) + }() if !sd.aggTx.d[kv.CommitmentDomain].d.replaceKeysInValues && sd.aggTx.a.commitmentValuesTransform { panic("domain.replaceKeysInValues is disabled, but agg.commitmentValuesTransform is enabled") } @@ -1117,6 +1126,12 @@ type SharedDomainsCommitmentContext struct { justRestored atomic.Bool limitReadAsOfTxNum uint64 + accountReadTime time.Duration + storageReadTime time.Duration + commReadTime time.Duration + replaceTime time.Duration + commDbTime time.Duration + commFilesTime time.Duration } func (sdc *SharedDomainsCommitmentContext) SetLimitReadAsOfTxNum(txNum uint64) { @@ -1185,6 +1200,10 @@ func (sdc *SharedDomainsCommitmentContext) PutBranch(prefix []byte, data []byte, } func (sdc *SharedDomainsCommitmentContext) Account(plainKey []byte) (u *commitment.Update, err error) { + a := time.Now() + defer func() { + sdc.accountReadTime += time.Since(a) + }() var encAccount []byte if sdc.limitReadAsOfTxNum == 0 { encAccount, _, err = sdc.sharedDomains.GetLatest(kv.AccountsDomain, plainKey, nil) @@ -1246,6 +1265,10 @@ func (sdc *SharedDomainsCommitmentContext) Account(plainKey []byte) (u *commitme } func (sdc *SharedDomainsCommitmentContext) Storage(plainKey []byte) (u *commitment.Update, err error) { + a := time.Now() + defer func() { + sdc.storageReadTime += time.Since(a) + }() // Look in the summary table first var enc []byte if sdc.limitReadAsOfTxNum == 0 { @@ -1341,6 +1364,26 @@ func (sdc *SharedDomainsCommitmentContext) ComputeCommitment(ctx context.Context return rootHash, err } +func (sdc *SharedDomainsCommitmentContext) ResetPerfCounters() { + sdc.accountReadTime = 0 + sdc.storageReadTime = 0 + sdc.commReadTime = 0 + sdc.replaceTime = 0 + sdc.commDbTime = 0 + sdc.commFilesTime = 0 +} + +func (sdc *SharedDomainsCommitmentContext) PerfCounters() map[string]time.Duration { + m := map[string]time.Duration{} + m["read_account"] = sdc.accountReadTime + m["read_storage"] = sdc.storageReadTime + m["read_comm"] = sdc.commReadTime + m["read_comm_db"] = sdc.commDbTime + m["read_comm_files"] = sdc.commFilesTime + m["read_comm_replace"] = sdc.replaceTime + return m +} + func (sdc *SharedDomainsCommitmentContext) storeCommitmentState(blockNum uint64, rootHash []byte) error { if sdc.sharedDomains.aggTx == nil { return fmt.Errorf("store commitment state: AggregatorContext is not initialized") From d1848ca1472404c284d914f4afa7f53292197bf9 Mon Sep 17 00:00:00 2001 From: Alex Sharp Date: Wed, 20 Nov 2024 21:17:49 +0000 Subject: [PATCH 6/8] fix lint --- erigon-lib/commitment/patricia_state_mock_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/erigon-lib/commitment/patricia_state_mock_test.go b/erigon-lib/commitment/patricia_state_mock_test.go index b620de4a0a5..e4c58f7eada 100644 --- a/erigon-lib/commitment/patricia_state_mock_test.go +++ b/erigon-lib/commitment/patricia_state_mock_test.go @@ -23,6 +23,7 @@ import ( "fmt" "slices" "testing" + "time" "github.com/holiman/uint256" "golang.org/x/crypto/sha3" @@ -129,6 +130,10 @@ func (ms *MockState) Storage(plainKey []byte) (*Update, error) { return &ex, nil } +func (ms *MockState) ResetPerfCounters() {} + +func (ms *MockState) PerfCounters() map[string]time.Duration + func (ms *MockState) applyPlainUpdates(plainKeys [][]byte, updates []Update) error { for i, key := range plainKeys { update := updates[i] From ef0b7b9822d212066c665276deb44ccf5b5a0288 Mon Sep 17 00:00:00 2001 From: Alex Sharp Date: Wed, 20 Nov 2024 21:28:44 +0000 Subject: [PATCH 7/8] Fix lint --- erigon-lib/commitment/patricia_state_mock_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/erigon-lib/commitment/patricia_state_mock_test.go b/erigon-lib/commitment/patricia_state_mock_test.go index e4c58f7eada..81a23fe3556 100644 --- a/erigon-lib/commitment/patricia_state_mock_test.go +++ b/erigon-lib/commitment/patricia_state_mock_test.go @@ -131,8 +131,9 @@ func (ms *MockState) Storage(plainKey []byte) (*Update, error) { } func (ms *MockState) ResetPerfCounters() {} - -func (ms *MockState) PerfCounters() map[string]time.Duration +func (ms *MockState) PerfCounters() map[string]time.Duration { + return map[string]time.Duration{} +} func (ms *MockState) applyPlainUpdates(plainKeys [][]byte, updates []Update) error { for i, key := range plainKeys { From 92c55ee355038bd0ecc8e0965eef48615441cfc1 Mon Sep 17 00:00:00 2001 From: Alex Sharp Date: Wed, 20 Nov 2024 21:32:30 +0000 Subject: [PATCH 8/8] Fix lint --- erigon-lib/commitment/hex_patricia_hashed.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/erigon-lib/commitment/hex_patricia_hashed.go b/erigon-lib/commitment/hex_patricia_hashed.go index fb3052a8bc1..350cad28d95 100644 --- a/erigon-lib/commitment/hex_patricia_hashed.go +++ b/erigon-lib/commitment/hex_patricia_hashed.go @@ -1523,10 +1523,6 @@ var ( perfCountersCum map[string]time.Duration ) -func printShit() { - -} - func (hph *HexPatriciaHashed) Process(ctx context.Context, updates *Updates, logPrefix string) (rootHash []byte, err error) { var ( m runtime.MemStats