Skip to content

Commit

Permalink
polygon/heimdall: fix producers api on mainnet (#12030)
Browse files Browse the repository at this point in the history 
Running Astrid tests on bor-mainnet produced an error:
```
[DBUG] [09-12|11:33:58.007] [sync] applyNewBlockOnTip: couldn't connect a header to the local chain tip, ignoring err="canonicalChainBuilder.Connect: invalid header error Block 61732633 was created too soon. Signer turn-ness number is 18\n"
```

Node just gets stuck here and can't proceed forward.

This is caused by a bug in the Producers API that didn't surface on
Amoy, only on Mainnet, since the producers change more frequently there.

This PR adds tests using mainnet span data that manage to reproduce the
issue. Note instead of setting up tests for block 61,732,633 I only
setup tests up to block 15,000,000 since the error can be first
reproduced around block 14,329,855 and tests up to block 15 mil are much
quicker (only 2-3 secs).
  • Loading branch information
@taratorio
taratorio authored Sep 20, 2024
1 parent d64dd8f commit 787e10f
Show file tree
Hide file tree
Showing 2,372 changed files with 961,001 additions and 134 deletions.
345 changes: 216 additions & 129 deletions polygon/heimdall/service_test.go
View file

Large diffs are not rendered by default.

25 changes: 20 additions & 5 deletions polygon/heimdall/span_block_producers_tracker.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"errors"
"fmt"
"sync/atomic"
"time"

"github.com/erigontech/erigon-lib/log/v3"
"github.com/erigontech/erigon/polygon/bor/valset"
Expand Down Expand Up @@ -112,7 +113,9 @@ func (t *spanBlockProducersTracker) ObserveSpan(ctx context.Context, newSpan *Sp
SpanId: newSpan.Id,
StartBlock: newSpan.StartBlock,
EndBlock: newSpan.EndBlock,
Producers: valset.NewValidatorSet(newSpan.Producers()),
// https://github.com/maticnetwork/genesis-contracts/blob/master/contracts/BorValidatorSet.template#L82-L89
// initial producers == initial validators
Producers: valset.NewValidatorSet(newSpan.ValidatorSet.Validators),
}
err = t.store.PutEntity(ctx, uint64(newProducerSelection.SpanId), newProducerSelection)
if err != nil {
Expand Down Expand Up @@ -145,9 +148,11 @@ func (t *spanBlockProducersTracker) ObserveSpan(ctx context.Context, newSpan *Sp
spanStartSprintNum := t.calculateSprintNumber(lastProducerSelection.StartBlock)
spanEndSprintNum := t.calculateSprintNumber(lastProducerSelection.EndBlock)
increments := int(spanEndSprintNum - spanStartSprintNum)
if increments > 0 {
producers.IncrementProposerPriority(increments)
for i := 0; i < increments; i++ {
producers = valset.GetUpdatedValidatorSet(producers, producers.Validators, t.logger)
producers.IncrementProposerPriority(1)
}

newProducers := valset.GetUpdatedValidatorSet(producers, newSpan.Producers(), t.logger)
newProducers.IncrementProposerPriority(1)
newProducerSelection := &SpanBlockProducerSelection{
Expand All @@ -166,6 +171,7 @@ func (t *spanBlockProducersTracker) ObserveSpan(ctx context.Context, newSpan *Sp
}

func (t *spanBlockProducersTracker) Producers(ctx context.Context, blockNum uint64) (*valset.ValidatorSet, error) {
startTime := time.Now()
spanId := SpanIdAt(blockNum)
producerSelection, ok, err := t.store.Entity(ctx, uint64(spanId))
if err != nil {
Expand All @@ -185,8 +191,17 @@ func (t *spanBlockProducersTracker) Producers(ctx context.Context, blockNum uint
spanStartSprintNum := t.calculateSprintNumber(producerSelection.StartBlock)
currentSprintNum := t.calculateSprintNumber(blockNum)
increments := int(currentSprintNum - spanStartSprintNum)
if increments > 0 {
producers.IncrementProposerPriority(increments)
for i := 0; i < increments; i++ {
producers = valset.GetUpdatedValidatorSet(producers, producers.Validators, t.logger)
producers.IncrementProposerPriority(1)
}

t.logger.Debug(
heimdallLogPrefix("producers api timing"),
"blockNum", blockNum,
"time", time.Since(startTime),
"increments", increments,
)

return producers, nil
}
9 changes: 9 additions & 0 deletions polygon/heimdall/testdata/mainnet/checkpoints/checkpoint_1.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
{
"id": 1,
"proposer": "0x0375b2fc7140977c9c76d45421564e354ed42277",
"start_block": 0,
"end_block": 137,
"root_hash": "0xa665048b94b49056fe4b03e776a8db473cba50e8d6e8b91c8d1f580e47a5c6b0",
"bor_chain_id": "137",
"timestamp": 1590856507
}
38 changes: 38 additions & 0 deletions polygon/heimdall/testdata/mainnet/getSnapshotProposerSequence/blockNum_1.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
{
"jsonrpc": "2.0",
"id": 1,
"result": {
"Signers": [
{
"Signer": "0x0375b2fc7140977c9c76d45421564e354ed42277",
"Difficulty": 7
},
{
"Signer": "0x42eefcda06ead475cde3731b8eb138e88cd0bac3",
"Difficulty": 6
},
{
"Signer": "0x5973918275c01f50555d44e92c9d9b353cadad54",
"Difficulty": 5
},
{
"Signer": "0x7fcd58c2d53d980b247f1612fdba93e9a76193e6",
"Difficulty": 4
},
{
"Signer": "0xb702f1c9154ac9c08da247a8e30ee6f2f3373f41",
"Difficulty": 3
},
{
"Signer": "0xb8bb158b93c94ed35c1970d610d1e2b34e26652c",
"Difficulty": 2
},
{
"Signer": "0xf84c74dea96df0ec22e11e7c33996c73fcc2d822",
"Difficulty": 1
}
],
"Diff": 7,
"Author": "0x0375b2fc7140977c9c76d45421564e354ed42277"
}
}
66 changes: 66 additions & 0 deletions polygon/heimdall/testdata/mainnet/getSnapshotProposerSequence/blockNum_10000000.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
{
"jsonrpc": "2.0",
"id": 1,
"result": {
"Signers": [
{
"Signer": "0xc6869257205e20c2a43cb31345db534aecb49f6e",
"Difficulty": 14
},
{
"Signer": "0xe77bbfd8ed65720f187efdd109e38d75eaca7385",
"Difficulty": 13
},
{
"Signer": "0xf84c74dea96df0ec22e11e7c33996c73fcc2d822",
"Difficulty": 12
},
{
"Signer": "0x0375b2fc7140977c9c76d45421564e354ed42277",
"Difficulty": 11
},
{
"Signer": "0x1ca971963bdb4ba2bf337c90660674acff5beb3f",
"Difficulty": 10
},
{
"Signer": "0x4f856f79f54592a48c8a1a1fafa1b0a3ac053f99",
"Difficulty": 9
},
{
"Signer": "0x5973918275c01f50555d44e92c9d9b353cadad54",
"Difficulty": 8
},
{
"Signer": "0x7fcd58c2d53d980b247f1612fdba93e9a76193e6",
"Difficulty": 7
},
{
"Signer": "0x8a08cfd1cc3012576d6e2d3937b0d5f248701f24",
"Difficulty": 6
},
{
"Signer": "0x8cb120478e9503760656c1fcac9c1539158bdb55",
"Difficulty": 5
},
{
"Signer": "0xa4b56cab3d878f76364a0657aa4954f4cbd0c67e",
"Difficulty": 4
},
{
"Signer": "0xaa139169c7e2f444c959499544476e5d64252d3d",
"Difficulty": 3
},
{
"Signer": "0xb702f1c9154ac9c08da247a8e30ee6f2f3373f41",
"Difficulty": 2
},
{
"Signer": "0xb8bb158b93c94ed35c1970d610d1e2b34e26652c",
"Difficulty": 1
}
],
"Diff": 14,
"Author": "0xc6869257205e20c2a43cb31345db534aecb49f6e"
}
}
70 changes: 70 additions & 0 deletions polygon/heimdall/testdata/mainnet/getSnapshotProposerSequence/blockNum_12000000.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
{
"jsonrpc": "2.0",
"id": 1,
"result": {
"Signers": [
{
"Signer": "0x7fcd58c2d53d980b247f1612fdba93e9a76193e6",
"Difficulty": 15
},
{
"Signer": "0x8a08cfd1cc3012576d6e2d3937b0d5f248701f24",
"Difficulty": 14
},
{
"Signer": "0xa5d5a7c2ebd2a381f7e958754c0d6a2d469b131b",
"Difficulty": 13
},
{
"Signer": "0xb8bb158b93c94ed35c1970d610d1e2b34e26652c",
"Difficulty": 12
},
{
"Signer": "0xc6869257205e20c2a43cb31345db534aecb49f6e",
"Difficulty": 11
},
{
"Signer": "0xe05ae0e76f582817c9e31d9c1a5c02287a31d689",
"Difficulty": 10
},
{
"Signer": "0xe4cd4c302befddf3d544301369ae3ed1481652fd",
"Difficulty": 9
},
{
"Signer": "0xe77bbfd8ed65720f187efdd109e38d75eaca7385",
"Difficulty": 8
},
{
"Signer": "0xf84c74dea96df0ec22e11e7c33996c73fcc2d822",
"Difficulty": 7
},
{
"Signer": "0x0375b2fc7140977c9c76d45421564e354ed42277",
"Difficulty": 6
},
{
"Signer": "0x127685d6dd6683085da4b6a041efcef1681e5c9c",
"Difficulty": 5
},
{
"Signer": "0x1ca971963bdb4ba2bf337c90660674acff5beb3f",
"Difficulty": 4
},
{
"Signer": "0x42eefcda06ead475cde3731b8eb138e88cd0bac3",
"Difficulty": 3
},
{
"Signer": "0x4f856f79f54592a48c8a1a1fafa1b0a3ac053f99",
"Difficulty": 2
},
{
"Signer": "0x7b5000af8ab69fd59eb0d4f5762bff57c9c04385",
"Difficulty": 1
}
],
"Diff": 15,
"Author": "0x7fcd58c2d53d980b247f1612fdba93e9a76193e6"
}
}
70 changes: 70 additions & 0 deletions polygon/heimdall/testdata/mainnet/getSnapshotProposerSequence/blockNum_13000000.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
{
"jsonrpc": "2.0",
"id": 1,
"result": {
"Signers": [
{
"Signer": "0xe77bbfd8ed65720f187efdd109e38d75eaca7385",
"Difficulty": 15
},
{
"Signer": "0x0375b2fc7140977c9c76d45421564e354ed42277",
"Difficulty": 14
},
{
"Signer": "0x055bd801ca712b4ddf67db8bc23fb6c8510d52b9",
"Difficulty": 13
},
{
"Signer": "0x1ca971963bdb4ba2bf337c90660674acff5beb3f",
"Difficulty": 12
},
{
"Signer": "0x42eefcda06ead475cde3731b8eb138e88cd0bac3",
"Difficulty": 11
},
{
"Signer": "0x4f856f79f54592a48c8a1a1fafa1b0a3ac053f99",
"Difficulty": 10
},
{
"Signer": "0x5973918275c01f50555d44e92c9d9b353cadad54",
"Difficulty": 9
},
{
"Signer": "0x6776786c6590dcd66b2018db5425749f6334ae69",
"Difficulty": 8
},
{
"Signer": "0x72f93a2740e00112d5f2cef404c0aa16fae21fa4",
"Difficulty": 7
},
{
"Signer": "0x7b5000af8ab69fd59eb0d4f5762bff57c9c04385",
"Difficulty": 6
},
{
"Signer": "0x7c7379531b2aee82e4ca06d4175d13b9cbeafd49",
"Difficulty": 5
},
{
"Signer": "0x7fcd58c2d53d980b247f1612fdba93e9a76193e6",
"Difficulty": 4
},
{
"Signer": "0x8e9700392f9246a6c5b32ee3ecef586f156ed683",
"Difficulty": 3
},
{
"Signer": "0x99073526dae19bd07046cf35fa18ab61de846623",
"Difficulty": 2
},
{
"Signer": "0xc6869257205e20c2a43cb31345db534aecb49f6e",
"Difficulty": 1
}
],
"Diff": 15,
"Author": "0xe77bbfd8ed65720f187efdd109e38d75eaca7385"
}
}
78 changes: 78 additions & 0 deletions polygon/heimdall/testdata/mainnet/getSnapshotProposerSequence/blockNum_14000000.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
{
"jsonrpc": "2.0",
"id": 1,
"result": {
"Signers": [
{
"Signer": "0x7b5000af8ab69fd59eb0d4f5762bff57c9c04385",
"Difficulty": 17
},
{
"Signer": "0x7c7379531b2aee82e4ca06d4175d13b9cbeafd49",
"Difficulty": 16
},
{
"Signer": "0x7fcd58c2d53d980b247f1612fdba93e9a76193e6",
"Difficulty": 15
},
{
"Signer": "0xbdbd4347b082d9d6bdf2da4555a37ce52a2e2120",
"Difficulty": 14
},
{
"Signer": "0xc6869257205e20c2a43cb31345db534aecb49f6e",
"Difficulty": 13
},
{
"Signer": "0xddb833b9e0e3f2f521480e7bcb3e676e0737047d",
"Difficulty": 12
},
{
"Signer": "0xe4cd4c302befddf3d544301369ae3ed1481652fd",
"Difficulty": 11
},
{
"Signer": "0xe77bbfd8ed65720f187efdd109e38d75eaca7385",
"Difficulty": 10
},
{
"Signer": "0xf84c74dea96df0ec22e11e7c33996c73fcc2d822",
"Difficulty": 9
},
{
"Signer": "0x127685d6dd6683085da4b6a041efcef1681e5c9c",
"Difficulty": 8
},
{
"Signer": "0x1ca971963bdb4ba2bf337c90660674acff5beb3f",
"Difficulty": 7
},
{
"Signer": "0x1d25c827abd466387bda00b429fe728627d6eee6",
"Difficulty": 6
},
{
"Signer": "0x30dd252c7c150f26a3a06e4eada9e706db3fa58c",
"Difficulty": 5
},
{
"Signer": "0x448aa1665fe1fae6d1a00a9209ea62d7dcd81a4b",
"Difficulty": 4
},
{
"Signer": "0x46a3a41bd932244dd08186e4c19f1a7e48cbcdf4",
"Difficulty": 3
},
{
"Signer": "0x4f856f79f54592a48c8a1a1fafa1b0a3ac053f99",
"Difficulty": 2
},
{
"Signer": "0x5fe93ddf4490a02257bef079f2498650c97c44de",
"Difficulty": 1
}
],
"Diff": 17,
"Author": "0x7b5000af8ab69fd59eb0d4f5762bff57c9c04385"
}
}
Loading

0 comments on commit 787e10f

Please sign in to comment.