From ce987c4a64d392af1b6c33d4ec010cda4e10a42c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 27 Jan 2023 09:12:39 +0100 Subject: [PATCH 001/117] Delete `is_merge_transition_block` check from Capella --- specs/capella/fork-choice.md | 55 ++++++++++++++++++++++++++++++++++++ specs/eip4844/fork-choice.md | 4 --- 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/specs/capella/fork-choice.md b/specs/capella/fork-choice.md index 0e0a393c34..c08be1c3f6 100644 --- a/specs/capella/fork-choice.md +++ b/specs/capella/fork-choice.md @@ -14,6 +14,8 @@ - [`notify_forkchoice_updated`](#notify_forkchoice_updated) - [Helpers](#helpers) - [Extended `PayloadAttributes`](#extended-payloadattributes) +- [Updated fork-choice handlers](#updated-fork-choice-handlers) + - [`on_block`](#on_block) @@ -60,3 +62,56 @@ class PayloadAttributes(object): suggested_fee_recipient: ExecutionAddress withdrawals: Sequence[Withdrawal] # [New in Capella] ``` + +## Updated fork-choice handlers + +### `on_block` + +*Note*: The only modification is the deletion of the verification of merge transition block conditions. + +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + """ + Run ``on_block`` upon receiving a new block. + """ + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + pre_state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root + + # Check the block is valid and compute the post-state + state = pre_state.copy() + state_transition(state, signed_block, True) + + # Add new block to the store + store.blocks[hash_tree_root(block)] = block + # Add new state for this block to the store + store.block_states[hash_tree_root(block)] = state + + # Add proposer score boost if the block is timely + time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT + is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT + if get_current_slot(store) == block.slot and is_before_attesting_interval: + store.proposer_boost_root = hash_tree_root(block) + + # Update justified checkpoint + if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: + if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch: + store.best_justified_checkpoint = state.current_justified_checkpoint + if should_update_justified_checkpoint(store, state.current_justified_checkpoint): + store.justified_checkpoint = state.current_justified_checkpoint + + # Update finalized checkpoint + if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: + store.finalized_checkpoint = state.finalized_checkpoint + store.justified_checkpoint = state.current_justified_checkpoint +``` diff --git a/specs/eip4844/fork-choice.md b/specs/eip4844/fork-choice.md index 8dea28dedc..865443e04e 100644 --- a/specs/eip4844/fork-choice.md +++ b/specs/eip4844/fork-choice.md @@ -108,10 +108,6 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: state = pre_state.copy() state_transition(state, signed_block, True) - # Check the merge transition - if is_merge_transition_block(pre_state, block.body): - validate_merge_block(block) - # Add new block to the store store.blocks[hash_tree_root(block)] = block # Add new state for this block to the store From d0839dfec4e9753183e65d3754d891b015221b70 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 Mar 2023 20:58:43 +0800 Subject: [PATCH 002/117] Add EIP-6110 to the pytest scope --- configs/mainnet.yaml | 5 +++-- configs/minimal.yaml | 3 +++ presets/mainnet/eip6110.yaml | 6 ++++++ presets/minimal/eip6110.yaml | 6 ++++++ tests/core/pyspec/eth2spec/test/context.py | 6 +++++- .../core/pyspec/eth2spec/test/helpers/constants.py | 13 +++++++------ tests/core/pyspec/eth2spec/test/helpers/forks.py | 7 +++++++ 7 files changed, 37 insertions(+), 9 deletions(-) create mode 100644 presets/mainnet/eip6110.yaml create mode 100644 presets/minimal/eip6110.yaml diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index f204d1746e..e62108b0aa 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -50,8 +50,9 @@ CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC # Deneb DENEB_FORK_VERSION: 0x04000000 DENEB_FORK_EPOCH: 18446744073709551615 - - +# EIP6110 +EIP6110_FORK_VERSION: 0x04000001 +EIP6110_FORK_EPOCH: 18446744073709551615 # Time parameters diff --git a/configs/minimal.yaml b/configs/minimal.yaml index abecb18813..271a0c14ae 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -49,6 +49,9 @@ CAPELLA_FORK_EPOCH: 18446744073709551615 # DENEB DENEB_FORK_VERSION: 0x04000001 DENEB_FORK_EPOCH: 18446744073709551615 +# EIP6110 +EIP6110_FORK_VERSION: 0x04000001 +EIP6110_FORK_EPOCH: 18446744073709551615 # Time parameters diff --git a/presets/mainnet/eip6110.yaml b/presets/mainnet/eip6110.yaml new file mode 100644 index 0000000000..16bf787d0c --- /dev/null +++ b/presets/mainnet/eip6110.yaml @@ -0,0 +1,6 @@ +# Mainnet preset - EIP6110 + +# Execution +# --------------------------------------------------------------- +# 2**13 (= 8192) receipts +MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 8192 diff --git a/presets/minimal/eip6110.yaml b/presets/minimal/eip6110.yaml new file mode 100644 index 0000000000..7486aa16e7 --- /dev/null +++ b/presets/minimal/eip6110.yaml @@ -0,0 +1,6 @@ +# Minimal preset - EIP6110 + +# Execution +# --------------------------------------------------------------- +# [customized] +MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 4 diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 38e7f0b715..b859bf27c6 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -8,11 +8,13 @@ from eth2spec.bellatrix import mainnet as spec_bellatrix_mainnet, minimal as spec_bellatrix_minimal from eth2spec.capella import mainnet as spec_capella_mainnet, minimal as spec_capella_minimal from eth2spec.deneb import mainnet as spec_deneb_mainnet, minimal as spec_deneb_minimal +from eth2spec.eip6110 import mainnet as spec_eip6110_mainnet, minimal as spec_eip6110_minimal from eth2spec.utils import bls from .exceptions import SkippedTest from .helpers.constants import ( PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, + EIP6110, MINIMAL, MAINNET, ALL_PHASES, ALL_FORK_UPGRADES, @@ -79,13 +81,15 @@ class ForkMeta: BELLATRIX: spec_bellatrix_minimal, CAPELLA: spec_capella_minimal, DENEB: spec_deneb_minimal, + EIP6110: spec_eip6110_minimal, }, MAINNET: { PHASE0: spec_phase0_mainnet, ALTAIR: spec_altair_mainnet, BELLATRIX: spec_bellatrix_mainnet, CAPELLA: spec_capella_mainnet, - DENEB: spec_deneb_mainnet + DENEB: spec_deneb_mainnet, + EIP6110: spec_eip6110_mainnet, }, } diff --git a/tests/core/pyspec/eth2spec/test/helpers/constants.py b/tests/core/pyspec/eth2spec/test/helpers/constants.py index 0d31adb431..83e7e40dbb 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/constants.py +++ b/tests/core/pyspec/eth2spec/test/helpers/constants.py @@ -9,24 +9,25 @@ ALTAIR = SpecForkName('altair') BELLATRIX = SpecForkName('bellatrix') CAPELLA = SpecForkName('capella') +DENEB = SpecForkName('deneb') # Experimental phases (not included in default "ALL_PHASES"): SHARDING = SpecForkName('sharding') CUSTODY_GAME = SpecForkName('custody_game') DAS = SpecForkName('das') -DENEB = SpecForkName('deneb') +EIP6110 = SpecForkName('eip6110') # The forks that pytest can run with. ALL_PHASES = ( # Formal forks - PHASE0, ALTAIR, BELLATRIX, CAPELLA, + PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, # Experimental patches - DENEB, + EIP6110, ) # The forks that output to the test vectors. -TESTGEN_FORKS = (PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB) +TESTGEN_FORKS = (PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110) -# TODO: no DENEB fork tests now. Should add when we figure out the content of Capella. +# TODO: no EIP6110 fork tests now. ALL_FORK_UPGRADES = { # pre_fork_name: post_fork_name PHASE0: ALTAIR, @@ -41,7 +42,7 @@ if key not in [PHASE0, ALTAIR]} AFTER_CAPELLA_PRE_POST_FORKS = AFTER_CAPELLA_UPGRADES.items() AFTER_DENEB_UPGRADES = {key: value for key, value in ALL_FORK_UPGRADES.items() - if key not in [PHASE0, ALTAIR, BELLATRIX]} + if key not in [PHASE0, ALTAIR, BELLATRIX, EIP6110]} AFTER_DENEB_PRE_POST_FORKS = AFTER_DENEB_UPGRADES.items() # diff --git a/tests/core/pyspec/eth2spec/test/helpers/forks.py b/tests/core/pyspec/eth2spec/test/helpers/forks.py index be3103e67f..e6320cc9b3 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/forks.py +++ b/tests/core/pyspec/eth2spec/test/helpers/forks.py @@ -1,9 +1,12 @@ from .constants import ( PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, + EIP6110, ) def is_post_fork(a, b): + if a == EIP6110: + return b in [PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP6110] if a == DENEB: return b in [PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB] if a == CAPELLA: @@ -31,3 +34,7 @@ def is_post_capella(spec): def is_post_deneb(spec): return is_post_fork(spec.fork, DENEB) + + +def is_post_eip6110(spec): + return is_post_fork(spec.fork, EIP6110) From 890f574c126f79315e6745eb9bbe72ecd3a9f3a4 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 27 Mar 2023 21:00:40 +0800 Subject: [PATCH 003/117] Add EIP-6110 to CI scope --- .circleci/config.yml | 16 ++++++++++++++++ .github/workflows/run-tests.yml | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 665207bdd0..1d5b098111 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -155,6 +155,19 @@ jobs: command: make citest fork=deneb - store_test_results: path: tests/core/pyspec/test-reports + test-eip6110: + docker: + - image: circleci/python:3.8 + working_directory: ~/specs-repo + steps: + - restore_cache: + key: v3-specs-repo-{{ .Branch }}-{{ .Revision }} + - restore_pyspec_cached_venv + - run: + name: Run py-tests + command: make citest fork=eip6110 + - store_test_results: + path: tests/core/pyspec/test-reports table_of_contents: docker: - image: circleci/node:10.16.3 @@ -275,6 +288,9 @@ workflows: - test-deneb: requires: - install_pyspec_test + - test-eip6110: + requires: + - install_pyspec_test - table_of_contents - codespell - lint: diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml index 926c3fbbf9..41a80ab925 100644 --- a/.github/workflows/run-tests.yml +++ b/.github/workflows/run-tests.yml @@ -83,7 +83,7 @@ jobs: needs: [preclear,lint,codespell,table_of_contents] strategy: matrix: - version: ["phase0", "altair", "bellatrix", "capella", "deneb"] + version: ["phase0", "altair", "bellatrix", "capella", "deneb", "eip6110"] steps: - name: Checkout this repo uses: actions/checkout@v3.2.0 From f9b359be09df44e12732c90e87d9c281542dc859 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 28 Mar 2023 12:03:14 +0900 Subject: [PATCH 004/117] Reuse indexes with full sweep --- specs/_features/reuse_indexes/beacon-chain.md | 45 +++++++++++++++++++ specs/altair/beacon-chain.md | 22 ++++++--- 2 files changed, 62 insertions(+), 5 deletions(-) create mode 100644 specs/_features/reuse_indexes/beacon-chain.md diff --git a/specs/_features/reuse_indexes/beacon-chain.md b/specs/_features/reuse_indexes/beacon-chain.md new file mode 100644 index 0000000000..fcf1de8968 --- /dev/null +++ b/specs/_features/reuse_indexes/beacon-chain.md @@ -0,0 +1,45 @@ +# Reuse indexes -- The Beacon Chain + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Preset](#preset) + - [Time parameters](#time-parameters) +- [Beacon chain state transition function](#beacon-chain-state-transition-function) + - [Block processing](#block-processing) + - [Modified `assign_index_to_deposit`](#modified-assign_index_to_deposit) + + + + +## Introduction + +This is the beacon chain specification to assign new deposits to existing validator records that have withdrawn long ago. + +*Note:* This specification is built upon [Capella](../../capella/beacon_chain.md) and is under active development. + +## Preset + +### Time parameters + +| Name | Value | Unit | Duration | +| - | - | - | +| `REUSE_VALIDATOR_INDEX_DELAY` | `uint64(2**16)` (= 65,536) | epochs | ~1 year | + +## Beacon chain state transition function + +### Block processing + +#### Modified `assign_index_to_deposit` + +```python +def assign_index_to_deposit(state: BeaconState) -> int: + for index, validator in enumerate(state.validators): + if validator.withdrawable_epoch < get_current_epoch(state) - REUSE_VALIDATOR_INDEX_DELAY: + return index + return len(state.validators) +``` diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 58dfad608a..d5b89a387b 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -511,16 +511,28 @@ def apply_deposit(state: BeaconState, signing_root = compute_signing_root(deposit_message, domain) # Initialize validator if the deposit signature is valid if bls.Verify(pubkey, signing_root, signature): - state.validators.append(get_validator_from_deposit(pubkey, withdrawal_credentials, amount)) - state.balances.append(amount) + index = assign_index_to_deposit(state) + update_list(state.validators, index, get_validator_from_deposit(pubkey, withdrawal_credentials, amount)) + update_list(state.balances, index, amount) # [New in Altair] - state.previous_epoch_participation.append(ParticipationFlags(0b0000_0000)) - state.current_epoch_participation.append(ParticipationFlags(0b0000_0000)) - state.inactivity_scores.append(uint64(0)) + update_list(state.previous_epoch_participation, index, ParticipationFlags(0b0000_0000)) + update_list(state.current_epoch_participation, index, ParticipationFlags(0b0000_0000)) + update_list(state.inactivity_scores, index, uint64(0)) else: # Increase balance by deposit amount index = ValidatorIndex(validator_pubkeys.index(pubkey)) increase_balance(state, index, amount) + + +def assign_index_to_deposit(state: BeaconState) -> int: + return len(state.validators) + + +def update_list(list: List, index: int, value: Any) -> None: + if index == len(list): + list.append(value) + else: + list[index] = value ``` #### Sync aggregate processing From 201f113b5096978c2f2afca3f293870b2398f81a Mon Sep 17 00:00:00 2001 From: Roberto Saltini Date: Tue, 28 Mar 2023 15:27:13 +1100 Subject: [PATCH 005/117] Introduce get_epoch_boundary_block --- specs/phase0/fork-choice.md | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 6e281d5c3d..044d85053f 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -192,6 +192,17 @@ def get_ancestor(store: Store, root: Root, slot: Slot) -> Root: return root ``` +#### `get_epoch_boundary_block` + +```python +def get_epoch_boundary_block(store: Store, root: Root, epoch: Epoch) -> Root: + """ + Compute the epoch boundary block for epoch ``epoch`` in the chain of block ``root`` + """ + epoch_first_slot = compute_start_slot_at_epoch(epoch) + return get_ancestor(store, root, epoch_first_slot) +``` + #### `get_weight` ```python @@ -278,10 +289,9 @@ def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconB voting_source.epoch + 2 >= current_epoch ) - finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) correct_finalized = ( store.finalized_checkpoint.epoch == GENESIS_EPOCH - or store.finalized_checkpoint.root == get_ancestor(store, block_root, finalized_slot) + or store.finalized_checkpoint.root == get_epoch_boundary_block(store, block_root, store.finalized_checkpoint.epoch) ) # If expected finalized/justified, add to viable block-tree and signal viability to parent. @@ -442,8 +452,7 @@ def validate_on_attestation(store: Store, attestation: Attestation, is_from_bloc assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot # LMD vote must be consistent with FFG vote target - target_slot = compute_start_slot_at_epoch(target.epoch) - assert target.root == get_ancestor(store, attestation.data.beacon_block_root, target_slot) + assert target.root == get_epoch_boundary_block(store, attestation.data.beacon_block_root, target.epoch) # Attestations can only affect the fork choice of subsequent slots. # Delay consideration in the fork choice until their slot is in the past. @@ -506,7 +515,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root + assert get_epoch_boundary_block(store, block.parent_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root # Check the block is valid and compute the post-state state = pre_state.copy() From ddbd82e1be7fc38bcdcf672865bb036ab80837a7 Mon Sep 17 00:00:00 2001 From: Roberto Saltini Date: Tue, 28 Mar 2023 15:51:34 +1100 Subject: [PATCH 006/117] Add toc --- specs/phase0/fork-choice.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 044d85053f..058089a62c 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -18,6 +18,7 @@ - [`get_current_slot`](#get_current_slot) - [`compute_slots_since_epoch_start`](#compute_slots_since_epoch_start) - [`get_ancestor`](#get_ancestor) + - [`get_epoch_boundary_block`](#get_epoch_boundary_block) - [`get_weight`](#get_weight) - [`get_voting_source`](#get_voting_source) - [`filter_block_tree`](#filter_block_tree) From 8acc31adceeae3f23def8c1291c0b3bacc6d586c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 Mar 2023 14:06:10 +0800 Subject: [PATCH 007/117] Fix the default testing genesis by setting `deposit_receipts_start_index` and fork versions --- configs/mainnet.yaml | 2 +- specs/_features/eip6110/beacon-chain.md | 2 +- tests/core/pyspec/eth2spec/test/context.py | 1 + tests/core/pyspec/eth2spec/test/helpers/genesis.py | 10 ++++++++-- 4 files changed, 11 insertions(+), 4 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index e62108b0aa..8703af1a4f 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -51,7 +51,7 @@ CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC DENEB_FORK_VERSION: 0x04000000 DENEB_FORK_EPOCH: 18446744073709551615 # EIP6110 -EIP6110_FORK_VERSION: 0x04000001 +EIP6110_FORK_VERSION: 0x04000001 # temporary stub EIP6110_FORK_EPOCH: 18446744073709551615 diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index 70a72a5f45..9e27b393cf 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -204,7 +204,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: for_ops(body.proposer_slashings, process_proposer_slashing) for_ops(body.attester_slashings, process_attester_slashing) for_ops(body.attestations, process_attestation) - for_ops(body.deposits, process_deposit) # [Modified in EIP-6110] + for_ops(body.deposits, process_deposit) for_ops(body.voluntary_exits, process_voluntary_exit) for_ops(body.bls_to_execution_changes, process_bls_to_execution_change) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index b859bf27c6..901fd273a8 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -432,6 +432,7 @@ def decorator(fn): with_bellatrix_and_later = with_all_phases_from(BELLATRIX) with_capella_and_later = with_all_phases_from(CAPELLA) with_deneb_and_later = with_all_phases_from(DENEB) +with_eip6110_and_later = with_all_phases_from(EIP6110) def _get_preset_targets(kw): diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index 0610f11ad8..db4f922515 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -1,11 +1,11 @@ from eth2spec.test.helpers.constants import ( - ALTAIR, BELLATRIX, CAPELLA, DENEB, + ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110, ) from eth2spec.test.helpers.execution_payload import ( compute_el_header_block_hash, ) from eth2spec.test.helpers.forks import ( - is_post_altair, is_post_bellatrix, is_post_capella, + is_post_altair, is_post_bellatrix, is_post_capella, is_post_eip6110, ) from eth2spec.test.helpers.keys import pubkeys @@ -80,6 +80,9 @@ def create_genesis_state(spec, validator_balances, activation_threshold): elif spec.fork == DENEB: previous_version = spec.config.CAPELLA_FORK_VERSION current_version = spec.config.DENEB_FORK_VERSION + elif spec.fork == EIP6110: + previous_version = spec.config.CAPELLA_FORK_VERSION + current_version = spec.config.EIP6110_FORK_VERSION state = spec.BeaconState( genesis_time=0, @@ -129,4 +132,7 @@ def create_genesis_state(spec, validator_balances, activation_threshold): eth1_block_hash=eth1_block_hash, ) + if is_post_eip6110(spec): + state.deposit_receipts_start_index = spec.UNSET_DEPOSIT_RECEIPTS_START_INDEX + return state From ee3e1ac63e6e675e58d22cf055431b70a6f39696 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 28 Mar 2023 15:34:07 +0900 Subject: [PATCH 008/117] Apply review comments --- specs/_features/reuse_indexes/beacon-chain.md | 28 ++++++++++++++++--- specs/altair/beacon-chain.md | 16 +++++------ 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/specs/_features/reuse_indexes/beacon-chain.md b/specs/_features/reuse_indexes/beacon-chain.md index fcf1de8968..8677631e56 100644 --- a/specs/_features/reuse_indexes/beacon-chain.md +++ b/specs/_features/reuse_indexes/beacon-chain.md @@ -9,9 +9,12 @@ - [Introduction](#introduction) - [Preset](#preset) - [Time parameters](#time-parameters) +- [Helpers](#helpers) + - [Predicates](#predicates) + - [`is_reusable_validator`](#is_reusable_validator) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - [Block processing](#block-processing) - - [Modified `assign_index_to_deposit`](#modified-assign_index_to_deposit) + - [Modified `get_index_for_new_validator`](#modified-get_index_for_new_validator) @@ -30,16 +33,33 @@ This is the beacon chain specification to assign new deposits to existing valida | - | - | - | | `REUSE_VALIDATOR_INDEX_DELAY` | `uint64(2**16)` (= 65,536) | epochs | ~1 year | +## Helper functions + +### Predicates + +#### `is_reusable_validator` + +```python +def is_reusable_validator(validator: Validator, balance: Gwei, epoch: Epoch) -> bool: + """ + Check if ``validator`` index can be re-assigned to a new deposit. + """ + return ( + validator.withdrawable_epoch < epoch - REUSE_VALIDATOR_INDEX_DELAY + and balance == 0 + ) +``` + ## Beacon chain state transition function ### Block processing -#### Modified `assign_index_to_deposit` +#### Modified `get_index_for_new_validator` ```python -def assign_index_to_deposit(state: BeaconState) -> int: +def get_index_for_new_validator(state: BeaconState) -> int: for index, validator in enumerate(state.validators): - if validator.withdrawable_epoch < get_current_epoch(state) - REUSE_VALIDATOR_INDEX_DELAY: + if is_reusable_validator(validator, state.balances[index], get_current_epoch(state)): return index return len(state.validators) ``` diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index d5b89a387b..5e24df83e0 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -511,24 +511,24 @@ def apply_deposit(state: BeaconState, signing_root = compute_signing_root(deposit_message, domain) # Initialize validator if the deposit signature is valid if bls.Verify(pubkey, signing_root, signature): - index = assign_index_to_deposit(state) - update_list(state.validators, index, get_validator_from_deposit(pubkey, withdrawal_credentials, amount)) - update_list(state.balances, index, amount) + index = get_index_for_new_validator(state) + update_or_append_to_list(state.validators, index, get_validator_from_deposit(pubkey, withdrawal_credentials, amount)) + update_or_append_to_list(state.balances, index, amount) # [New in Altair] - update_list(state.previous_epoch_participation, index, ParticipationFlags(0b0000_0000)) - update_list(state.current_epoch_participation, index, ParticipationFlags(0b0000_0000)) - update_list(state.inactivity_scores, index, uint64(0)) + update_or_append_to_list(state.previous_epoch_participation, index, ParticipationFlags(0b0000_0000)) + update_or_append_to_list(state.current_epoch_participation, index, ParticipationFlags(0b0000_0000)) + update_or_append_to_list(state.inactivity_scores, index, uint64(0)) else: # Increase balance by deposit amount index = ValidatorIndex(validator_pubkeys.index(pubkey)) increase_balance(state, index, amount) -def assign_index_to_deposit(state: BeaconState) -> int: +def get_index_for_new_validator(state: BeaconState) -> int: return len(state.validators) -def update_list(list: List, index: int, value: Any) -> None: +def update_or_append_to_list(list: List, index: int, value: Any) -> None: if index == len(list): list.append(value) else: From c1273bbfa405f7c335d36561c92d169552ef7432 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 28 Mar 2023 15:46:01 +0900 Subject: [PATCH 009/117] Add epoch > REUSE_VALIDATOR_INDEX_DELAY condition --- specs/_features/reuse_indexes/beacon-chain.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/specs/_features/reuse_indexes/beacon-chain.md b/specs/_features/reuse_indexes/beacon-chain.md index 8677631e56..3efa0c8698 100644 --- a/specs/_features/reuse_indexes/beacon-chain.md +++ b/specs/_features/reuse_indexes/beacon-chain.md @@ -45,8 +45,9 @@ def is_reusable_validator(validator: Validator, balance: Gwei, epoch: Epoch) -> Check if ``validator`` index can be re-assigned to a new deposit. """ return ( - validator.withdrawable_epoch < epoch - REUSE_VALIDATOR_INDEX_DELAY - and balance == 0 + epoch > REUSE_VALIDATOR_INDEX_DELAY + and validator.withdrawable_epoch < epoch - REUSE_VALIDATOR_INDEX_DELAY + and balance == 0 ) ``` From c1b16a2333b58c98a2e9dd2d47a84e65c0e37207 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 Mar 2023 14:49:13 +0800 Subject: [PATCH 010/117] Fix EIP-6110 configs --- configs/mainnet.yaml | 2 +- configs/minimal.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 8703af1a4f..5ad394c082 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -51,7 +51,7 @@ CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC DENEB_FORK_VERSION: 0x04000000 DENEB_FORK_EPOCH: 18446744073709551615 # EIP6110 -EIP6110_FORK_VERSION: 0x04000001 # temporary stub +EIP6110_FORK_VERSION: 0x05000000 # temporary stub EIP6110_FORK_EPOCH: 18446744073709551615 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 271a0c14ae..5895cfc707 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -50,7 +50,7 @@ CAPELLA_FORK_EPOCH: 18446744073709551615 DENEB_FORK_VERSION: 0x04000001 DENEB_FORK_EPOCH: 18446744073709551615 # EIP6110 -EIP6110_FORK_VERSION: 0x04000001 +EIP6110_FORK_VERSION: 0x05000001 EIP6110_FORK_EPOCH: 18446744073709551615 From cd7783e59d38a0cc9ec5ba02c81a38ff1bbeb22c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 Mar 2023 15:08:03 +0800 Subject: [PATCH 011/117] EIP-6110: Fix `compute_fork_version` and add light client specs --- setup.py | 4 + specs/_features/eip6110/beacon-chain.md | 2 +- specs/_features/eip6110/fork.md | 2 +- specs/_features/eip6110/light-client/fork.md | 111 ++++++++++++++++++ .../eip6110/light-client/full-node.md | 74 ++++++++++++ .../eip6110/light-client/p2p-interface.md | 105 +++++++++++++++++ .../eip6110/light-client/sync-protocol.md | 87 ++++++++++++++ .../test/altair/light_client/test_sync.py | 11 ++ .../eth2spec/test/helpers/fork_transition.py | 4 + 9 files changed, 398 insertions(+), 2 deletions(-) create mode 100644 specs/_features/eip6110/light-client/fork.md create mode 100644 specs/_features/eip6110/light-client/full-node.md create mode 100644 specs/_features/eip6110/light-client/p2p-interface.md create mode 100644 specs/_features/eip6110/light-client/sync-protocol.md diff --git a/setup.py b/setup.py index 52bad2b71b..2f8c4b12ec 100644 --- a/setup.py +++ b/setup.py @@ -1037,6 +1037,10 @@ def finalize_options(self): """ if self.spec_fork == EIP6110: self.md_doc_paths += """ + specs/_features/eip6110/light-client/fork.md + specs/_features/eip6110/light-client/full-node.md + specs/deneb/light-client/p2p-interface.md + specs/_features/eip6110/light-client/sync-protocol.md specs/_features/eip6110/beacon-chain.md specs/_features/eip6110/fork.md """ diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index 9e27b393cf..3ed77bafbb 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -157,7 +157,7 @@ class BeaconState(Container): current_sync_committee: SyncCommittee next_sync_committee: SyncCommittee # Execution - latest_execution_payload_header: ExecutionPayloadHeader + latest_execution_payload_header: ExecutionPayloadHeader # [Modified in EIP-6110] # Withdrawals next_withdrawal_index: WithdrawalIndex next_withdrawal_validator_index: ValidatorIndex diff --git a/specs/_features/eip6110/fork.md b/specs/_features/eip6110/fork.md index b08661e5fa..df98c4c69e 100644 --- a/specs/_features/eip6110/fork.md +++ b/specs/_features/eip6110/fork.md @@ -43,7 +43,7 @@ def compute_fork_version(epoch: Epoch) -> Version: Return the fork version at the given ``epoch``. """ if epoch >= EIP6110_FORK_EPOCH: - return EIP6110_FORK_EPOCH + return EIP6110_FORK_VERSION if epoch >= CAPELLA_FORK_EPOCH: return CAPELLA_FORK_VERSION if epoch >= BELLATRIX_FORK_EPOCH: diff --git a/specs/_features/eip6110/light-client/fork.md b/specs/_features/eip6110/light-client/fork.md new file mode 100644 index 0000000000..6ffa3d8697 --- /dev/null +++ b/specs/_features/eip6110/light-client/fork.md @@ -0,0 +1,111 @@ +# eip6110 Light Client -- Fork Logic + +## Table of contents + + + + + +- [Introduction](#introduction) + - [Upgrading light client data](#upgrading-light-client-data) + - [Upgrading the store](#upgrading-the-store) + + + + +## Introduction + +This document describes how to upgrade existing light client objects based on the [Capella specification](../../capella/light-client/sync-protocol.md) to eip6110. This is necessary when processing pre-eip6110 data with a post-eip6110 `LightClientStore`. Note that the data being exchanged over the network protocols uses the original format. + +### Upgrading light client data + +A eip6110 `LightClientStore` can still process earlier light client data. In order to do so, that pre-eip6110 data needs to be locally upgraded to eip6110 before processing. + +```python +def upgrade_lc_header_to_eip6110(pre: capella.LightClientHeader) -> LightClientHeader: + return LightClientHeader( + beacon=pre.beacon, + execution=ExecutionPayloadHeader( + parent_hash=pre.execution.parent_hash, + fee_recipient=pre.execution.fee_recipient, + state_root=pre.execution.state_root, + receipts_root=pre.execution.receipts_root, + logs_bloom=pre.execution.logs_bloom, + prev_randao=pre.execution.prev_randao, + block_number=pre.execution.block_number, + gas_limit=pre.execution.gas_limit, + gas_used=pre.execution.gas_used, + timestamp=pre.execution.timestamp, + extra_data=pre.execution.extra_data, + base_fee_per_gas=pre.execution.base_fee_per_gas, + block_hash=pre.execution.block_hash, + transactions_root=pre.execution.transactions_root, + withdrawals_root=pre.execution.withdrawals_root, + deposit_receipts_root=Root(), + ), + execution_branch=pre.execution_branch, + ) +``` + +```python +def upgrade_lc_bootstrap_to_eip6110(pre: capella.LightClientBootstrap) -> LightClientBootstrap: + return LightClientBootstrap( + header=upgrade_lc_header_to_eip6110(pre.header), + current_sync_committee=pre.current_sync_committee, + current_sync_committee_branch=pre.current_sync_committee_branch, + ) +``` + +```python +def upgrade_lc_update_to_eip6110(pre: capella.LightClientUpdate) -> LightClientUpdate: + return LightClientUpdate( + attested_header=upgrade_lc_header_to_eip6110(pre.attested_header), + next_sync_committee=pre.next_sync_committee, + next_sync_committee_branch=pre.next_sync_committee_branch, + finalized_header=upgrade_lc_header_to_eip6110(pre.finalized_header), + finality_branch=pre.finality_branch, + sync_aggregate=pre.sync_aggregate, + signature_slot=pre.signature_slot, + ) +``` + +```python +def upgrade_lc_finality_update_to_eip6110(pre: capella.LightClientFinalityUpdate) -> LightClientFinalityUpdate: + return LightClientFinalityUpdate( + attested_header=upgrade_lc_header_to_eip6110(pre.attested_header), + finalized_header=upgrade_lc_header_to_eip6110(pre.finalized_header), + finality_branch=pre.finality_branch, + sync_aggregate=pre.sync_aggregate, + signature_slot=pre.signature_slot, + ) +``` + +```python +def upgrade_lc_optimistic_update_to_eip6110(pre: capella.LightClientOptimisticUpdate) -> LightClientOptimisticUpdate: + return LightClientOptimisticUpdate( + attested_header=upgrade_lc_header_to_eip6110(pre.attested_header), + sync_aggregate=pre.sync_aggregate, + signature_slot=pre.signature_slot, + ) +``` + +### Upgrading the store + +Existing `LightClientStore` objects based on Capella MUST be upgraded to eip6110 before eip6110 based light client data can be processed. The `LightClientStore` upgrade MAY be performed before `eip6110_FORK_EPOCH`. + +```python +def upgrade_lc_store_to_eip6110(pre: capella.LightClientStore) -> LightClientStore: + if pre.best_valid_update is None: + best_valid_update = None + else: + best_valid_update = upgrade_lc_update_to_eip6110(pre.best_valid_update) + return LightClientStore( + finalized_header=upgrade_lc_header_to_eip6110(pre.finalized_header), + current_sync_committee=pre.current_sync_committee, + next_sync_committee=pre.next_sync_committee, + best_valid_update=best_valid_update, + optimistic_header=upgrade_lc_header_to_eip6110(pre.optimistic_header), + previous_max_active_participants=pre.previous_max_active_participants, + current_max_active_participants=pre.current_max_active_participants, + ) +``` diff --git a/specs/_features/eip6110/light-client/full-node.md b/specs/_features/eip6110/light-client/full-node.md new file mode 100644 index 0000000000..0e400dd161 --- /dev/null +++ b/specs/_features/eip6110/light-client/full-node.md @@ -0,0 +1,74 @@ +# Deneb Light Client -- Full Node + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Helper functions](#helper-functions) + - [Modified `block_to_light_client_header`](#modified-block_to_light_client_header) + + + + +## Introduction + +This upgrade adds information about the execution payload to light client data as part of the Deneb upgrade. + +## Helper functions + +### Modified `block_to_light_client_header` + +```python +def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: + epoch = compute_epoch_at_slot(block.message.slot) + + if epoch >= CAPELLA_FORK_EPOCH: + payload = block.message.body.execution_payload + execution_header = ExecutionPayloadHeader( + parent_hash=payload.parent_hash, + fee_recipient=payload.fee_recipient, + state_root=payload.state_root, + receipts_root=payload.receipts_root, + logs_bloom=payload.logs_bloom, + prev_randao=payload.prev_randao, + block_number=payload.block_number, + gas_limit=payload.gas_limit, + gas_used=payload.gas_used, + timestamp=payload.timestamp, + extra_data=payload.extra_data, + base_fee_per_gas=payload.base_fee_per_gas, + block_hash=payload.block_hash, + transactions_root=hash_tree_root(payload.transactions), + withdrawals_root=hash_tree_root(payload.withdrawals), + ) + + # [New in Deneb] + if epoch >= EIP6110_FORK_EPOCH: + execution_header.deposit_receipts_root = hash_tree_root(payload.deposit_receipts) + + execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX) + else: + # Note that during fork transitions, `finalized_header` may still point to earlier forks. + # While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`), + # it was not included in the corresponding light client data. To ensure compatibility + # with legacy data going through `upgrade_lc_header_to_capella`, leave out execution data. + execution_header = ExecutionPayloadHeader() + execution_branch = [Bytes32() for _ in range(floorlog2(EXECUTION_PAYLOAD_INDEX))] + + return LightClientHeader( + beacon=BeaconBlockHeader( + slot=block.message.slot, + proposer_index=block.message.proposer_index, + parent_root=block.message.parent_root, + state_root=block.message.state_root, + body_root=hash_tree_root(block.message.body), + ), + execution=execution_header, + execution_branch=execution_branch, + ) +``` diff --git a/specs/_features/eip6110/light-client/p2p-interface.md b/specs/_features/eip6110/light-client/p2p-interface.md new file mode 100644 index 0000000000..9b33d59ffd --- /dev/null +++ b/specs/_features/eip6110/light-client/p2p-interface.md @@ -0,0 +1,105 @@ +# EIP-6110 Light Client -- Networking + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Networking](#networking) + - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) + - [Topics and messages](#topics-and-messages) + - [Global topics](#global-topics) + - [`light_client_finality_update`](#light_client_finality_update) + - [`light_client_optimistic_update`](#light_client_optimistic_update) + - [The Req/Resp domain](#the-reqresp-domain) + - [Messages](#messages) + - [GetLightClientBootstrap](#getlightclientbootstrap) + - [LightClientUpdatesByRange](#lightclientupdatesbyrange) + - [GetLightClientFinalityUpdate](#getlightclientfinalityupdate) + - [GetLightClientOptimisticUpdate](#getlightclientoptimisticupdate) + + + + +## Networking + +The [Capella light client networking specification](../../capella/light-client/p2p-interface.md) is extended to exchange [EIP-6110 light client data](./sync-protocol.md). + +### The gossip domain: gossipsub + +#### Topics and messages + +##### Global topics + +###### `light_client_finality_update` + +[0]: # (eth2spec: skip) + +| `fork_version` | Message SSZ type | +|--------------------------------------------------------|-------------------------------------| +| `GENESIS_FORK_VERSION` | n/a | +| `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientFinalityUpdate` | +| `CAPELLA_FORK_VERSION` | `capella.LightClientFinalityUpdate` | +| `EIP6110_FORK_VERSION` and later | `eip6110.LightClientFinalityUpdate` | + +###### `light_client_optimistic_update` + +[0]: # (eth2spec: skip) + +| `fork_version` | Message SSZ type | +|--------------------------------------------------------|---------------------------------------| +| `GENESIS_FORK_VERSION` | n/a | +| `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientOptimisticUpdate` | +| `CAPELLA_FORK_VERSION` | `capella.LightClientOptimisticUpdate` | +| `EIP6110_FORK_VERSION` and later | `eip6110.LightClientOptimisticUpdate` | + +### The Req/Resp domain + +#### Messages + +##### GetLightClientBootstrap + +[0]: # (eth2spec: skip) + +| `fork_version` | Response SSZ type | +|--------------------------------------------------------|------------------------------------| +| `GENESIS_FORK_VERSION` | n/a | +| `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientBootstrap` | +| `CAPELLA_FORK_VERSION` | `capella.LightClientBootstrap` | +| `EIP6110_FORK_VERSION` and later | `eip6110.LightClientBootstrap` | + +##### LightClientUpdatesByRange + +[0]: # (eth2spec: skip) + +| `fork_version` | Response chunk SSZ type | +|--------------------------------------------------------|----------------------------------| +| `GENESIS_FORK_VERSION` | n/a | +| `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientUpdate` | +| `CAPELLA_FORK_VERSION` | `capella.LightClientUpdate` | +| `EIP6110_FORK_VERSION` and later | `eip6110.LightClientUpdate` | + +##### GetLightClientFinalityUpdate + +[0]: # (eth2spec: skip) + +| `fork_version` | Response SSZ type | +|--------------------------------------------------------|-------------------------------------| +| `GENESIS_FORK_VERSION` | n/a | +| `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientFinalityUpdate` | +| `CAPELLA_FORK_VERSION` | `capella.LightClientFinalityUpdate` | +| `EIP6110_FORK_VERSION` and later | `eip6110.LightClientFinalityUpdate` | + +##### GetLightClientOptimisticUpdate + +[0]: # (eth2spec: skip) + +| `fork_version` | Response SSZ type | +|--------------------------------------------------------|---------------------------------------| +| `GENESIS_FORK_VERSION` | n/a | +| `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientOptimisticUpdate` | +| `CAPELLA_FORK_VERSION` | `capella.LightClientOptimisticUpdate` | +| `EIP6110_FORK_VERSION` and later | `eip6110.LightClientOptimisticUpdate` | diff --git a/specs/_features/eip6110/light-client/sync-protocol.md b/specs/_features/eip6110/light-client/sync-protocol.md new file mode 100644 index 0000000000..867aa2730d --- /dev/null +++ b/specs/_features/eip6110/light-client/sync-protocol.md @@ -0,0 +1,87 @@ +# Deneb Light Client -- Sync Protocol + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Helper functions](#helper-functions) + - [Modified `get_lc_execution_root`](#modified-get_lc_execution_root) + - [Modified `is_valid_light_client_header`](#modified-is_valid_light_client_header) + + + + +## Introduction + +This upgrade updates light client data to include the EIP-6110 changes to the [`ExecutionPayload`](../beacon-chain.md) structure. It extends the [Capella Light Client specifications](../../capella/light-client/sync-protocol.md). The [fork document](./fork.md) explains how to upgrade existing Capella based deployments to EIP-6110. + +Additional documents describes the impact of the upgrade on certain roles: +- [Full node](./full-node.md) +- [Networking](./p2p-interface.md) + +## Helper functions + +### Modified `get_lc_execution_root` + +```python +def get_lc_execution_root(header: LightClientHeader) -> Root: + epoch = compute_epoch_at_slot(header.beacon.slot) + + # [New in EIP-6110] + if epoch >= EIP6110_FORK_EPOCH: + return hash_tree_root(header.execution) + + # [Modified in EIP-6110] + if epoch >= CAPELLA_FORK_EPOCH: + execution_header = capella.ExecutionPayloadHeader( + parent_hash=header.execution.parent_hash, + fee_recipient=header.execution.fee_recipient, + state_root=header.execution.state_root, + receipts_root=header.execution.receipts_root, + logs_bloom=header.execution.logs_bloom, + prev_randao=header.execution.prev_randao, + block_number=header.execution.block_number, + gas_limit=header.execution.gas_limit, + gas_used=header.execution.gas_used, + timestamp=header.execution.timestamp, + extra_data=header.execution.extra_data, + base_fee_per_gas=header.execution.base_fee_per_gas, + block_hash=header.execution.block_hash, + transactions_root=header.execution.transactions_root, + withdrawals_root=header.execution.withdrawals_root, + ) + return hash_tree_root(execution_header) + + return Root() +``` + +### Modified `is_valid_light_client_header` + +```python +def is_valid_light_client_header(header: LightClientHeader) -> bool: + epoch = compute_epoch_at_slot(header.beacon.slot) + + # [New in EIP-6110] + if epoch < EIP6110_FORK_EPOCH: + if header.execution.withdrawals_root != Root(): + return False + + if epoch < CAPELLA_FORK_EPOCH: + return ( + header.execution == ExecutionPayloadHeader() + and header.execution_branch == [Bytes32() for _ in range(floorlog2(EXECUTION_PAYLOAD_INDEX))] + ) + + return is_valid_merkle_branch( + leaf=get_lc_execution_root(header), + branch=header.execution_branch, + depth=floorlog2(EXECUTION_PAYLOAD_INDEX), + index=get_subtree_index(EXECUTION_PAYLOAD_INDEX), + root=header.beacon.body_root, + ) +``` diff --git a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py index d33e68961d..1a527a767a 100644 --- a/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py +++ b/tests/core/pyspec/eth2spec/test/altair/light_client/test_sync.py @@ -26,6 +26,7 @@ from eth2spec.test.helpers.forks import ( is_post_capella, is_post_deneb, is_post_fork, + is_post_eip6110, ) from eth2spec.test.helpers.light_client import ( get_sync_aggregate, @@ -57,6 +58,10 @@ def needs_upgrade_to_deneb(d_spec, s_spec): return is_post_deneb(s_spec) and not is_post_deneb(d_spec) +def needs_upgrade_to_eip6110(d_spec, s_spec): + return is_post_eip6110(s_spec) and not is_post_eip6110(d_spec) + + def check_lc_header_equal(d_spec, s_spec, data, upgraded): assert upgraded.beacon.slot == data.beacon.slot assert upgraded.beacon.hash_tree_root() == data.beacon.hash_tree_root() @@ -84,6 +89,10 @@ def upgrade_lc_bootstrap_to_store(d_spec, s_spec, data): upgraded = s_spec.upgrade_lc_bootstrap_to_deneb(upgraded) check_lc_bootstrap_equal(d_spec, s_spec, data, upgraded) + if needs_upgrade_to_eip6110(d_spec, s_spec): + upgraded = s_spec.upgrade_lc_bootstrap_to_eip6110(upgraded) + check_lc_bootstrap_equal(d_spec, s_spec, data, upgraded) + return upgraded @@ -145,6 +154,8 @@ class LightClientSyncTest(object): def get_store_fork_version(s_spec): + if is_post_eip6110(s_spec): + return s_spec.config.EIP6110_FORK_VERSION if is_post_deneb(s_spec): return s_spec.config.DENEB_FORK_VERSION if is_post_capella(s_spec): diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py b/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py index 241c7dc37e..20c20b938c 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py @@ -15,6 +15,7 @@ BELLATRIX, CAPELLA, DENEB, + EIP6110, ) from eth2spec.test.helpers.deposits import ( prepare_state_and_deposit, @@ -173,6 +174,9 @@ def do_fork(state, spec, post_spec, fork_epoch, with_block=True, sync_aggregate= elif post_spec.fork == DENEB: assert state.fork.previous_version == post_spec.config.CAPELLA_FORK_VERSION assert state.fork.current_version == post_spec.config.DENEB_FORK_VERSION + elif post_spec.fork == EIP6110: + assert state.fork.previous_version == post_spec.config.CAPELLA_FORK_VERSION + assert state.fork.current_version == post_spec.config.EIP6110_FORK_VERSION if with_block: return state, _state_transition_and_sign_block_at_slot( From 9dfee5ef48301de5e9bcf6a3610f689dd350aaf1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 Mar 2023 15:21:09 +0800 Subject: [PATCH 012/117] Update execution_payload helpers for new EIP-6110 field --- .../eth2spec/test/helpers/execution_payload.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index c0a70aca1d..7e2c7c976b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -4,7 +4,11 @@ from rlp.sedes import big_endian_int, Binary, List from eth2spec.debug.random_value import get_random_bytes_list -from eth2spec.test.helpers.forks import is_post_capella, is_post_deneb +from eth2spec.test.helpers.forks import ( + is_post_capella, + is_post_deneb, + is_post_eip6110, +) def get_execution_payload_header(spec, execution_payload): @@ -28,6 +32,8 @@ def get_execution_payload_header(spec, execution_payload): payload_header.withdrawals_root = spec.hash_tree_root(execution_payload.withdrawals) if is_post_deneb(spec): payload_header.excess_data_gas = execution_payload.excess_data_gas + if is_post_eip6110(spec): + payload_header.deposit_receipts_root = spec.hash_tree_root(execution_payload.deposit_receipts) return payload_header @@ -48,7 +54,8 @@ def compute_trie_root_from_indexed_data(data): def compute_el_header_block_hash(spec, payload_header, transactions_trie_root, - withdrawals_trie_root=None): + withdrawals_trie_root=None, + deposit_receipts_trie_root=None): """ Computes the RLP execution block hash described by an `ExecutionPayloadHeader`. """ @@ -92,6 +99,10 @@ def compute_el_header_block_hash(spec, if is_post_deneb(spec): # excess_data_gas execution_payload_header_rlp.append((big_endian_int, payload_header.excess_data_gas)) + if is_post_eip6110(spec): + # TODO: RLP or SSZ for `deposit_receipts_root` + # FIXME: if using RLP, we need to implement `get_deposit_receipt_rlp` helper + ... sedes = List([schema for schema, _ in execution_payload_header_rlp]) values = [value for _, value in execution_payload_header_rlp] @@ -165,6 +176,9 @@ def build_empty_execution_payload(spec, state, randao_mix=None): ) if is_post_capella(spec): payload.withdrawals = spec.get_expected_withdrawals(state) + if is_post_eip6110(spec): + # just to be clear + payload.deposit_receipts = [] payload.block_hash = compute_el_block_hash(spec, payload) From 7476c1e0c9a67b39e4fa7e6ba8412ef1b5da7254 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 28 Mar 2023 15:34:07 +0800 Subject: [PATCH 013/117] Fix wrong doc path --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2f8c4b12ec..7b4ab6ac2e 100644 --- a/setup.py +++ b/setup.py @@ -1039,7 +1039,7 @@ def finalize_options(self): self.md_doc_paths += """ specs/_features/eip6110/light-client/fork.md specs/_features/eip6110/light-client/full-node.md - specs/deneb/light-client/p2p-interface.md + specs/_features/eip6110/light-client/p2p-interface.md specs/_features/eip6110/light-client/sync-protocol.md specs/_features/eip6110/beacon-chain.md specs/_features/eip6110/fork.md From 314b040fff2d5a5aa2ab8e7524c76200f3a64d79 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 28 Mar 2023 17:45:52 +0900 Subject: [PATCH 014/117] Reduce line len --- specs/altair/beacon-chain.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 5e24df83e0..9d7c8180e3 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -512,7 +512,8 @@ def apply_deposit(state: BeaconState, # Initialize validator if the deposit signature is valid if bls.Verify(pubkey, signing_root, signature): index = get_index_for_new_validator(state) - update_or_append_to_list(state.validators, index, get_validator_from_deposit(pubkey, withdrawal_credentials, amount)) + validator = get_validator_from_deposit(pubkey, withdrawal_credentials, amount) + update_or_append_to_list(state.validators, index, validator) update_or_append_to_list(state.balances, index, amount) # [New in Altair] update_or_append_to_list(state.previous_epoch_participation, index, ParticipationFlags(0b0000_0000)) From c7029ce19e3f25d2d0e3ea301650561d1ac67d12 Mon Sep 17 00:00:00 2001 From: Roberto Saltini Date: Wed, 29 Mar 2023 12:40:58 +1100 Subject: [PATCH 015/117] Rename get_epoch_boundary_block to get_ancestor_at_epoch_boundary --- specs/phase0/fork-choice.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 058089a62c..2ee6ecb926 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -18,7 +18,7 @@ - [`get_current_slot`](#get_current_slot) - [`compute_slots_since_epoch_start`](#compute_slots_since_epoch_start) - [`get_ancestor`](#get_ancestor) - - [`get_epoch_boundary_block`](#get_epoch_boundary_block) + - [`get_ancestor_at_epoch_boundary`](#get_ancestor_at_epoch_boundary) - [`get_weight`](#get_weight) - [`get_voting_source`](#get_voting_source) - [`filter_block_tree`](#filter_block_tree) @@ -193,10 +193,10 @@ def get_ancestor(store: Store, root: Root, slot: Slot) -> Root: return root ``` -#### `get_epoch_boundary_block` +#### `get_ancestor_at_epoch_boundary` ```python -def get_epoch_boundary_block(store: Store, root: Root, epoch: Epoch) -> Root: +def get_ancestor_at_epoch_boundary(store: Store, root: Root, epoch: Epoch) -> Root: """ Compute the epoch boundary block for epoch ``epoch`` in the chain of block ``root`` """ @@ -292,7 +292,7 @@ def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconB correct_finalized = ( store.finalized_checkpoint.epoch == GENESIS_EPOCH - or store.finalized_checkpoint.root == get_epoch_boundary_block(store, block_root, store.finalized_checkpoint.epoch) + or store.finalized_checkpoint.root == get_ancestor_at_epoch_boundary(store, block_root, store.finalized_checkpoint.epoch) ) # If expected finalized/justified, add to viable block-tree and signal viability to parent. @@ -453,7 +453,7 @@ def validate_on_attestation(store: Store, attestation: Attestation, is_from_bloc assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot # LMD vote must be consistent with FFG vote target - assert target.root == get_epoch_boundary_block(store, attestation.data.beacon_block_root, target.epoch) + assert target.root == get_ancestor_at_epoch_boundary(store, attestation.data.beacon_block_root, target.epoch) # Attestations can only affect the fork choice of subsequent slots. # Delay consideration in the fork choice until their slot is in the past. @@ -516,7 +516,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert get_epoch_boundary_block(store, block.parent_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root + assert get_ancestor_at_epoch_boundary(store, block.parent_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root # Check the block is valid and compute the post-state state = pre_state.copy() From f696b30608a1b6402d32c3c87ab01be99e2096bd Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 30 Mar 2023 09:11:36 +0900 Subject: [PATCH 016/117] Address PR review --- specs/_features/reuse_indexes/beacon-chain.md | 13 ++++++------- specs/altair/beacon-chain.md | 6 +++--- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/specs/_features/reuse_indexes/beacon-chain.md b/specs/_features/reuse_indexes/beacon-chain.md index 3efa0c8698..5ec08ed199 100644 --- a/specs/_features/reuse_indexes/beacon-chain.md +++ b/specs/_features/reuse_indexes/beacon-chain.md @@ -1,4 +1,4 @@ -# Reuse indexes -- The Beacon Chain +# Reuse indices -- The Beacon Chain ## Table of contents @@ -31,7 +31,7 @@ This is the beacon chain specification to assign new deposits to existing valida | Name | Value | Unit | Duration | | - | - | - | -| `REUSE_VALIDATOR_INDEX_DELAY` | `uint64(2**16)` (= 65,536) | epochs | ~1 year | +| `REUSE_VALIDATOR_INDEX_DELAY` | `uint64(2**16)` (= 65,536) | epochs | ~0.8 year | ## Helper functions @@ -45,8 +45,7 @@ def is_reusable_validator(validator: Validator, balance: Gwei, epoch: Epoch) -> Check if ``validator`` index can be re-assigned to a new deposit. """ return ( - epoch > REUSE_VALIDATOR_INDEX_DELAY - and validator.withdrawable_epoch < epoch - REUSE_VALIDATOR_INDEX_DELAY + epoch > validator.withdrawable_epoch + REUSE_VALIDATOR_INDEX_DELAY and balance == 0 ) ``` @@ -58,9 +57,9 @@ def is_reusable_validator(validator: Validator, balance: Gwei, epoch: Epoch) -> #### Modified `get_index_for_new_validator` ```python -def get_index_for_new_validator(state: BeaconState) -> int: +def get_index_for_new_validator(state: BeaconState) -> ValidatorIndex: for index, validator in enumerate(state.validators): if is_reusable_validator(validator, state.balances[index], get_current_epoch(state)): - return index - return len(state.validators) + return ValidatorIndex(index) + return ValidatorIndex(len(state.validators)) ``` diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 9d7c8180e3..df65d984c5 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -525,11 +525,11 @@ def apply_deposit(state: BeaconState, increase_balance(state, index, amount) -def get_index_for_new_validator(state: BeaconState) -> int: - return len(state.validators) +def get_index_for_new_validator(state: BeaconState) -> ValidatorIndex: + return ValidatorIndex(len(state.validators)) -def update_or_append_to_list(list: List, index: int, value: Any) -> None: +def update_or_append_to_list(list: List, index: ValidatorIndex, value: Any) -> None: if index == len(list): list.append(value) else: From 3d9c87b27ccc9e8b96c9749abd7c727452bbb44a Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 30 Mar 2023 09:44:34 +0900 Subject: [PATCH 017/117] Move to misc helpers --- specs/altair/beacon-chain.md | 40 +++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index df65d984c5..4c7bb3f9a1 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -30,6 +30,8 @@ - [Misc](#misc-1) - [`add_flag`](#add_flag) - [`has_flag`](#has_flag) + - [`get_index_for_new_validator`](#get_index_for_new_validator) + - [`set_or_append_list`](#set_or_append_list) - [Beacon state accessors](#beacon-state-accessors) - [`get_next_sync_committee_indices`](#get_next_sync_committee_indices) - [`get_next_sync_committee`](#get_next_sync_committee) @@ -248,6 +250,23 @@ def has_flag(flags: ParticipationFlags, flag_index: int) -> bool: return flags & flag == flag ``` +#### `get_index_for_new_validator` + +```python +def get_index_for_new_validator(state: BeaconState) -> ValidatorIndex: + return ValidatorIndex(len(state.validators)) +``` + +#### `set_or_append_list` + +```python +def set_or_append_list(list: List[Any], index: ValidatorIndex, value: Any) -> None: + if index == len(list): + list.append(value) + else: + list[index] = value +``` + ### Beacon state accessors #### `get_next_sync_committee_indices` @@ -513,27 +532,16 @@ def apply_deposit(state: BeaconState, if bls.Verify(pubkey, signing_root, signature): index = get_index_for_new_validator(state) validator = get_validator_from_deposit(pubkey, withdrawal_credentials, amount) - update_or_append_to_list(state.validators, index, validator) - update_or_append_to_list(state.balances, index, amount) + set_or_append_list(state.validators, index, validator) + set_or_append_list(state.balances, index, amount) # [New in Altair] - update_or_append_to_list(state.previous_epoch_participation, index, ParticipationFlags(0b0000_0000)) - update_or_append_to_list(state.current_epoch_participation, index, ParticipationFlags(0b0000_0000)) - update_or_append_to_list(state.inactivity_scores, index, uint64(0)) + set_or_append_list(state.previous_epoch_participation, index, ParticipationFlags(0b0000_0000)) + set_or_append_list(state.current_epoch_participation, index, ParticipationFlags(0b0000_0000)) + set_or_append_list(state.inactivity_scores, index, uint64(0)) else: # Increase balance by deposit amount index = ValidatorIndex(validator_pubkeys.index(pubkey)) increase_balance(state, index, amount) - - -def get_index_for_new_validator(state: BeaconState) -> ValidatorIndex: - return ValidatorIndex(len(state.validators)) - - -def update_or_append_to_list(list: List, index: ValidatorIndex, value: Any) -> None: - if index == len(list): - list.append(value) - else: - list[index] = value ``` #### Sync aggregate processing From 7cb163090246e152927bbc5857b4fd8c9ad53bf9 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 30 Mar 2023 14:09:16 +1100 Subject: [PATCH 018/117] Attnet revamp draft --- specs/phase0/validator.md | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 54b344791e..2ed047f0ff 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -88,10 +88,11 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | Name | Value | Unit | Duration | | - | - | :-: | :-: | -| `TARGET_AGGREGATORS_PER_COMMITTEE` | `2**4` (= 16) | validators | | -| `RANDOM_SUBNETS_PER_VALIDATOR` | `2**0` (= 1) | subnets | | -| `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | +| `TARGET_AGGREGATORS_PER_COMMITTEE` | `2**4` (= 16) | validators | +| `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | | `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | +| `ATTESTATION_SUBNET_EXTRA_BITS` | 0 | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | +| `SUBNETS_PER_NODE` | 2 | The number of long-lived subnets a beacon node should be subscribed to. | ## Containers @@ -606,15 +607,29 @@ def get_aggregate_and_proof_signature(state: BeaconState, ## Phase 0 attestation subnet stability -Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each validator must: +Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each beacon node should: -* Randomly select and remain subscribed to `RANDOM_SUBNETS_PER_VALIDATOR` attestation subnets -* Maintain advertisement of the randomly selected subnets in their node's ENR `attnets` entry by setting the randomly selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets -* Set the lifetime of each random subscription to a random number of epochs between `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` and `2 * EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION]`. At the end of life for a subscription, select a new random subnet, update subnet subscriptions, and publish an updated ENR +* Remain subscribed to `SUBNETS_PER_NODE` for `SUBNET_DURATION_IN_EPOCHS` epochs. +* Maintain advertisement of the selected subnets in their node's ENR `attnets` entry by setting the selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets. +* Select these subnets based on their node-id as specified by the following + `compute_subnets(node_id,epoch)` function. -*Note*: Short lived beacon committee assignments should not be added in into the ENR `attnets` entry. +```python +ATTESTATION_SUBNET_PREFIX_BITS = ceil(log2(ATTESTATION_SUBNET_COUNT)) + ATTESTATION_SUBNET_EXTRA_BITS + +def compute_subnet(node_id, epoch, index): + node_id_prefix = node_id >> (256 - ATTESTATION_SUBNET_PREFIX_BITS) + permutation_seed = hash(uint_to_bytes(epoch // SUBNET_DURATION_IN_EPOCHS)) + permutated_prefix = compute_shuffled_index(node_id_prefix, 1 << ATTESTATION_SUBNET_PREFIX_BITS, permutation_seed) + return (permutated_prefix + index) % ATTESTATION_SUBNET_COUNT + +def compute_subnets(node_id, epoch): + return [compute_subnet(node_id, epoch, idx) for idx in range(SUBNETS_PER_NODE)] +``` + +*Note*: Nodes should subscribe to new subnets and remain subscribed to old subnets for at least one epoch. Nodes should pick a random duration to unsubscribe from old subnets to smooth the transition on the exact epoch boundary of which the shuffling changes. -*Note*: When preparing for a hard fork, a validator must select and subscribe to random subnets of the future fork versioning at least `EPOCHS_PER_RANDOM_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. +*Note*: When preparing for a hard fork, a validator must select and subscribe to subnets of the future fork versioning at least `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. ## How to avoid slashing From 0dd8db76cd7d21a2853f0aad5995d027daf8c0e3 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 30 Mar 2023 14:51:41 +0800 Subject: [PATCH 019/117] Make linter happy. Add `SUBNET_DURATION_IN_EPOCHS` definition. --- specs/phase0/validator.md | 27 ++++++++++--------- .../unittests/test_config_invariants.py | 2 +- 2 files changed, 15 insertions(+), 14 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 2ed047f0ff..4df4437d04 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -91,8 +91,10 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | `TARGET_AGGREGATORS_PER_COMMITTEE` | `2**4` (= 16) | validators | | `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | | `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | -| `ATTESTATION_SUBNET_EXTRA_BITS` | 0 | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | -| `SUBNETS_PER_NODE` | 2 | The number of long-lived subnets a beacon node should be subscribed to. | +| `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | +| `SUBNETS_PER_NODE` | `2` | The number of long-lived subnets a beacon node should be subscribed to. | +| `ATTESTATION_SUBNET_PREFIX_BITS` | `(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | | +| `SUBNET_DURATION_IN_EPOCHS` | `2` | | ## Containers @@ -611,20 +613,19 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th * Remain subscribed to `SUBNETS_PER_NODE` for `SUBNET_DURATION_IN_EPOCHS` epochs. * Maintain advertisement of the selected subnets in their node's ENR `attnets` entry by setting the selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets. -* Select these subnets based on their node-id as specified by the following - `compute_subnets(node_id,epoch)` function. +* Select these subnets based on their node-id as specified by the following `compute_subnets(node_id,epoch)` function. ```python -ATTESTATION_SUBNET_PREFIX_BITS = ceil(log2(ATTESTATION_SUBNET_COUNT)) + ATTESTATION_SUBNET_EXTRA_BITS - -def compute_subnet(node_id, epoch, index): - node_id_prefix = node_id >> (256 - ATTESTATION_SUBNET_PREFIX_BITS) - permutation_seed = hash(uint_to_bytes(epoch // SUBNET_DURATION_IN_EPOCHS)) - permutated_prefix = compute_shuffled_index(node_id_prefix, 1 << ATTESTATION_SUBNET_PREFIX_BITS, permutation_seed) - return (permutated_prefix + index) % ATTESTATION_SUBNET_COUNT +def compute_subnet(node_id: int, epoch: Epoch, index: int) -> int: + node_id_prefix = node_id >> (256 - ATTESTATION_SUBNET_PREFIX_BITS) + permutation_seed = hash(uint_to_bytes(epoch // SUBNET_DURATION_IN_EPOCHS)) + permutated_prefix = compute_shuffled_index(node_id_prefix, 1 << ATTESTATION_SUBNET_PREFIX_BITS, permutation_seed) + return (permutated_prefix + index) % ATTESTATION_SUBNET_COUNT +``` -def compute_subnets(node_id, epoch): - return [compute_subnet(node_id, epoch, idx) for idx in range(SUBNETS_PER_NODE)] +```python +def compute_subnets(node_id: int, epoch: Epoch) -> Sequence[int]: + return [compute_subnet(node_id, epoch, idx) for idx in range(SUBNETS_PER_NODE)] ``` *Note*: Nodes should subscribe to new subnets and remain subscribed to old subnets for at least one epoch. Nodes should pick a random duration to unsubscribe from old subnets to smooth the transition on the exact epoch boundary of which the shuffling changes. diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py index 9b27d1deb9..69aa3eb2a5 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py @@ -75,7 +75,7 @@ def test_time(spec, state): @with_all_phases @spec_state_test def test_networking(spec, state): - assert spec.RANDOM_SUBNETS_PER_VALIDATOR <= spec.ATTESTATION_SUBNET_COUNT + assert spec.SUBNETS_PER_NODE <= spec.ATTESTATION_SUBNET_COUNT @with_all_phases From e255d09840855e16e62f0e531f2ca50fa45e2724 Mon Sep 17 00:00:00 2001 From: Roberto Saltini Date: Fri, 31 Mar 2023 10:52:52 +1100 Subject: [PATCH 020/117] Apply changes to Bellatrix and Daneb --- specs/bellatrix/fork-choice.md | 2 +- specs/deneb/fork-choice.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/bellatrix/fork-choice.md b/specs/bellatrix/fork-choice.md index ed7d60a932..aac26f3dd9 100644 --- a/specs/bellatrix/fork-choice.md +++ b/specs/bellatrix/fork-choice.md @@ -170,7 +170,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root + assert get_ancestor_at_epoch_boundary(store, block.parent_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root # Check the block is valid and compute the post-state state = pre_state.copy() diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index 61714cf1a8..83cdb9972f 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -82,7 +82,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root + assert get_ancestor_at_epoch_boundary(store, block.parent_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root # [New in Deneb] # Check if blob data is available From 4c401d6575e06902e82cd427e8c99c7011c652c2 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Mon, 3 Apr 2023 18:17:03 +0600 Subject: [PATCH 021/117] Port process_deposit tests --- .../test_process_deposit_receipt.py | 233 ++++++++++++++++++ .../pyspec/eth2spec/test/helpers/deposits.py | 135 ++++++++++ 2 files changed, 368 insertions(+) create mode 100644 tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_deposit_receipt.py diff --git a/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_deposit_receipt.py b/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_deposit_receipt.py new file mode 100644 index 0000000000..411a9f6af8 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_deposit_receipt.py @@ -0,0 +1,233 @@ +from eth2spec.test.context import spec_state_test, always_bls, with_eip6110_and_later +from eth2spec.test.helpers.deposits import ( + prepare_deposit_receipt, + run_deposit_receipt_processing, + run_deposit_receipt_processing_with_specific_fork_version +) + + +@with_eip6110_and_later +@spec_state_test +def test_new_deposit_under_max(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + # effective balance will be 1 EFFECTIVE_BALANCE_INCREMENT smaller because of this small decrement. + amount = spec.MAX_EFFECTIVE_BALANCE - 1 + deposit_receipt = prepare_deposit_receipt(spec, validator_index, amount, signed=True) + + yield from run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index) + + +@with_eip6110_and_later +@spec_state_test +def test_new_deposit_max(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + # effective balance will be exactly the same as balance. + amount = spec.MAX_EFFECTIVE_BALANCE + deposit_receipt = prepare_deposit_receipt(spec, validator_index, amount, signed=True) + + yield from run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index) + + +@with_eip6110_and_later +@spec_state_test +def test_new_deposit_over_max(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + # just 1 over the limit, effective balance should be set MAX_EFFECTIVE_BALANCE during processing + amount = spec.MAX_EFFECTIVE_BALANCE + 1 + deposit_receipt = prepare_deposit_receipt(spec, validator_index, amount, signed=True) + + yield from run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index) + + +@with_eip6110_and_later +@spec_state_test +def test_new_deposit_eth1_withdrawal_credentials(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + withdrawal_credentials = ( + spec.ETH1_ADDRESS_WITHDRAWAL_PREFIX + + b'\x00' * 11 # specified 0s + + b'\x59' * 20 # a 20-byte eth1 address + ) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit_receipt = prepare_deposit_receipt( + spec, + validator_index, + amount, + withdrawal_credentials=withdrawal_credentials, + signed=True, + ) + + yield from run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index) + + +@with_eip6110_and_later +@spec_state_test +def test_new_deposit_non_versioned_withdrawal_credentials(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + withdrawal_credentials = ( + b'\xFF' # Non specified withdrawal credentials version + + b'\x02' * 31 # Garabage bytes + ) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit_receipt = prepare_deposit_receipt( + spec, + validator_index, + amount, + withdrawal_credentials=withdrawal_credentials, + signed=True, + ) + + yield from run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index) + + +@with_eip6110_and_later +@spec_state_test +@always_bls +def test_correct_sig_but_forked_state(spec, state): + validator_index = len(state.validators) + amount = spec.MAX_EFFECTIVE_BALANCE + # deposits will always be valid, regardless of the current fork + state.fork.current_version = spec.Version('0x1234abcd') + deposit_receipt = prepare_deposit_receipt(spec, validator_index, amount, signed=True) + yield from run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index) + + +@with_eip6110_and_later +@spec_state_test +@always_bls +def test_incorrect_sig_new_deposit(spec, state): + # fresh deposit = next validator index = validator appended to registry + validator_index = len(state.validators) + amount = spec.MAX_EFFECTIVE_BALANCE + deposit_receipt = prepare_deposit_receipt(spec, validator_index, amount) + yield from run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index, effective=False) + + +@with_eip6110_and_later +@spec_state_test +def test_top_up__max_effective_balance(spec, state): + validator_index = 0 + amount = spec.MAX_EFFECTIVE_BALANCE // 4 + deposit_receipt = prepare_deposit_receipt(spec, validator_index, amount, signed=True) + + state.balances[validator_index] = spec.MAX_EFFECTIVE_BALANCE + state.validators[validator_index].effective_balance = spec.MAX_EFFECTIVE_BALANCE + + yield from run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index) + + assert state.balances[validator_index] == spec.MAX_EFFECTIVE_BALANCE + amount + assert state.validators[validator_index].effective_balance == spec.MAX_EFFECTIVE_BALANCE + + +@with_eip6110_and_later +@spec_state_test +def test_top_up__less_effective_balance(spec, state): + validator_index = 0 + amount = spec.MAX_EFFECTIVE_BALANCE // 4 + deposit_receipt = prepare_deposit_receipt(spec, validator_index, amount, signed=True) + + initial_balance = spec.MAX_EFFECTIVE_BALANCE - 1000 + initial_effective_balance = spec.MAX_EFFECTIVE_BALANCE - spec.EFFECTIVE_BALANCE_INCREMENT + state.balances[validator_index] = initial_balance + state.validators[validator_index].effective_balance = initial_effective_balance + + yield from run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index) + + assert state.balances[validator_index] == initial_balance + amount + # unchanged effective balance + assert state.validators[validator_index].effective_balance == initial_effective_balance + + +@with_eip6110_and_later +@spec_state_test +def test_top_up__zero_balance(spec, state): + validator_index = 0 + amount = spec.MAX_EFFECTIVE_BALANCE // 4 + deposit_receipt = prepare_deposit_receipt(spec, validator_index, amount, signed=True) + + initial_balance = 0 + initial_effective_balance = 0 + state.balances[validator_index] = initial_balance + state.validators[validator_index].effective_balance = initial_effective_balance + + yield from run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index) + + assert state.balances[validator_index] == initial_balance + amount + # unchanged effective balance + assert state.validators[validator_index].effective_balance == initial_effective_balance + + +@with_eip6110_and_later +@spec_state_test +@always_bls +def test_incorrect_sig_top_up(spec, state): + validator_index = 0 + amount = spec.MAX_EFFECTIVE_BALANCE // 4 + deposit_receipt = prepare_deposit_receipt(spec, validator_index, amount) + + # invalid signatures, in top-ups, are allowed! + yield from run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index) + + +@with_eip6110_and_later +@spec_state_test +def test_incorrect_withdrawal_credentials_top_up(spec, state): + validator_index = 0 + amount = spec.MAX_EFFECTIVE_BALANCE // 4 + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(b"junk")[1:] + deposit_receipt = prepare_deposit_receipt( + spec, + validator_index, + amount, + withdrawal_credentials=withdrawal_credentials + ) + + # inconsistent withdrawal credentials, in top-ups, are allowed! + yield from run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index) + + +@with_eip6110_and_later +@spec_state_test +def test_key_validate_invalid_subgroup(spec, state): + validator_index = len(state.validators) + amount = spec.MAX_EFFECTIVE_BALANCE + + # All-zero pubkey would not pass `bls.KeyValidate`, but `process_deposit` would not throw exception. + pubkey = b'\x00' * 48 + + deposit_receipt = prepare_deposit_receipt(spec, validator_index, amount, pubkey=pubkey, signed=True) + + yield from run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index) + + +@with_eip6110_and_later +@spec_state_test +def test_key_validate_invalid_decompression(spec, state): + validator_index = len(state.validators) + amount = spec.MAX_EFFECTIVE_BALANCE + + # `deserialization_fails_infinity_with_true_b_flag` BLS G1 deserialization test case. + # This pubkey would not pass `bls.KeyValidate`, but `process_deposit` would not throw exception. + pubkey_hex = 'c01000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000' + pubkey = bytes.fromhex(pubkey_hex) + + deposit_receipt = prepare_deposit_receipt(spec, validator_index, amount, pubkey=pubkey, signed=True) + + yield from run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index) + + +@with_eip6110_and_later +@spec_state_test +@always_bls +def test_ineffective_deposit_with_bad_fork_version(spec, state): + yield from run_deposit_receipt_processing_with_specific_fork_version( + spec, + state, + fork_version=spec.Version('0xAaBbCcDd'), + effective=False, + ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/deposits.py b/tests/core/pyspec/eth2spec/test/helpers/deposits.py index cfff9c5ef9..789b27c8fd 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/deposits.py +++ b/tests/core/pyspec/eth2spec/test/helpers/deposits.py @@ -171,6 +171,54 @@ def prepare_state_and_deposit(spec, state, validator_index, amount, return deposit +def build_deposit_receipt(spec, + index, + pubkey, + privkey, + amount, + withdrawal_credentials, + signed): + deposit_data = build_deposit_data(spec, pubkey, privkey, amount, withdrawal_credentials, signed=signed) + return spec.DepositReceipt( + pubkey=deposit_data.pubkey, + withdrawal_credentials=deposit_data.withdrawal_credentials, + amount=deposit_data.amount, + signature=deposit_data.signature, + index=index) + + +def prepare_deposit_receipt(spec, validator_index, amount, + index=None, + pubkey=None, + privkey=None, + withdrawal_credentials=None, + signed=False): + """ + Create a deposit receipt for the given validator, depositing the given amount. + """ + if index is None: + index = validator_index + + if pubkey is None: + pubkey = pubkeys[validator_index] + + if privkey is None: + privkey = privkeys[validator_index] + + # insecurely use pubkey as withdrawal key if no credentials provided + if withdrawal_credentials is None: + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(pubkey)[1:] + + return build_deposit_receipt( + spec, + index, + pubkey, + privkey, + amount, + withdrawal_credentials, + signed, + ) + # # Run processing # @@ -255,3 +303,90 @@ def run_deposit_processing_with_specific_fork_version( state.eth1_data.deposit_count = 1 yield from run_deposit_processing(spec, state, deposit, validator_index, valid=valid, effective=effective) + + +def run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index, valid=True, effective=True): + """ + Run ``process_deposit_receipt``, yielding: + - pre-state ('pre') + - deposit_receipt ('deposit_receipt') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + pre_validator_count = len(state.validators) + pre_balance = 0 + is_top_up = False + # is a top-up + if validator_index < pre_validator_count: + is_top_up = True + pre_balance = get_balance(state, validator_index) + pre_effective_balance = state.validators[validator_index].effective_balance + + yield 'pre', state + yield 'deposit_receipt', deposit_receipt + + if not valid: + expect_assertion_error(lambda: spec.process_deposit_receipt(state, deposit_receipt)) + yield 'post', None + return + + spec.process_deposit_receipt(state, deposit_receipt) + + yield 'post', state + + if not effective or not bls.KeyValidate(deposit_receipt.pubkey): + assert len(state.validators) == pre_validator_count + assert len(state.balances) == pre_validator_count + if is_top_up: + assert get_balance(state, validator_index) == pre_balance + else: + if is_top_up: + # Top-ups do not change effective balance + assert state.validators[validator_index].effective_balance == pre_effective_balance + assert len(state.validators) == pre_validator_count + assert len(state.balances) == pre_validator_count + else: + # new validator + assert len(state.validators) == pre_validator_count + 1 + assert len(state.balances) == pre_validator_count + 1 + effective_balance = min(spec.MAX_EFFECTIVE_BALANCE, deposit_receipt.amount) + effective_balance -= effective_balance % spec.EFFECTIVE_BALANCE_INCREMENT + assert state.validators[validator_index].effective_balance == effective_balance + + assert get_balance(state, validator_index) == pre_balance + deposit_receipt.amount + + +def run_deposit_receipt_processing_with_specific_fork_version( + spec, + state, + fork_version, + valid=True, + effective=True): + validator_index = len(state.validators) + amount = spec.MAX_EFFECTIVE_BALANCE + + pubkey = pubkeys[validator_index] + privkey = privkeys[validator_index] + withdrawal_credentials = spec.BLS_WITHDRAWAL_PREFIX + spec.hash(pubkey)[1:] + + deposit_message = spec.DepositMessage(pubkey=pubkey, withdrawal_credentials=withdrawal_credentials, amount=amount) + domain = spec.compute_domain(domain_type=spec.DOMAIN_DEPOSIT, fork_version=fork_version) + deposit_data = spec.DepositData( + pubkey=pubkey, withdrawal_credentials=withdrawal_credentials, amount=amount, + signature=bls.Sign(privkey, spec.compute_signing_root(deposit_message, domain)) + ) + deposit_receipt = spec.DepositReceipt( + pubkey=deposit_data.pubkey, + withdrawal_credentials=deposit_data.withdrawal_credentials, + amount=deposit_data.amount, + signature=deposit_data.signature, + index=validator_index) + + yield from run_deposit_receipt_processing( + spec, + state, + deposit_receipt, + validator_index, + valid=valid, + effective=effective + ) From 25ea243859329cb7d7131a03051ed78e0abd03c3 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 4 Apr 2023 11:45:08 +0900 Subject: [PATCH 022/117] PR comments --- .../_features/{reuse_indexes => reuse_indices}/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename specs/_features/{reuse_indexes => reuse_indices}/beacon-chain.md (99%) diff --git a/specs/_features/reuse_indexes/beacon-chain.md b/specs/_features/reuse_indices/beacon-chain.md similarity index 99% rename from specs/_features/reuse_indexes/beacon-chain.md rename to specs/_features/reuse_indices/beacon-chain.md index 5ec08ed199..ecac7fb308 100644 --- a/specs/_features/reuse_indexes/beacon-chain.md +++ b/specs/_features/reuse_indices/beacon-chain.md @@ -30,7 +30,7 @@ This is the beacon chain specification to assign new deposits to existing valida ### Time parameters | Name | Value | Unit | Duration | -| - | - | - | +| - | - | - | - | | `REUSE_VALIDATOR_INDEX_DELAY` | `uint64(2**16)` (= 65,536) | epochs | ~0.8 year | ## Helper functions From 622c9b97bac1a586a8b6692fe1998d3e621a8c30 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 4 Apr 2023 12:00:09 +0900 Subject: [PATCH 023/117] Fix CI --- specs/_features/reuse_indices/beacon-chain.md | 2 +- specs/altair/beacon-chain.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/_features/reuse_indices/beacon-chain.md b/specs/_features/reuse_indices/beacon-chain.md index ecac7fb308..6dd71e36fe 100644 --- a/specs/_features/reuse_indices/beacon-chain.md +++ b/specs/_features/reuse_indices/beacon-chain.md @@ -9,7 +9,7 @@ - [Introduction](#introduction) - [Preset](#preset) - [Time parameters](#time-parameters) -- [Helpers](#helpers) +- [Helper functions](#helper-functions) - [Predicates](#predicates) - [`is_reusable_validator`](#is_reusable_validator) - [Beacon chain state transition function](#beacon-chain-state-transition-function) diff --git a/specs/altair/beacon-chain.md b/specs/altair/beacon-chain.md index 4c7bb3f9a1..8c3a8877e8 100644 --- a/specs/altair/beacon-chain.md +++ b/specs/altair/beacon-chain.md @@ -260,7 +260,7 @@ def get_index_for_new_validator(state: BeaconState) -> ValidatorIndex: #### `set_or_append_list` ```python -def set_or_append_list(list: List[Any], index: ValidatorIndex, value: Any) -> None: +def set_or_append_list(list: List, index: ValidatorIndex, value: Any) -> None: if index == len(list): list.append(value) else: From 502745e012214098115b506065d78f629dccc7be Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Tue, 4 Apr 2023 16:30:44 +0600 Subject: [PATCH 024/117] Port tests from bellatrix and capella --- .../test_process_deposit_receipt.py | 53 ++++++++++++++++++- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_deposit_receipt.py b/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_deposit_receipt.py index 411a9f6af8..d78c18ecb7 100644 --- a/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_deposit_receipt.py +++ b/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_process_deposit_receipt.py @@ -4,6 +4,8 @@ run_deposit_receipt_processing, run_deposit_receipt_processing_with_specific_fork_version ) +from eth2spec.test.helpers.state import next_epoch_via_block +from eth2spec.test.helpers.withdrawals import set_validator_fully_withdrawable @with_eip6110_and_later @@ -224,10 +226,57 @@ def test_key_validate_invalid_decompression(spec, state): @with_eip6110_and_later @spec_state_test @always_bls -def test_ineffective_deposit_with_bad_fork_version(spec, state): +def test_ineffective_deposit_with_previous_fork_version(spec, state): + # Since deposits are valid across forks, the domain is always set with `GENESIS_FORK_VERSION`. + # It's an ineffective deposit because it fails at BLS sig verification. + # NOTE: it was effective in Altair. + assert state.fork.previous_version != state.fork.current_version + yield from run_deposit_receipt_processing_with_specific_fork_version( spec, state, - fork_version=spec.Version('0xAaBbCcDd'), + fork_version=state.fork.previous_version, effective=False, ) + + +@with_eip6110_and_later +@spec_state_test +@always_bls +def test_effective_deposit_with_genesis_fork_version(spec, state): + assert spec.config.GENESIS_FORK_VERSION not in (state.fork.previous_version, state.fork.current_version) + + yield from run_deposit_receipt_processing_with_specific_fork_version( + spec, + state, + fork_version=spec.config.GENESIS_FORK_VERSION, + ) + + +@with_eip6110_and_later +@spec_state_test +def test_success_top_up_to_withdrawn_validator(spec, state): + validator_index = 0 + + # Fully withdraw validator + set_validator_fully_withdrawable(spec, state, validator_index) + assert state.balances[validator_index] > 0 + next_epoch_via_block(spec, state) + assert state.balances[validator_index] == 0 + assert state.validators[validator_index].effective_balance > 0 + next_epoch_via_block(spec, state) + assert state.validators[validator_index].effective_balance == 0 + + # Make a top-up balance to validator + amount = spec.MAX_EFFECTIVE_BALANCE // 4 + deposit_receipt = prepare_deposit_receipt(spec, validator_index, amount, len(state.validators), signed=True) + + yield from run_deposit_receipt_processing(spec, state, deposit_receipt, validator_index) + + assert state.balances[validator_index] == amount + assert state.validators[validator_index].effective_balance == 0 + + validator = state.validators[validator_index] + balance = state.balances[validator_index] + current_epoch = spec.get_current_epoch(state) + assert spec.is_fully_withdrawable_validator(validator, balance, current_epoch) From 4cac76181827562a010c9c45dba1bb9f80f4f6cd Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 5 Apr 2023 11:38:20 +0800 Subject: [PATCH 025/117] make linter happy --- specs/bellatrix/fork-choice.md | 6 +++++- specs/deneb/fork-choice.md | 6 +++++- specs/phase0/fork-choice.md | 12 ++++++++++-- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/specs/bellatrix/fork-choice.md b/specs/bellatrix/fork-choice.md index aac26f3dd9..d22436c9d8 100644 --- a/specs/bellatrix/fork-choice.md +++ b/specs/bellatrix/fork-choice.md @@ -170,7 +170,11 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert get_ancestor_at_epoch_boundary(store, block.parent_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root + assert store.finalized_checkpoint.root == get_ancestor_at_epoch_boundary( + store, + block.parent_root, + store.finalized_checkpoint.epoch, + ) # Check the block is valid and compute the post-state state = pre_state.copy() diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index 83cdb9972f..e76e159c4f 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -82,7 +82,11 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert get_ancestor_at_epoch_boundary(store, block.parent_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root + assert store.finalized_checkpoint.root == get_ancestor_at_epoch_boundary( + store, + block.parent_root, + store.finalized_checkpoint.epoch, + ) # [New in Deneb] # Check if blob data is available diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 2ee6ecb926..478dd21427 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -292,7 +292,11 @@ def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconB correct_finalized = ( store.finalized_checkpoint.epoch == GENESIS_EPOCH - or store.finalized_checkpoint.root == get_ancestor_at_epoch_boundary(store, block_root, store.finalized_checkpoint.epoch) + or store.finalized_checkpoint.root == get_ancestor_at_epoch_boundary( + store, + block_root, + store.finalized_checkpoint.epoch, + ) ) # If expected finalized/justified, add to viable block-tree and signal viability to parent. @@ -516,7 +520,11 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert get_ancestor_at_epoch_boundary(store, block.parent_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root + assert store.finalized_checkpoint.root == get_ancestor_at_epoch_boundary( + store, + block.parent_root, + store.finalized_checkpoint.epoch, + ) # Check the block is valid and compute the post-state state = pre_state.copy() From 80e6b0d6657598f069b11096025bbc6a30e3d648 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Wed, 5 Apr 2023 19:46:28 +0600 Subject: [PATCH 026/117] Add deposits transition tests --- .../test_deposit_transition.py | 238 ++++++++++++++++++ .../test/helpers/execution_payload.py | 34 ++- .../pyspec/eth2spec/test/helpers/genesis.py | 7 +- 3 files changed, 272 insertions(+), 7 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_deposit_transition.py diff --git a/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_deposit_transition.py b/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_deposit_transition.py new file mode 100644 index 0000000000..92c3a96b55 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_deposit_transition.py @@ -0,0 +1,238 @@ +from eth2spec.test.helpers.block import ( + build_empty_block_for_next_slot, +) +from eth2spec.test.context import ( + spec_state_test, + with_phases, + EIP6110, + expect_assertion_error, +) +from eth2spec.test.helpers.deposits import ( + build_deposit_data, + deposit_from_context, + prepare_deposit_receipt, +) +from eth2spec.test.helpers.execution_payload import ( + compute_el_block_hash, +) +from eth2spec.test.helpers.keys import privkeys, pubkeys + + +def run_deposit_transition_block(spec, state, block, top_up_keys=[], valid=True): + """ + Run ``process_block``, yielding: + - pre-state ('pre') + - block ('block') + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + yield 'pre', state + yield 'block', block + + if not valid: + expect_assertion_error(lambda: spec.process_block(state, block)) + yield 'post', None + return + + spec.process_block(state, block) + yield 'post', state + + # Check that deposits are applied + expected_pubkeys = [d.data.pubkey for d in block.body.deposits] + deposit_receipts = block.body.execution_payload.deposit_receipts + expected_pubkeys = expected_pubkeys + [d.pubkey for d in deposit_receipts if (d.pubkey not in top_up_keys)] + actual_pubkeys = [v.pubkey for v in state.validators[len(state.validators) - len(expected_pubkeys):]] + + assert actual_pubkeys == expected_pubkeys + + +def prepare_state_and_block(spec, + state, + deposit_cnt, + deposit_receipt_cnt, + first_deposit_receipt_index=0, + deposit_receipts_start_index=None): + deposits = [] + deposit_receipts = [] + keypair_index = len(state.validators) + + # Prepare deposits + deposit_data_list = [] + for index in range(deposit_cnt): + deposit_data = build_deposit_data(spec, + pubkeys[keypair_index], + privkeys[keypair_index], + # use max effective balance + spec.MAX_EFFECTIVE_BALANCE, + # insecurely use pubkey as withdrawal key + spec.BLS_WITHDRAWAL_PREFIX + spec.hash(pubkeys[keypair_index])[1:], + signed=True) + deposit_data_list.append(deposit_data) + keypair_index += 1 + + deposit_root = None + for index in range(deposit_cnt): + deposit, deposit_root, _ = deposit_from_context(spec, deposit_data_list, index) + deposits.append(deposit) + + if deposit_root: + state.eth1_deposit_index = 0 + state.eth1_data = spec.Eth1Data(deposit_root=deposit_root, + deposit_count=deposit_cnt, + block_hash=state.eth1_data.block_hash) + + # Prepare deposit receipts + for offset in range(deposit_receipt_cnt): + deposit_receipt = prepare_deposit_receipt(spec, + keypair_index, + # use max effective balance + spec.MAX_EFFECTIVE_BALANCE, + first_deposit_receipt_index + offset, + signed=True) + deposit_receipts.append(deposit_receipt) + keypair_index += 1 + + # Set start index if defined + if deposit_receipts_start_index: + state.deposit_receipts_start_index = deposit_receipts_start_index + + block = build_empty_block_for_next_slot(spec, state) + + # Assign deposits and deposit receipts + block.body.deposits = deposits + block.body.execution_payload.deposit_receipts = deposit_receipts + block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + + # Advance a slot + spec.process_slots(state, block.slot) + + return state, block + + +@with_phases([EIP6110]) +@spec_state_test +def test_deposit_transition__start_index_is_set(spec, state): + # 0 deposits, 2 deposit receipts, unset deposit_receipts_start_index + state, block = prepare_state_and_block(spec, state, + deposit_cnt=0, + deposit_receipt_cnt=2, + first_deposit_receipt_index=state.eth1_data.deposit_count + 11) + + yield from run_deposit_transition_block(spec, state, block) + + # deposit_receipts_start_index must be set to the index of the first receipt + assert state.deposit_receipts_start_index == block.body.execution_payload.deposit_receipts[0].index + + +@with_phases([EIP6110]) +@spec_state_test +def test_deposit_transition__process_eth1_deposits(spec, state): + # 3 deposits, 1 deposit receipt, state.eth1_data.deposit_count < state.deposit_receipts_start_index + state, block = prepare_state_and_block(spec, state, + deposit_cnt=3, + deposit_receipt_cnt=1, + first_deposit_receipt_index=11, + deposit_receipts_start_index=7) + + yield from run_deposit_transition_block(spec, state, block) + + +@with_phases([EIP6110]) +@spec_state_test +def test_deposit_transition__process_max_eth1_deposits(spec, state): + # spec.MAX_DEPOSITS deposits, 1 deposit receipt, state.eth1_data.deposit_count > state.deposit_receipts_start_index + # state.deposit_receipts_start_index == spec.MAX_DEPOSITS + state, block = prepare_state_and_block(spec, state, + deposit_cnt=spec.MAX_DEPOSITS, + deposit_receipt_cnt=1, + first_deposit_receipt_index=spec.MAX_DEPOSITS + 1, + deposit_receipts_start_index=spec.MAX_DEPOSITS) + state.eth1_data = spec.Eth1Data(deposit_root=state.eth1_data.deposit_root, + deposit_count=23, + block_hash=state.eth1_data.block_hash) + + yield from run_deposit_transition_block(spec, state, block) + + +@with_phases([EIP6110]) +@spec_state_test +def test_deposit_transition__process_eth1_deposits_up_to_start_index(spec, state): + # 3 deposits, 1 deposit receipt, state.eth1_data.deposit_count == state.deposit_receipts_start_index + state, block = prepare_state_and_block(spec, state, + deposit_cnt=3, + deposit_receipt_cnt=1, + first_deposit_receipt_index=7, + deposit_receipts_start_index=3) + + yield from run_deposit_transition_block(spec, state, block) + + +@with_phases([EIP6110]) +@spec_state_test +def test_deposit_transition__invalid_not_enough_eth1_deposits(spec, state): + # 3 deposits, 1 deposit receipt, state.eth1_data.deposit_count < state.deposit_receipts_start_index + state, block = prepare_state_and_block(spec, state, + deposit_cnt=3, + deposit_receipt_cnt=1, + first_deposit_receipt_index=29, + deposit_receipts_start_index=23) + state.eth1_data = spec.Eth1Data(deposit_root=state.eth1_data.deposit_root, + deposit_count=17, + block_hash=state.eth1_data.block_hash) + + yield from run_deposit_transition_block(spec, state, block, valid=False) + + +@with_phases([EIP6110]) +@spec_state_test +def test_deposit_transition__invalid_too_many_eth1_deposits(spec, state): + # 3 deposits, 1 deposit receipt, state.eth1_data.deposit_count < state.eth1_data_index + state, block = prepare_state_and_block(spec, state, + deposit_cnt=3, + deposit_receipt_cnt=1, + first_deposit_receipt_index=11, + deposit_receipts_start_index=7) + state.eth1_data = spec.Eth1Data(deposit_root=state.eth1_data.deposit_root, + deposit_count=2, + block_hash=state.eth1_data.block_hash) + + yield from run_deposit_transition_block(spec, state, block, valid=False) + + +@with_phases([EIP6110]) +@spec_state_test +def test_deposit_transition__invalid_eth1_deposits_overlap_in_protocol_deposits(spec, state): + # spec.MAX_DEPOSITS deposits, 1 deposit receipt, state.eth1_data.deposit_count > state.deposit_receipts_start_index + # state.deposit_receipts_start_index == spec.MAX_DEPOSITS - 1 + state, block = prepare_state_and_block(spec, state, + deposit_cnt=spec.MAX_DEPOSITS, + deposit_receipt_cnt=1, + first_deposit_receipt_index=spec.MAX_DEPOSITS, + deposit_receipts_start_index=spec.MAX_DEPOSITS - 1) + state.eth1_data = spec.Eth1Data(deposit_root=state.eth1_data.deposit_root, + deposit_count=23, + block_hash=state.eth1_data.block_hash) + + yield from run_deposit_transition_block(spec, state, block, valid=False) + + +@with_phases([EIP6110]) +@spec_state_test +def test_deposit_transition__deposit_and_top_up_same_block(spec, state): + # 1 deposit, 1 deposit receipt that top ups deposited validator + state, block = prepare_state_and_block(spec, state, + deposit_cnt=1, + deposit_receipt_cnt=1, + first_deposit_receipt_index=11, + deposit_receipts_start_index=7) + + # Artificially assign deposit's pubkey to a deposit receipt of the same block + top_up_keys = [block.body.deposits[0].data.pubkey] + block.body.execution_payload.deposit_receipts[0].pubkey = top_up_keys[0] + block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + + yield from run_deposit_transition_block(spec, state, block, top_up_keys=top_up_keys) + + # Check the top up + expected_balance = block.body.deposits[0].data.amount + block.body.execution_payload.deposit_receipts[0].amount + assert state.balances[len(state.balances) - 1] == expected_balance diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index 7e2c7c976b..d9980810c3 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -100,9 +100,9 @@ def compute_el_header_block_hash(spec, # excess_data_gas execution_payload_header_rlp.append((big_endian_int, payload_header.excess_data_gas)) if is_post_eip6110(spec): - # TODO: RLP or SSZ for `deposit_receipts_root` - # FIXME: if using RLP, we need to implement `get_deposit_receipt_rlp` helper - ... + # deposit_receipts_root + assert deposit_receipts_trie_root is not None + execution_payload_header_rlp.append((Binary(32, 32), deposit_receipts_trie_root)) sedes = List([schema for schema, _ in execution_payload_header_rlp]) values = [value for _, value in execution_payload_header_rlp] @@ -129,14 +129,37 @@ def get_withdrawal_rlp(spec, withdrawal): return encode(values, sedes) +def get_deposit_receipt_rlp(spec, deposit_receipt): + deposit_receipt_rlp = [ + # pubkey + (Binary(48, 48), deposit_receipt.pubkey), + # withdrawal_credentials + (Binary(32, 32), deposit_receipt.withdrawal_credentials), + # amount + (big_endian_int, deposit_receipt.amount), + # pubkey + (Binary(96, 96), deposit_receipt.signature), + # index + (big_endian_int, deposit_receipt.index), + ] + + sedes = List([schema for schema, _ in deposit_receipt_rlp]) + values = [value for _, value in deposit_receipt_rlp] + return encode(values, sedes) + + def compute_el_block_hash(spec, payload): transactions_trie_root = compute_trie_root_from_indexed_data(payload.transactions) + withdrawals_trie_root = None + deposit_receipts_trie_root = None + if is_post_capella(spec): withdrawals_encoded = [get_withdrawal_rlp(spec, withdrawal) for withdrawal in payload.withdrawals] withdrawals_trie_root = compute_trie_root_from_indexed_data(withdrawals_encoded) - else: - withdrawals_trie_root = None + if is_post_eip6110(spec): + deposit_receipts_encoded = [get_deposit_receipt_rlp(spec, receipt) for receipt in payload.deposit_receipts] + deposit_receipts_trie_root = compute_trie_root_from_indexed_data(deposit_receipts_encoded) payload_header = get_execution_payload_header(spec, payload) @@ -145,6 +168,7 @@ def compute_el_block_hash(spec, payload): payload_header, transactions_trie_root, withdrawals_trie_root, + deposit_receipts_trie_root, ) diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index db4f922515..32ce8974d9 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -47,17 +47,20 @@ def get_sample_genesis_execution_payload_header(spec, ) transactions_trie_root = bytes.fromhex("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + withdrawals_trie_root = None + deposit_receipts_trie_root = None if is_post_capella(spec): withdrawals_trie_root = bytes.fromhex("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") - else: - withdrawals_trie_root = None + if is_post_eip6110(spec): + deposit_receipts_trie_root = bytes.fromhex("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") payload_header.block_hash = compute_el_header_block_hash( spec, payload_header, transactions_trie_root, withdrawals_trie_root, + deposit_receipts_trie_root, ) return payload_header From a0d03378fabf76cb91897ffe17310050f3996ee2 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 6 Apr 2023 12:40:55 +1000 Subject: [PATCH 027/117] Correct subnet subscription duration variable --- specs/phase0/validator.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 4df4437d04..1b06aecfbb 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -94,7 +94,6 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | | `SUBNETS_PER_NODE` | `2` | The number of long-lived subnets a beacon node should be subscribed to. | | `ATTESTATION_SUBNET_PREFIX_BITS` | `(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | | -| `SUBNET_DURATION_IN_EPOCHS` | `2` | | ## Containers @@ -611,14 +610,14 @@ def get_aggregate_and_proof_signature(state: BeaconState, Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each beacon node should: -* Remain subscribed to `SUBNETS_PER_NODE` for `SUBNET_DURATION_IN_EPOCHS` epochs. +* Remain subscribed to `SUBNETS_PER_NODE` for `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs. * Maintain advertisement of the selected subnets in their node's ENR `attnets` entry by setting the selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets. * Select these subnets based on their node-id as specified by the following `compute_subnets(node_id,epoch)` function. ```python def compute_subnet(node_id: int, epoch: Epoch, index: int) -> int: node_id_prefix = node_id >> (256 - ATTESTATION_SUBNET_PREFIX_BITS) - permutation_seed = hash(uint_to_bytes(epoch // SUBNET_DURATION_IN_EPOCHS)) + permutation_seed = hash(uint_to_bytes(epoch // EPOCHS_PER_SUBNET_SUBSCRIPTION)) permutated_prefix = compute_shuffled_index(node_id_prefix, 1 << ATTESTATION_SUBNET_PREFIX_BITS, permutation_seed) return (permutated_prefix + index) % ATTESTATION_SUBNET_COUNT ``` From 108f1eed860ff143d9b9481342d6788e96b78e5f Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 6 Apr 2023 16:53:31 +0600 Subject: [PATCH 028/117] Rebase EIP6110 to Deneb --- setup.py | 6 ++--- specs/_features/eip6110/beacon-chain.md | 24 +++++++++++-------- specs/_features/eip6110/fork.md | 7 ++++-- specs/_features/eip6110/light-client/fork.md | 19 ++++++++------- .../eip6110/light-client/full-node.md | 5 +++- .../eip6110/light-client/p2p-interface.md | 8 ++++++- .../eip6110/light-client/sync-protocol.md | 12 ++++++---- specs/_features/eip6110/validator.md | 2 +- specs/deneb/light-client/fork.md | 1 + .../test_polynomial_commitments.py | 5 +++- .../pyspec/eth2spec/test/helpers/forks.py | 2 +- 11 files changed, 57 insertions(+), 34 deletions(-) diff --git a/setup.py b/setup.py index 7b4ab6ac2e..a17655dc7a 100644 --- a/setup.py +++ b/setup.py @@ -671,13 +671,13 @@ def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: # # EIP6110SpecBuilder # -class EIP6110SpecBuilder(CapellaSpecBuilder): +class EIP6110SpecBuilder(DenebSpecBuilder): fork: str = EIP6110 @classmethod def imports(cls, preset_name: str): return super().imports(preset_name) + f''' -from eth2spec.capella import {preset_name} as capella +from eth2spec.deneb import {preset_name} as deneb ''' @@ -1022,7 +1022,7 @@ def finalize_options(self): specs/capella/validator.md specs/capella/p2p-interface.md """ - if self.spec_fork == DENEB: + if self.spec_fork in (DENEB, EIP6110): self.md_doc_paths += """ specs/deneb/light-client/fork.md specs/deneb/light-client/full-node.md diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index 3ed77bafbb..9693aa92ec 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -33,7 +33,7 @@ This is the beacon chain specification of in-protocol deposits processing mechanism. This mechanism relies on the changes proposed by [EIP-6110](http://eips.ethereum.org/EIPS/eip-6110). -*Note:* This specification is built upon [Capella](../../capella/beacon_chain.md) and is under active development. +*Note:* This specification is built upon [Deneb](../../deneb/beacon_chain.md) and is under active development. ## Constants @@ -91,7 +91,8 @@ class ExecutionPayload(Container): block_hash: Hash32 transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] - deposit_receipts: List[DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD] # [New in EIP-6110] + excess_data_gas: uint256 + deposit_receipts: List[DepositReceipt, MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD] # [New in EIP6110] ``` #### `ExecutionPayloadHeader` @@ -115,7 +116,8 @@ class ExecutionPayloadHeader(Container): block_hash: Hash32 transactions_root: Root withdrawals_root: Root - deposit_receipts_root: Root # [New in EIP-6110] + excess_data_gas: uint256 + deposit_receipts_root: Root # [New in EIP6110] ``` #### `BeaconState` @@ -157,13 +159,13 @@ class BeaconState(Container): current_sync_committee: SyncCommittee next_sync_committee: SyncCommittee # Execution - latest_execution_payload_header: ExecutionPayloadHeader # [Modified in EIP-6110] + latest_execution_payload_header: ExecutionPayloadHeader # [Modified in EIP6110] # Withdrawals next_withdrawal_index: WithdrawalIndex next_withdrawal_validator_index: ValidatorIndex # Deep history valid from Capella onwards historical_summaries: List[HistoricalSummary, HISTORICAL_ROOTS_LIMIT] - # [New in EIP-6110] + # [New in EIP6110] deposit_receipts_start_index: uint64 ``` @@ -176,11 +178,12 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) if is_execution_enabled(state, block.body): process_withdrawals(state, block.body.execution_payload) - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in EIP-6110] + process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in EIP6110] process_randao(state, block.body) process_eth1_data(state, block.body) - process_operations(state, block.body) # [Modified in EIP-6110] + process_operations(state, block.body) # [Modified in EIP6110] process_sync_aggregate(state, block.body.sync_aggregate) + process_blob_kzg_commitments(state, block.body) ``` #### Modified `process_operations` @@ -189,7 +192,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: ```python def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: - # [Modified in EIP-6110] + # [Modified in EIP6110] # Disable former deposit mechanism once all prior deposits are processed eth1_deposit_index_limit = min(state.eth1_data.deposit_count, state.deposit_receipts_start_index) if state.eth1_deposit_index < eth1_deposit_index_limit: @@ -208,7 +211,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: for_ops(body.voluntary_exits, process_voluntary_exit) for_ops(body.bls_to_execution_changes, process_bls_to_execution_change) - # [New in EIP-6110] + # [New in EIP6110] if is_execution_enabled(state, body): for_ops(body.execution_payload.deposit_receipts, process_deposit_receipt) ``` @@ -262,7 +265,8 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe block_hash=payload.block_hash, transactions_root=hash_tree_root(payload.transactions), withdrawals_root=hash_tree_root(payload.withdrawals), - deposit_receipts_root=hash_tree_root(payload.deposit_receipts), # [New in EIP-6110] + excess_data_gas=payload.excess_data_gas, + deposit_receipts_root=hash_tree_root(payload.deposit_receipts), # [New in EIP6110] ) ``` diff --git a/specs/_features/eip6110/fork.md b/specs/_features/eip6110/fork.md index df98c4c69e..2145a9d1a3 100644 --- a/specs/_features/eip6110/fork.md +++ b/specs/_features/eip6110/fork.md @@ -44,6 +44,8 @@ def compute_fork_version(epoch: Epoch) -> Version: """ if epoch >= EIP6110_FORK_EPOCH: return EIP6110_FORK_VERSION + if epoch >= DENEB_FORK_EPOCH: + return DENEB_FORK_VERSION if epoch >= CAPELLA_FORK_EPOCH: return CAPELLA_FORK_VERSION if epoch >= BELLATRIX_FORK_EPOCH: @@ -68,8 +70,8 @@ If `state.slot % SLOTS_PER_EPOCH == 0` and `compute_epoch_at_slot(state.slot) == an irregular state change is made to upgrade to EIP-6110. ```python -def upgrade_to_eip6110(pre: capella.BeaconState) -> BeaconState: - epoch = capella.get_current_epoch(pre) +def upgrade_to_eip6110(pre: deneb.BeaconState) -> BeaconState: + epoch = deneb.get_current_epoch(pre) latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=pre.latest_execution_payload_header.parent_hash, fee_recipient=pre.latest_execution_payload_header.fee_recipient, @@ -86,6 +88,7 @@ def upgrade_to_eip6110(pre: capella.BeaconState) -> BeaconState: block_hash=pre.latest_execution_payload_header.block_hash, transactions_root=pre.latest_execution_payload_header.transactions_root, withdrawals_root=pre.latest_execution_payload_header.withdrawals_root, + excess_data_gas=uint256(0), deposit_receipts_root=Root(), # [New in EIP-6110] ) post = BeaconState( diff --git a/specs/_features/eip6110/light-client/fork.md b/specs/_features/eip6110/light-client/fork.md index 6ffa3d8697..2aaae3d948 100644 --- a/specs/_features/eip6110/light-client/fork.md +++ b/specs/_features/eip6110/light-client/fork.md @@ -15,14 +15,14 @@ ## Introduction -This document describes how to upgrade existing light client objects based on the [Capella specification](../../capella/light-client/sync-protocol.md) to eip6110. This is necessary when processing pre-eip6110 data with a post-eip6110 `LightClientStore`. Note that the data being exchanged over the network protocols uses the original format. +This document describes how to upgrade existing light client objects based on the [Deneb specification](../../deneb/light-client/sync-protocol.md) to eip6110. This is necessary when processing pre-eip6110 data with a post-eip6110 `LightClientStore`. Note that the data being exchanged over the network protocols uses the original format. ### Upgrading light client data A eip6110 `LightClientStore` can still process earlier light client data. In order to do so, that pre-eip6110 data needs to be locally upgraded to eip6110 before processing. ```python -def upgrade_lc_header_to_eip6110(pre: capella.LightClientHeader) -> LightClientHeader: +def upgrade_lc_header_to_eip6110(pre: deneb.LightClientHeader) -> LightClientHeader: return LightClientHeader( beacon=pre.beacon, execution=ExecutionPayloadHeader( @@ -41,14 +41,15 @@ def upgrade_lc_header_to_eip6110(pre: capella.LightClientHeader) -> LightClientH block_hash=pre.execution.block_hash, transactions_root=pre.execution.transactions_root, withdrawals_root=pre.execution.withdrawals_root, - deposit_receipts_root=Root(), + excess_data_gas=pre.execution.excess_data_gas, + deposit_receipts_root=Root(), # [New in EIP6110] ), execution_branch=pre.execution_branch, ) ``` ```python -def upgrade_lc_bootstrap_to_eip6110(pre: capella.LightClientBootstrap) -> LightClientBootstrap: +def upgrade_lc_bootstrap_to_eip6110(pre: deneb.LightClientBootstrap) -> LightClientBootstrap: return LightClientBootstrap( header=upgrade_lc_header_to_eip6110(pre.header), current_sync_committee=pre.current_sync_committee, @@ -57,7 +58,7 @@ def upgrade_lc_bootstrap_to_eip6110(pre: capella.LightClientBootstrap) -> LightC ``` ```python -def upgrade_lc_update_to_eip6110(pre: capella.LightClientUpdate) -> LightClientUpdate: +def upgrade_lc_update_to_eip6110(pre: deneb.LightClientUpdate) -> LightClientUpdate: return LightClientUpdate( attested_header=upgrade_lc_header_to_eip6110(pre.attested_header), next_sync_committee=pre.next_sync_committee, @@ -70,7 +71,7 @@ def upgrade_lc_update_to_eip6110(pre: capella.LightClientUpdate) -> LightClientU ``` ```python -def upgrade_lc_finality_update_to_eip6110(pre: capella.LightClientFinalityUpdate) -> LightClientFinalityUpdate: +def upgrade_lc_finality_update_to_eip6110(pre: deneb.LightClientFinalityUpdate) -> LightClientFinalityUpdate: return LightClientFinalityUpdate( attested_header=upgrade_lc_header_to_eip6110(pre.attested_header), finalized_header=upgrade_lc_header_to_eip6110(pre.finalized_header), @@ -81,7 +82,7 @@ def upgrade_lc_finality_update_to_eip6110(pre: capella.LightClientFinalityUpdate ``` ```python -def upgrade_lc_optimistic_update_to_eip6110(pre: capella.LightClientOptimisticUpdate) -> LightClientOptimisticUpdate: +def upgrade_lc_optimistic_update_to_eip6110(pre: deneb.LightClientOptimisticUpdate) -> LightClientOptimisticUpdate: return LightClientOptimisticUpdate( attested_header=upgrade_lc_header_to_eip6110(pre.attested_header), sync_aggregate=pre.sync_aggregate, @@ -91,10 +92,10 @@ def upgrade_lc_optimistic_update_to_eip6110(pre: capella.LightClientOptimisticUp ### Upgrading the store -Existing `LightClientStore` objects based on Capella MUST be upgraded to eip6110 before eip6110 based light client data can be processed. The `LightClientStore` upgrade MAY be performed before `eip6110_FORK_EPOCH`. +Existing `LightClientStore` objects based on Deneb MUST be upgraded to eip6110 before eip6110 based light client data can be processed. The `LightClientStore` upgrade MAY be performed before `EIP6110_FORK_EPOCH`. ```python -def upgrade_lc_store_to_eip6110(pre: capella.LightClientStore) -> LightClientStore: +def upgrade_lc_store_to_eip6110(pre: deneb.LightClientStore) -> LightClientStore: if pre.best_valid_update is None: best_valid_update = None else: diff --git a/specs/_features/eip6110/light-client/full-node.md b/specs/_features/eip6110/light-client/full-node.md index 0e400dd161..27b7b30637 100644 --- a/specs/_features/eip6110/light-client/full-node.md +++ b/specs/_features/eip6110/light-client/full-node.md @@ -47,7 +47,10 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader: withdrawals_root=hash_tree_root(payload.withdrawals), ) - # [New in Deneb] + if epoch >= DENEB_FORK_EPOCH: + execution_header.excess_data_gas = payload.excess_data_gas + + # [New in EIP6110] if epoch >= EIP6110_FORK_EPOCH: execution_header.deposit_receipts_root = hash_tree_root(payload.deposit_receipts) diff --git a/specs/_features/eip6110/light-client/p2p-interface.md b/specs/_features/eip6110/light-client/p2p-interface.md index 9b33d59ffd..f55fb2f77e 100644 --- a/specs/_features/eip6110/light-client/p2p-interface.md +++ b/specs/_features/eip6110/light-client/p2p-interface.md @@ -26,7 +26,7 @@ ## Networking -The [Capella light client networking specification](../../capella/light-client/p2p-interface.md) is extended to exchange [EIP-6110 light client data](./sync-protocol.md). +The [Deneb light client networking specification](../../deneb/light-client/p2p-interface.md) is extended to exchange [EIP-6110 light client data](./sync-protocol.md). ### The gossip domain: gossipsub @@ -43,6 +43,7 @@ The [Capella light client networking specification](../../capella/light-client/p | `GENESIS_FORK_VERSION` | n/a | | `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientFinalityUpdate` | | `CAPELLA_FORK_VERSION` | `capella.LightClientFinalityUpdate` | +| `DENEB_FORK_VERSION` | `deneb.LightClientFinalityUpdate` | | `EIP6110_FORK_VERSION` and later | `eip6110.LightClientFinalityUpdate` | ###### `light_client_optimistic_update` @@ -54,6 +55,7 @@ The [Capella light client networking specification](../../capella/light-client/p | `GENESIS_FORK_VERSION` | n/a | | `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientOptimisticUpdate` | | `CAPELLA_FORK_VERSION` | `capella.LightClientOptimisticUpdate` | +| `DENEB_FORK_VERSION` | `deneb.LightClientOptimisticUpdate` | | `EIP6110_FORK_VERSION` and later | `eip6110.LightClientOptimisticUpdate` | ### The Req/Resp domain @@ -69,6 +71,7 @@ The [Capella light client networking specification](../../capella/light-client/p | `GENESIS_FORK_VERSION` | n/a | | `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientBootstrap` | | `CAPELLA_FORK_VERSION` | `capella.LightClientBootstrap` | +| `DENEB_FORK_VERSION` | `deneb.LightClientBootstrap` | | `EIP6110_FORK_VERSION` and later | `eip6110.LightClientBootstrap` | ##### LightClientUpdatesByRange @@ -80,6 +83,7 @@ The [Capella light client networking specification](../../capella/light-client/p | `GENESIS_FORK_VERSION` | n/a | | `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientUpdate` | | `CAPELLA_FORK_VERSION` | `capella.LightClientUpdate` | +| `DENEB_FORK_VERSION` | `deneb.LightClientUpdate` | | `EIP6110_FORK_VERSION` and later | `eip6110.LightClientUpdate` | ##### GetLightClientFinalityUpdate @@ -91,6 +95,7 @@ The [Capella light client networking specification](../../capella/light-client/p | `GENESIS_FORK_VERSION` | n/a | | `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientFinalityUpdate` | | `CAPELLA_FORK_VERSION` | `capella.LightClientFinalityUpdate` | +| `DENEB_FORK_VERSION` | `deneb.LightClientFinalityUpdate` | | `EIP6110_FORK_VERSION` and later | `eip6110.LightClientFinalityUpdate` | ##### GetLightClientOptimisticUpdate @@ -102,4 +107,5 @@ The [Capella light client networking specification](../../capella/light-client/p | `GENESIS_FORK_VERSION` | n/a | | `ALTAIR_FORK_VERSION` through `BELLATRIX_FORK_VERSION` | `altair.LightClientOptimisticUpdate` | | `CAPELLA_FORK_VERSION` | `capella.LightClientOptimisticUpdate` | +| `DENEB_FORK_VERSION` | `deneb.LightClientOptimisticUpdate` | | `EIP6110_FORK_VERSION` and later | `eip6110.LightClientOptimisticUpdate` | diff --git a/specs/_features/eip6110/light-client/sync-protocol.md b/specs/_features/eip6110/light-client/sync-protocol.md index 867aa2730d..406a4b9347 100644 --- a/specs/_features/eip6110/light-client/sync-protocol.md +++ b/specs/_features/eip6110/light-client/sync-protocol.md @@ -18,7 +18,7 @@ ## Introduction -This upgrade updates light client data to include the EIP-6110 changes to the [`ExecutionPayload`](../beacon-chain.md) structure. It extends the [Capella Light Client specifications](../../capella/light-client/sync-protocol.md). The [fork document](./fork.md) explains how to upgrade existing Capella based deployments to EIP-6110. +This upgrade updates light client data to include the EIP-6110 changes to the [`ExecutionPayload`](../beacon-chain.md) structure. It extends the [Deneb Light Client specifications](../../deneb/light-client/sync-protocol.md). The [fork document](./fork.md) explains how to upgrade existing Deneb based deployments to EIP-6110. Additional documents describes the impact of the upgrade on certain roles: - [Full node](./full-node.md) @@ -32,11 +32,9 @@ Additional documents describes the impact of the upgrade on certain roles: def get_lc_execution_root(header: LightClientHeader) -> Root: epoch = compute_epoch_at_slot(header.beacon.slot) - # [New in EIP-6110] - if epoch >= EIP6110_FORK_EPOCH: + if epoch >= DENEB_FORK_EPOCH: return hash_tree_root(header.execution) - # [Modified in EIP-6110] if epoch >= CAPELLA_FORK_EPOCH: execution_header = capella.ExecutionPayloadHeader( parent_hash=header.execution.parent_hash, @@ -68,7 +66,11 @@ def is_valid_light_client_header(header: LightClientHeader) -> bool: # [New in EIP-6110] if epoch < EIP6110_FORK_EPOCH: - if header.execution.withdrawals_root != Root(): + if header.execution.deposit_receipts_root != Root(): + return False + + if epoch < DENEB_FORK_EPOCH: + if header.execution.excess_data_gas != uint256(0): return False if epoch < CAPELLA_FORK_EPOCH: diff --git a/specs/_features/eip6110/validator.md b/specs/_features/eip6110/validator.md index ae9d493a6f..6770ef56af 100644 --- a/specs/_features/eip6110/validator.md +++ b/specs/_features/eip6110/validator.md @@ -20,7 +20,7 @@ This document represents the changes to be made in the code of an "honest valida ## Prerequisites -This document is an extension of the [Capella -- Honest Validator](../../capella/validator.md) guide. +This document is an extension of the [Deneb -- Honest Validator](../../deneb/validator.md) guide. All behaviors and definitions defined in this document, and documents it extends, carry over unless explicitly noted or overridden. All terminology, constants, functions, and protocol mechanics defined in the updated Beacon Chain doc of [EIP-6110](./beacon-chain.md) are requisite for this document and used throughout. diff --git a/specs/deneb/light-client/fork.md b/specs/deneb/light-client/fork.md index 8c552937a5..46a0930283 100644 --- a/specs/deneb/light-client/fork.md +++ b/specs/deneb/light-client/fork.md @@ -41,6 +41,7 @@ def upgrade_lc_header_to_deneb(pre: capella.LightClientHeader) -> LightClientHea block_hash=pre.execution.block_hash, transactions_root=pre.execution.transactions_root, withdrawals_root=pre.execution.withdrawals_root, + excess_data_gas=uint256(0), # [New in Deneb] ), execution_branch=pre.execution_branch, ) diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py index 7d89a9788e..6d3f377a33 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py @@ -4,7 +4,8 @@ spec_test, single_phase, with_deneb_and_later, - expect_assertion_error + expect_assertion_error, + always_bls ) from eth2spec.test.helpers.sharding import ( get_sample_blob, @@ -263,6 +264,7 @@ def test_validate_kzg_g1_neutral_element(spec): @with_deneb_and_later @spec_test @single_phase +@always_bls def test_validate_kzg_g1_not_in_g1(spec): """ Verify that `validate_kzg_g1` fails on point not in G1 @@ -274,6 +276,7 @@ def test_validate_kzg_g1_not_in_g1(spec): @with_deneb_and_later @spec_test @single_phase +@always_bls def test_validate_kzg_g1_not_on_curve(spec): """ Verify that `validate_kzg_g1` fails on point not in G1 diff --git a/tests/core/pyspec/eth2spec/test/helpers/forks.py b/tests/core/pyspec/eth2spec/test/helpers/forks.py index e6320cc9b3..5e97522dbb 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/forks.py +++ b/tests/core/pyspec/eth2spec/test/helpers/forks.py @@ -6,7 +6,7 @@ def is_post_fork(a, b): if a == EIP6110: - return b in [PHASE0, ALTAIR, BELLATRIX, CAPELLA, EIP6110] + return b in [PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110] if a == DENEB: return b in [PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB] if a == CAPELLA: From 3e7e780b7722ce7a33bc47961acd31e2927998d8 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 6 Apr 2023 17:04:49 +0600 Subject: [PATCH 029/117] Apply suggestions from code review Co-authored-by: Hsiao-Wei Wang --- specs/_features/eip6110/light-client/fork.md | 2 +- specs/_features/eip6110/light-client/full-node.md | 4 ++-- specs/_features/eip6110/light-client/sync-protocol.md | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/_features/eip6110/light-client/fork.md b/specs/_features/eip6110/light-client/fork.md index 2aaae3d948..34f0fef8ce 100644 --- a/specs/_features/eip6110/light-client/fork.md +++ b/specs/_features/eip6110/light-client/fork.md @@ -1,4 +1,4 @@ -# eip6110 Light Client -- Fork Logic +# EIP-6110 Light Client -- Fork Logic ## Table of contents diff --git a/specs/_features/eip6110/light-client/full-node.md b/specs/_features/eip6110/light-client/full-node.md index 27b7b30637..03c0f17bd8 100644 --- a/specs/_features/eip6110/light-client/full-node.md +++ b/specs/_features/eip6110/light-client/full-node.md @@ -1,4 +1,4 @@ -# Deneb Light Client -- Full Node +# EIP-6110 Light Client -- Full Node **Notice**: This document is a work-in-progress for researchers and implementers. @@ -17,7 +17,7 @@ ## Introduction -This upgrade adds information about the execution payload to light client data as part of the Deneb upgrade. +This upgrade adds information about the execution payload to light client data as part of the EIP-6110 upgrade. ## Helper functions diff --git a/specs/_features/eip6110/light-client/sync-protocol.md b/specs/_features/eip6110/light-client/sync-protocol.md index 406a4b9347..bcb9d50e43 100644 --- a/specs/_features/eip6110/light-client/sync-protocol.md +++ b/specs/_features/eip6110/light-client/sync-protocol.md @@ -1,4 +1,4 @@ -# Deneb Light Client -- Sync Protocol +# EIP-6110 Light Client -- Sync Protocol **Notice**: This document is a work-in-progress for researchers and implementers. From 389b79408b1ce9a6a075f7338dabc72c8d2a377a Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Thu, 6 Apr 2023 17:39:05 +0600 Subject: [PATCH 030/117] Add EIP6110 operations gen, and to fork upgrades list --- tests/core/pyspec/eth2spec/test/helpers/constants.py | 4 ++-- tests/formats/operations/README.md | 1 + tests/generators/operations/main.py | 6 ++++++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/constants.py b/tests/core/pyspec/eth2spec/test/helpers/constants.py index 83e7e40dbb..2140c96e45 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/constants.py +++ b/tests/core/pyspec/eth2spec/test/helpers/constants.py @@ -27,13 +27,13 @@ # The forks that output to the test vectors. TESTGEN_FORKS = (PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110) -# TODO: no EIP6110 fork tests now. ALL_FORK_UPGRADES = { # pre_fork_name: post_fork_name PHASE0: ALTAIR, ALTAIR: BELLATRIX, BELLATRIX: CAPELLA, CAPELLA: DENEB, + DENEB: EIP6110, } ALL_PRE_POST_FORKS = ALL_FORK_UPGRADES.items() AFTER_BELLATRIX_UPGRADES = {key: value for key, value in ALL_FORK_UPGRADES.items() if key != PHASE0} @@ -42,7 +42,7 @@ if key not in [PHASE0, ALTAIR]} AFTER_CAPELLA_PRE_POST_FORKS = AFTER_CAPELLA_UPGRADES.items() AFTER_DENEB_UPGRADES = {key: value for key, value in ALL_FORK_UPGRADES.items() - if key not in [PHASE0, ALTAIR, BELLATRIX, EIP6110]} + if key not in [PHASE0, ALTAIR, BELLATRIX]} AFTER_DENEB_PRE_POST_FORKS = AFTER_DENEB_UPGRADES.items() # diff --git a/tests/formats/operations/README.md b/tests/formats/operations/README.md index 810d62578e..245ce85653 100644 --- a/tests/formats/operations/README.md +++ b/tests/formats/operations/README.md @@ -45,6 +45,7 @@ Operations: | `execution_payload` | `ExecutionPayload` | `execution_payload` | `process_execution_payload(state, execution_payload)` (new in Bellatrix) | | `withdrawals` | `ExecutionPayload` | `execution_payload` | `process_withdrawals(state, execution_payload)` (new in Capella) | | `bls_to_execution_change` | `SignedBLSToExecutionChange` | `address_change` | `process_bls_to_execution_change(state, address_change)` (new in Capella) | +| `deposit_receipt` | `DepositReceipt` | `deposit_receipt` | `process_deposit_receipt(state, deposit_receipt)` (new in EIP6110) | Note that `block_header` is not strictly an operation (and is a full `Block`), but processed in the same manner, and hence included here. diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index ed4c6c26c8..7b382c838b 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -38,6 +38,11 @@ deneb_mods = capella_mods + _new_eip6110_mods = {key: 'eth2spec.test.eip6110.block_processing.test_process_' + key for key in [ + 'deposit_receipt', + ]} + eip6110_mods = combine_mods(_new_eip6110_mods, deneb_mods) + # TODO Custody Game testgen is disabled for now # _new_custody_game_mods = {key: 'eth2spec.test.custody_game.block_processing.test_process_' + key for key in [ # 'attestation', @@ -54,6 +59,7 @@ BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, DENEB: deneb_mods, + EIP6110: eip6110_mods, } run_state_test_generators(runner_name="operations", all_mods=all_mods) From 11842c9e2a1d18227d4c12c6ea5aa0d19fa49656 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 6 Apr 2023 20:04:28 +0800 Subject: [PATCH 031/117] Update test generators. Move `test_deposit_transition` to `sanity/blocks/` --- tests/core/pyspec/eth2spec/test/eip6110/__init__.py | 0 .../eth2spec/test/eip6110/block_processing/__init__.py | 0 .../core/pyspec/eth2spec/test/eip6110/sanity/__init__.py | 0 .../eth2spec/test/eip6110/sanity/blocks/__init__.py | 1 + .../blocks}/test_deposit_transition.py | 0 tests/generators/operations/main.py | 2 +- tests/generators/sanity/main.py | 8 +++++++- 7 files changed, 9 insertions(+), 2 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/eip6110/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/eip6110/block_processing/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/eip6110/sanity/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/__init__.py rename tests/core/pyspec/eth2spec/test/eip6110/{block_processing => sanity/blocks}/test_deposit_transition.py (100%) diff --git a/tests/core/pyspec/eth2spec/test/eip6110/__init__.py b/tests/core/pyspec/eth2spec/test/eip6110/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/pyspec/eth2spec/test/eip6110/block_processing/__init__.py b/tests/core/pyspec/eth2spec/test/eip6110/block_processing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/pyspec/eth2spec/test/eip6110/sanity/__init__.py b/tests/core/pyspec/eth2spec/test/eip6110/sanity/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/__init__.py b/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/__init__.py new file mode 100644 index 0000000000..3c0e060f3d --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/__init__.py @@ -0,0 +1 @@ +from .test_deposit_transition import * # noqa: F401 F403 diff --git a/tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_deposit_transition.py b/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/test_deposit_transition.py similarity index 100% rename from tests/core/pyspec/eth2spec/test/eip6110/block_processing/test_deposit_transition.py rename to tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/test_deposit_transition.py diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index 7b382c838b..fc22179176 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -1,5 +1,5 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110 if __name__ == "__main__": diff --git a/tests/generators/sanity/main.py b/tests/generators/sanity/main.py index 8a6c7b39cc..b9f6d7fbb1 100644 --- a/tests/generators/sanity/main.py +++ b/tests/generators/sanity/main.py @@ -1,4 +1,4 @@ -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110 from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods @@ -28,12 +28,18 @@ ]} deneb_mods = combine_mods(_new_deneb_mods, capella_mods) + _new_eip6110_mods = {key: 'eth2spec.test.eip6110.sanity.' + key for key in [ + 'blocks', + ]} + eip6110_mods = combine_mods(_new_eip6110_mods, deneb_mods) + all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, DENEB: deneb_mods, + EIP6110: eip6110_mods, } run_state_test_generators(runner_name="sanity", all_mods=all_mods) From d78c7ada03bc497ed76ae55fa212a1c9d73726dc Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 6 Apr 2023 20:08:23 +0800 Subject: [PATCH 032/117] Fix previous fork version --- tests/core/pyspec/eth2spec/test/helpers/fork_transition.py | 4 +++- tests/core/pyspec/eth2spec/test/helpers/genesis.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py b/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py index 20c20b938c..68444c4726 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py +++ b/tests/core/pyspec/eth2spec/test/helpers/fork_transition.py @@ -159,6 +159,8 @@ def do_fork(state, spec, post_spec, fork_epoch, with_block=True, sync_aggregate= state = post_spec.upgrade_to_capella(state) elif post_spec.fork == DENEB: state = post_spec.upgrade_to_deneb(state) + elif post_spec.fork == EIP6110: + state = post_spec.upgrade_to_eip6110(state) assert state.fork.epoch == fork_epoch @@ -175,7 +177,7 @@ def do_fork(state, spec, post_spec, fork_epoch, with_block=True, sync_aggregate= assert state.fork.previous_version == post_spec.config.CAPELLA_FORK_VERSION assert state.fork.current_version == post_spec.config.DENEB_FORK_VERSION elif post_spec.fork == EIP6110: - assert state.fork.previous_version == post_spec.config.CAPELLA_FORK_VERSION + assert state.fork.previous_version == post_spec.config.DENEB_FORK_VERSION assert state.fork.current_version == post_spec.config.EIP6110_FORK_VERSION if with_block: diff --git a/tests/core/pyspec/eth2spec/test/helpers/genesis.py b/tests/core/pyspec/eth2spec/test/helpers/genesis.py index 32ce8974d9..fea259013b 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/genesis.py +++ b/tests/core/pyspec/eth2spec/test/helpers/genesis.py @@ -84,7 +84,7 @@ def create_genesis_state(spec, validator_balances, activation_threshold): previous_version = spec.config.CAPELLA_FORK_VERSION current_version = spec.config.DENEB_FORK_VERSION elif spec.fork == EIP6110: - previous_version = spec.config.CAPELLA_FORK_VERSION + previous_version = spec.config.DENEB_FORK_VERSION current_version = spec.config.EIP6110_FORK_VERSION state = spec.BeaconState( From c2473e7b8a3ca5b346abf7ea93b298345c557e07 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Fri, 7 Apr 2023 14:20:28 +0600 Subject: [PATCH 033/117] Make transition tests comply to sanity format --- .../test/eip6110/sanity/blocks/test_deposit_transition.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/test_deposit_transition.py b/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/test_deposit_transition.py index 92c3a96b55..2e03f1b0d7 100644 --- a/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/test_deposit_transition.py +++ b/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/test_deposit_transition.py @@ -1,5 +1,6 @@ from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, + sign_block, ) from eth2spec.test.context import ( spec_state_test, @@ -27,7 +28,8 @@ def run_deposit_transition_block(spec, state, block, top_up_keys=[], valid=True) If ``valid == False``, run expecting ``AssertionError`` """ yield 'pre', state - yield 'block', block + signed_block = sign_block(spec, state, block, proposer_index=block.proposer_index) + yield 'blocks', [signed_block] if not valid: expect_assertion_error(lambda: spec.process_block(state, block)) From 41386092b761d61792c2ad0d2c07a76c4511a823 Mon Sep 17 00:00:00 2001 From: Roberto Saltini Date: Sat, 8 Apr 2023 14:00:01 +1000 Subject: [PATCH 034/117] Apply changes to p2p-interface.md --- specs/phase0/p2p-interface.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index f527529316..56c1b8cfb1 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -317,7 +317,7 @@ The following validations MUST pass before forwarding the `signed_beacon_block` - _[REJECT]_ The block's parent (defined by `block.parent_root`) passes validation. - _[REJECT]_ The block is from a higher slot than its parent. - _[REJECT]_ The current `finalized_checkpoint` is an ancestor of `block` -- i.e. - `get_ancestor(store, block.parent_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) + `get_ancestor_at_epoch_boundary(store, block.parent_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root` - _[REJECT]_ The block is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `parent_root`/`slot`). @@ -356,7 +356,7 @@ The following validations MUST pass before forwarding the `signed_aggregate_and_ (a client MAY queue aggregates for processing once block is retrieved). - _[REJECT]_ The block being voted for (`aggregate.data.beacon_block_root`) passes validation. - _[IGNORE]_ The current `finalized_checkpoint` is an ancestor of the `block` defined by `aggregate.data.beacon_block_root` -- i.e. - `get_ancestor(store, aggregate.data.beacon_block_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) + `get_ancestor_at_epoch_boundary(store, aggregate.data.beacon_block_root, finalized_checkpoint.epoch) == store.finalized_checkpoint.root` @@ -425,9 +425,9 @@ The following validations MUST pass before forwarding the `attestation` on the s (a client MAY queue attestations for processing once block is retrieved). - _[REJECT]_ The block being voted for (`attestation.data.beacon_block_root`) passes validation. - _[REJECT]_ The attestation's target block is an ancestor of the block named in the LMD vote -- i.e. - `get_ancestor(store, attestation.data.beacon_block_root, compute_start_slot_at_epoch(attestation.data.target.epoch)) == attestation.data.target.root` + `get_ancestor_at_epoch_boundary(store, attestation.data.beacon_block_root, attestation.data.target.epoch) == attestation.data.target.root` - _[IGNORE]_ The current `finalized_checkpoint` is an ancestor of the `block` defined by `attestation.data.beacon_block_root` -- i.e. - `get_ancestor(store, attestation.data.beacon_block_root, compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)) + `get_ancestor_at_epoch_boundary(store, attestation.data.beacon_block_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root` From e49a30f85ba7eef5ec22f6c17a260c6731f5c839 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Mon, 10 Apr 2023 13:47:21 +0600 Subject: [PATCH 035/117] Fix deposit transition tests --- .../sanity/blocks/test_deposit_transition.py | 59 ++++++++----------- 1 file changed, 25 insertions(+), 34 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/test_deposit_transition.py b/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/test_deposit_transition.py index 2e03f1b0d7..1477a04fb6 100644 --- a/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/test_deposit_transition.py +++ b/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/test_deposit_transition.py @@ -17,6 +17,9 @@ compute_el_block_hash, ) from eth2spec.test.helpers.keys import privkeys, pubkeys +from eth2spec.test.helpers.state import ( + state_transition_and_sign_block +) def run_deposit_transition_block(spec, state, block, top_up_keys=[], valid=True): @@ -28,24 +31,20 @@ def run_deposit_transition_block(spec, state, block, top_up_keys=[], valid=True) If ``valid == False``, run expecting ``AssertionError`` """ yield 'pre', state - signed_block = sign_block(spec, state, block, proposer_index=block.proposer_index) - yield 'blocks', [signed_block] - if not valid: - expect_assertion_error(lambda: spec.process_block(state, block)) - yield 'post', None - return + signed_block = state_transition_and_sign_block(spec, state, block, not valid) - spec.process_block(state, block) - yield 'post', state + yield 'blocks', [signed_block] + yield 'post', state if valid else None # Check that deposits are applied - expected_pubkeys = [d.data.pubkey for d in block.body.deposits] - deposit_receipts = block.body.execution_payload.deposit_receipts - expected_pubkeys = expected_pubkeys + [d.pubkey for d in deposit_receipts if (d.pubkey not in top_up_keys)] - actual_pubkeys = [v.pubkey for v in state.validators[len(state.validators) - len(expected_pubkeys):]] + if valid: + expected_pubkeys = [d.data.pubkey for d in block.body.deposits] + deposit_receipts = block.body.execution_payload.deposit_receipts + expected_pubkeys = expected_pubkeys + [d.pubkey for d in deposit_receipts if (d.pubkey not in top_up_keys)] + actual_pubkeys = [v.pubkey for v in state.validators[len(state.validators) - len(expected_pubkeys):]] - assert actual_pubkeys == expected_pubkeys + assert actual_pubkeys == expected_pubkeys def prepare_state_and_block(spec, @@ -53,7 +52,8 @@ def prepare_state_and_block(spec, deposit_cnt, deposit_receipt_cnt, first_deposit_receipt_index=0, - deposit_receipts_start_index=None): + deposit_receipts_start_index=None, + eth1_data_deposit_count=None): deposits = [] deposit_receipts = [] keypair_index = len(state.validators) @@ -79,8 +79,10 @@ def prepare_state_and_block(spec, if deposit_root: state.eth1_deposit_index = 0 + if not eth1_data_deposit_count: + eth1_data_deposit_count = deposit_cnt state.eth1_data = spec.Eth1Data(deposit_root=deposit_root, - deposit_count=deposit_cnt, + deposit_count=eth1_data_deposit_count, block_hash=state.eth1_data.block_hash) # Prepare deposit receipts @@ -105,9 +107,6 @@ def prepare_state_and_block(spec, block.body.execution_payload.deposit_receipts = deposit_receipts block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - # Advance a slot - spec.process_slots(state, block.slot) - return state, block @@ -148,10 +147,8 @@ def test_deposit_transition__process_max_eth1_deposits(spec, state): deposit_cnt=spec.MAX_DEPOSITS, deposit_receipt_cnt=1, first_deposit_receipt_index=spec.MAX_DEPOSITS + 1, - deposit_receipts_start_index=spec.MAX_DEPOSITS) - state.eth1_data = spec.Eth1Data(deposit_root=state.eth1_data.deposit_root, - deposit_count=23, - block_hash=state.eth1_data.block_hash) + deposit_receipts_start_index=spec.MAX_DEPOSITS, + eth1_data_deposit_count=23) yield from run_deposit_transition_block(spec, state, block) @@ -177,10 +174,8 @@ def test_deposit_transition__invalid_not_enough_eth1_deposits(spec, state): deposit_cnt=3, deposit_receipt_cnt=1, first_deposit_receipt_index=29, - deposit_receipts_start_index=23) - state.eth1_data = spec.Eth1Data(deposit_root=state.eth1_data.deposit_root, - deposit_count=17, - block_hash=state.eth1_data.block_hash) + deposit_receipts_start_index=23, + eth1_data_deposit_count=17) yield from run_deposit_transition_block(spec, state, block, valid=False) @@ -193,10 +188,8 @@ def test_deposit_transition__invalid_too_many_eth1_deposits(spec, state): deposit_cnt=3, deposit_receipt_cnt=1, first_deposit_receipt_index=11, - deposit_receipts_start_index=7) - state.eth1_data = spec.Eth1Data(deposit_root=state.eth1_data.deposit_root, - deposit_count=2, - block_hash=state.eth1_data.block_hash) + deposit_receipts_start_index=7, + eth1_data_deposit_count=2) yield from run_deposit_transition_block(spec, state, block, valid=False) @@ -210,10 +203,8 @@ def test_deposit_transition__invalid_eth1_deposits_overlap_in_protocol_deposits( deposit_cnt=spec.MAX_DEPOSITS, deposit_receipt_cnt=1, first_deposit_receipt_index=spec.MAX_DEPOSITS, - deposit_receipts_start_index=spec.MAX_DEPOSITS - 1) - state.eth1_data = spec.Eth1Data(deposit_root=state.eth1_data.deposit_root, - deposit_count=23, - block_hash=state.eth1_data.block_hash) + deposit_receipts_start_index=spec.MAX_DEPOSITS - 1, + eth1_data_deposit_count=23) yield from run_deposit_transition_block(spec, state, block, valid=False) From 1505a04a94791cffdcc25e3171ac2ee306ae2ff9 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Mon, 10 Apr 2023 16:08:13 +0600 Subject: [PATCH 036/117] Make linter happy --- .../test/eip6110/sanity/blocks/test_deposit_transition.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/test_deposit_transition.py b/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/test_deposit_transition.py index 1477a04fb6..51ef109605 100644 --- a/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/test_deposit_transition.py +++ b/tests/core/pyspec/eth2spec/test/eip6110/sanity/blocks/test_deposit_transition.py @@ -1,12 +1,10 @@ from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, - sign_block, ) from eth2spec.test.context import ( spec_state_test, with_phases, EIP6110, - expect_assertion_error, ) from eth2spec.test.helpers.deposits import ( build_deposit_data, From 1c5b9fddaf5069e642a16180aac73364e289754a Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Tue, 11 Apr 2023 13:34:42 +0900 Subject: [PATCH 037/117] add `EIP-4788` feature --- specs/_features/eip4788/beacon-chain.md | 72 +++++++++++++++++ specs/_features/eip4788/validator.md | 103 ++++++++++++++++++++++++ 2 files changed, 175 insertions(+) create mode 100644 specs/_features/eip4788/beacon-chain.md create mode 100644 specs/_features/eip4788/validator.md diff --git a/specs/_features/eip4788/beacon-chain.md b/specs/_features/eip4788/beacon-chain.md new file mode 100644 index 0000000000..6cd876de99 --- /dev/null +++ b/specs/_features/eip4788/beacon-chain.md @@ -0,0 +1,72 @@ +# EIP-4788 -- The Beacon Chain + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Containers](#containers) + - [Extended Containers](#extended-containers) + - [`ExecutionPayload`](#executionpayload) + - [`ExecutionPayloadHeader`](#executionpayloadheader) + + + + +## Introduction + +TODO + +## Containers + +### Extended Containers + +#### `ExecutionPayload` + +```python +class ExecutionPayload(Container): + # Execution block header fields + parent_hash: Hash32 + fee_recipient: ExecutionAddress # 'beneficiary' in the yellow paper + state_root: Bytes32 + receipts_root: Bytes32 + logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] + prev_randao: Bytes32 # 'difficulty' in the yellow paper + block_number: uint64 # 'number' in the yellow paper + gas_limit: uint64 + gas_used: uint64 + timestamp: uint64 + extra_data: ByteList[MAX_EXTRA_DATA_BYTES] + base_fee_per_gas: uint256 + # Extra payload fields + block_hash: Hash32 # Hash of execution block + transactions: List[Transaction, MAX_TRANSACTIONS_PER_PAYLOAD] + withdrawals: List[Withdrawal, MAX_WITHDRAWALS_PER_PAYLOAD] + parent_beacon_block_root: Root # [New in EIP-4788] +``` + +#### `ExecutionPayloadHeader` + +```python +class ExecutionPayloadHeader(Container): + # Execution block header fields + parent_hash: Hash32 + fee_recipient: ExecutionAddress + state_root: Bytes32 + receipts_root: Bytes32 + logs_bloom: ByteVector[BYTES_PER_LOGS_BLOOM] + prev_randao: Bytes32 + block_number: uint64 + gas_limit: uint64 + gas_used: uint64 + timestamp: uint64 + extra_data: ByteList[MAX_EXTRA_DATA_BYTES] + base_fee_per_gas: uint256 + # Extra payload fields + block_hash: Hash32 # Hash of execution block + transactions_root: Root + withdrawals_root: Root + parent_beacon_block_root: Root # [New in EIP-4788] +``` diff --git a/specs/_features/eip4788/validator.md b/specs/_features/eip4788/validator.md new file mode 100644 index 0000000000..421e297ce9 --- /dev/null +++ b/specs/_features/eip4788/validator.md @@ -0,0 +1,103 @@ +# EIP-4788 -- Honest Validator + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + +- [Introduction](#introduction) +- [Prerequisites](#prerequisites) +- [Helpers](#helpers) +- [Protocols](#protocols) + - [`ExecutionEngine`](#executionengine) + - [`get_payload`](#get_payload) +- [Beacon chain responsibilities](#beacon-chain-responsibilities) + - [Block proposal](#block-proposal) + - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) + - [ExecutionPayload](#executionpayload) + + + + +## Introduction + +This document represents the changes to be made in the code of an "honest validator" to implement the EIP-4788 feature. + +## Prerequisites + +This document is an extension of the [Capella -- Honest Validator](../capella/validator.md) guide. +All behaviors and definitions defined in this document, and documents it extends, carry over unless explicitly noted or overridden. + +All terminology, constants, functions, and protocol mechanics defined in the updated Beacon Chain doc of [Capella](../capella/beacon-chain.md) are requisite for this document and used throughout. +Please see related Beacon Chain doc before continuing and use them as a reference throughout. + +## Helpers + +## Protocols + +### `ExecutionEngine` + +#### `get_payload` + +`get_payload` returns the upgraded EIP-4788 `ExecutionPayload` type. + +## Beacon chain responsibilities + +All validator responsibilities remain unchanged other than those noted below. + +### Block proposal + +#### Constructing the `BeaconBlockBody` + +##### ExecutionPayload + +`ExecutionPayload`s are constructed as they were in Capella, except that the parent beacon block root is also supplied. + +*Note*: In this section, `state` is the state of the slot for the block proposal _without_ the block yet applied. +That is, `state` is the `previous_state` processed through any empty slots up to the assigned slot using `process_slots(previous_state, slot)`. + +*Note*: The only change made to `prepare_execution_payload` is to add the parent beacon block root as an additional +parameter to the `PayloadAttributes`. + +```python +def prepare_execution_payload(state: BeaconState, + pow_chain: Dict[Hash32, PowBlock], + safe_block_hash: Hash32, + finalized_block_hash: Hash32, + suggested_fee_recipient: ExecutionAddress, + execution_engine: ExecutionEngine) -> Optional[PayloadId]: + if not is_merge_transition_complete(state): + is_terminal_block_hash_set = TERMINAL_BLOCK_HASH != Hash32() + is_activation_epoch_reached = get_current_epoch(state) >= TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH + if is_terminal_block_hash_set and not is_activation_epoch_reached: + # Terminal block hash is set but activation epoch is not yet reached, no prepare payload call is needed + return None + + terminal_pow_block = get_terminal_pow_block(pow_chain) + if terminal_pow_block is None: + # Pre-merge, no prepare payload call is needed + return None + # Signify merge via producing on top of the terminal PoW block + parent_hash = terminal_pow_block.block_hash + else: + # Post-merge, normal payload + parent_hash = state.latest_execution_payload_header.block_hash + + # Set the forkchoice head and initiate the payload build process + payload_attributes = PayloadAttributes( + timestamp=compute_timestamp_at_slot(state, state.slot), + prev_randao=get_randao_mix(state, get_current_epoch(state)), + suggested_fee_recipient=suggested_fee_recipient, + withdrawals=get_expected_withdrawals(state), + parent_beacon_block_root=hash_tree_root(state.latest_block_header), # [New in EIP-4788] + ) + return execution_engine.notify_forkchoice_updated( + head_block_hash=parent_hash, + safe_block_hash=safe_block_hash, + finalized_block_hash=finalized_block_hash, + payload_attributes=payload_attributes, + ) +``` From 09e5fc7ebe29608fedad2828dd57cb82d89f1a41 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Tue, 11 Apr 2023 13:42:16 +0600 Subject: [PATCH 038/117] Add eip6110 to generators --- tests/generators/epoch_processing/main.py | 3 +++ tests/generators/finality/main.py | 2 ++ tests/generators/fork_choice/main.py | 4 +++- tests/generators/genesis/main.py | 2 ++ tests/generators/light_client/main.py | 2 ++ tests/generators/rewards/main.py | 2 ++ tests/generators/sync/main.py | 2 ++ 7 files changed, 16 insertions(+), 1 deletion(-) diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index a485f646aa..26fe6d72d4 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -34,6 +34,8 @@ deneb_mods = capella_mods + eip6110_mods = deneb_mods + # TODO Custody Game testgen is disabled for now # custody_game_mods = {**{key: 'eth2spec.test.custody_game.epoch_processing.test_process_' + key for key in [ # 'reveal_deadlines', @@ -47,6 +49,7 @@ BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, DENEB: deneb_mods, + EIP6110: eip6110_mods, } run_state_test_generators(runner_name="epoch_processing", all_mods=all_mods) diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index a25f3b8e7a..601afdd053 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -8,6 +8,7 @@ bellatrix_mods = altair_mods # No additional Bellatrix specific finality tests capella_mods = bellatrix_mods # No additional Capella specific finality tests deneb_mods = capella_mods # No additional Deneb specific finality tests + eip6110_mods = deneb_mods # No additional EIP6110 specific finality tests all_mods = { PHASE0: phase_0_mods, @@ -15,6 +16,7 @@ BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, DENEB: deneb_mods, + EIP6110: eip6110_mods, } run_state_test_generators(runner_name="finality", all_mods=all_mods) diff --git a/tests/generators/fork_choice/main.py b/tests/generators/fork_choice/main.py index 4456c2546b..49b4f83cef 100644 --- a/tests/generators/fork_choice/main.py +++ b/tests/generators/fork_choice/main.py @@ -19,13 +19,15 @@ ]} bellatrix_mods = combine_mods(_new_bellatrix_mods, altair_mods) capella_mods = bellatrix_mods # No additional Capella specific fork choice tests - deneb_mods = capella_mods # No additional Capella specific fork choice tests + deneb_mods = capella_mods # No additional Deneb specific fork choice tests + eip6110_mods = deneb_mods # No additional EIP6110 specific fork choice tests all_mods = { ALTAIR: altair_mods, BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, DENEB: deneb_mods, + EIP6110: eip6110_mods, } run_state_test_generators(runner_name="fork_choice", all_mods=all_mods) diff --git a/tests/generators/genesis/main.py b/tests/generators/genesis/main.py index e95afcde19..6db5ccf722 100644 --- a/tests/generators/genesis/main.py +++ b/tests/generators/genesis/main.py @@ -17,12 +17,14 @@ bellatrix_mods = combine_mods(_new_bellatrix_mods, altair_mods) capella_mods = bellatrix_mods # No additional Capella specific genesis tests deneb_mods = capella_mods # No additional Deneb specific genesis tests + eip6110_mods = deneb_mods # No additional EIP6110 specific genesis tests all_mods = { PHASE0: phase_0_mods, ALTAIR: altair_mods, BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, DENEB: deneb_mods, + EIP6110: eip6110_mods, } run_state_test_generators(runner_name="genesis", all_mods=all_mods) diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index cfe34aee4b..2f9451d1ed 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -15,12 +15,14 @@ ]} capella_mods = combine_mods(_new_capella_mods, bellatrix_mods) deneb_mods = capella_mods + eip6110_mods = deneb_mods all_mods = { ALTAIR: altair_mods, BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, DENEB: deneb_mods, + EIP6110: eip6110_mods, } run_state_test_generators(runner_name="light_client", all_mods=all_mods) diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index e6244d1720..5c84d9da03 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -17,6 +17,7 @@ bellatrix_mods = altair_mods capella_mods = bellatrix_mods deneb_mods = capella_mods + eip6110_mods = deneb_mods all_mods = { PHASE0: phase_0_mods, @@ -24,6 +25,7 @@ BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, DENEB: deneb_mods, + EIP6110: eip6110_mods, } run_state_test_generators(runner_name="rewards", all_mods=all_mods) diff --git a/tests/generators/sync/main.py b/tests/generators/sync/main.py index 11f05a741f..68e38cebe6 100644 --- a/tests/generators/sync/main.py +++ b/tests/generators/sync/main.py @@ -8,11 +8,13 @@ ]} capella_mods = bellatrix_mods deneb_mods = capella_mods + eip6110_mods = deneb_mods all_mods = { BELLATRIX: bellatrix_mods, CAPELLA: capella_mods, DENEB: deneb_mods, + EIP6110: eip6110_mods, } run_state_test_generators(runner_name="sync", all_mods=all_mods) From 0230c643b0ee5f7eb37cc974d419a66df98b85e8 Mon Sep 17 00:00:00 2001 From: Mikhail Kalinin Date: Tue, 11 Apr 2023 21:29:30 +0600 Subject: [PATCH 039/117] Fix EIP6110 import in generators --- tests/generators/epoch_processing/main.py | 2 +- tests/generators/finality/main.py | 2 +- tests/generators/fork_choice/main.py | 2 +- tests/generators/genesis/main.py | 2 +- tests/generators/light_client/main.py | 2 +- tests/generators/rewards/main.py | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/generators/epoch_processing/main.py b/tests/generators/epoch_processing/main.py index 26fe6d72d4..645c84cb6b 100644 --- a/tests/generators/epoch_processing/main.py +++ b/tests/generators/epoch_processing/main.py @@ -1,5 +1,5 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110 if __name__ == "__main__": diff --git a/tests/generators/finality/main.py b/tests/generators/finality/main.py index 601afdd053..15c6cad8dd 100644 --- a/tests/generators/finality/main.py +++ b/tests/generators/finality/main.py @@ -1,5 +1,5 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110 if __name__ == "__main__": diff --git a/tests/generators/fork_choice/main.py b/tests/generators/fork_choice/main.py index 49b4f83cef..b0c9a9bb9d 100644 --- a/tests/generators/fork_choice/main.py +++ b/tests/generators/fork_choice/main.py @@ -1,5 +1,5 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods -from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB +from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110 if __name__ == "__main__": diff --git a/tests/generators/genesis/main.py b/tests/generators/genesis/main.py index 6db5ccf722..feffde8e38 100644 --- a/tests/generators/genesis/main.py +++ b/tests/generators/genesis/main.py @@ -1,5 +1,5 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators, combine_mods -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110 if __name__ == "__main__": diff --git a/tests/generators/light_client/main.py b/tests/generators/light_client/main.py index 2f9451d1ed..c6b0e01b9b 100644 --- a/tests/generators/light_client/main.py +++ b/tests/generators/light_client/main.py @@ -1,4 +1,4 @@ -from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB +from eth2spec.test.helpers.constants import ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110 from eth2spec.gen_helpers.gen_from_tests.gen import combine_mods, run_state_test_generators diff --git a/tests/generators/rewards/main.py b/tests/generators/rewards/main.py index 5c84d9da03..d01d4a424e 100644 --- a/tests/generators/rewards/main.py +++ b/tests/generators/rewards/main.py @@ -1,5 +1,5 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB +from eth2spec.test.helpers.constants import PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110 if __name__ == "__main__": From 6e423f6c4275608515158b5c483c351f0eb61b19 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Wed, 12 Apr 2023 11:29:48 +1000 Subject: [PATCH 040/117] Stagger node rotations --- specs/phase0/validator.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 1b06aecfbb..56ca50732c 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -612,23 +612,22 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th * Remain subscribed to `SUBNETS_PER_NODE` for `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs. * Maintain advertisement of the selected subnets in their node's ENR `attnets` entry by setting the selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets. -* Select these subnets based on their node-id as specified by the following `compute_subnets(node_id,epoch)` function. +* Select these subnets based on their node-id as specified by the following `compute_subscribed_subnets(node_id,epoch)` function. ```python -def compute_subnet(node_id: int, epoch: Epoch, index: int) -> int: +def compute_subscribed_subnet(node_id: int, epoch: Epoch, index: int) -> int: node_id_prefix = node_id >> (256 - ATTESTATION_SUBNET_PREFIX_BITS) - permutation_seed = hash(uint_to_bytes(epoch // EPOCHS_PER_SUBNET_SUBSCRIPTION)) + node_offset = node_id % EPOCHS_PER_SUBNET_SUBSCRIPTION + permutation_seed = hash(uint_to_bytes((epoch + node_offset) // EPOCHS_PER_SUBNET_SUBSCRIPTION)) permutated_prefix = compute_shuffled_index(node_id_prefix, 1 << ATTESTATION_SUBNET_PREFIX_BITS, permutation_seed) return (permutated_prefix + index) % ATTESTATION_SUBNET_COUNT ``` ```python -def compute_subnets(node_id: int, epoch: Epoch) -> Sequence[int]: - return [compute_subnet(node_id, epoch, idx) for idx in range(SUBNETS_PER_NODE)] +def compute_subscribed_subnets(node_id: int, epoch: Epoch) -> Sequence[int]: + return [compute_subscribed_subnet(node_id, epoch, idx) for idx in range(SUBNETS_PER_NODE)] ``` -*Note*: Nodes should subscribe to new subnets and remain subscribed to old subnets for at least one epoch. Nodes should pick a random duration to unsubscribe from old subnets to smooth the transition on the exact epoch boundary of which the shuffling changes. - *Note*: When preparing for a hard fork, a validator must select and subscribe to subnets of the future fork versioning at least `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. ## How to avoid slashing From 334114d9d373d3a71ab49720ba4831d0b1fce6dd Mon Sep 17 00:00:00 2001 From: Roberto Saltini Date: Tue, 18 Apr 2023 13:14:53 +1000 Subject: [PATCH 041/117] Rename get_ancestor_at_epoch_boundary to get_checkpoint_block --- specs/bellatrix/fork-choice.md | 2 +- specs/deneb/fork-choice.md | 2 +- specs/phase0/fork-choice.md | 12 ++++++------ specs/phase0/p2p-interface.md | 8 ++++---- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/specs/bellatrix/fork-choice.md b/specs/bellatrix/fork-choice.md index d22436c9d8..6c7a31508b 100644 --- a/specs/bellatrix/fork-choice.md +++ b/specs/bellatrix/fork-choice.md @@ -170,7 +170,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert store.finalized_checkpoint.root == get_ancestor_at_epoch_boundary( + assert store.finalized_checkpoint.root == get_checkpoint_block( store, block.parent_root, store.finalized_checkpoint.epoch, diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index e76e159c4f..8a33fecc56 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -82,7 +82,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert store.finalized_checkpoint.root == get_ancestor_at_epoch_boundary( + assert store.finalized_checkpoint.root == get_checkpoint_block( store, block.parent_root, store.finalized_checkpoint.epoch, diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 478dd21427..a2bbb8f629 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -18,7 +18,7 @@ - [`get_current_slot`](#get_current_slot) - [`compute_slots_since_epoch_start`](#compute_slots_since_epoch_start) - [`get_ancestor`](#get_ancestor) - - [`get_ancestor_at_epoch_boundary`](#get_ancestor_at_epoch_boundary) + - [`get_checkpoint_block`](#get_checkpoint_block) - [`get_weight`](#get_weight) - [`get_voting_source`](#get_voting_source) - [`filter_block_tree`](#filter_block_tree) @@ -193,10 +193,10 @@ def get_ancestor(store: Store, root: Root, slot: Slot) -> Root: return root ``` -#### `get_ancestor_at_epoch_boundary` +#### `get_checkpoint_block` ```python -def get_ancestor_at_epoch_boundary(store: Store, root: Root, epoch: Epoch) -> Root: +def get_checkpoint_block(store: Store, root: Root, epoch: Epoch) -> Root: """ Compute the epoch boundary block for epoch ``epoch`` in the chain of block ``root`` """ @@ -292,7 +292,7 @@ def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconB correct_finalized = ( store.finalized_checkpoint.epoch == GENESIS_EPOCH - or store.finalized_checkpoint.root == get_ancestor_at_epoch_boundary( + or store.finalized_checkpoint.root == get_checkpoint_block( store, block_root, store.finalized_checkpoint.epoch, @@ -457,7 +457,7 @@ def validate_on_attestation(store: Store, attestation: Attestation, is_from_bloc assert store.blocks[attestation.data.beacon_block_root].slot <= attestation.data.slot # LMD vote must be consistent with FFG vote target - assert target.root == get_ancestor_at_epoch_boundary(store, attestation.data.beacon_block_root, target.epoch) + assert target.root == get_checkpoint_block(store, attestation.data.beacon_block_root, target.epoch) # Attestations can only affect the fork choice of subsequent slots. # Delay consideration in the fork choice until their slot is in the past. @@ -520,7 +520,7 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert store.finalized_checkpoint.root == get_ancestor_at_epoch_boundary( + assert store.finalized_checkpoint.root == get_checkpoint_block( store, block.parent_root, store.finalized_checkpoint.epoch, diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 56c1b8cfb1..5401a15da5 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -317,7 +317,7 @@ The following validations MUST pass before forwarding the `signed_beacon_block` - _[REJECT]_ The block's parent (defined by `block.parent_root`) passes validation. - _[REJECT]_ The block is from a higher slot than its parent. - _[REJECT]_ The current `finalized_checkpoint` is an ancestor of `block` -- i.e. - `get_ancestor_at_epoch_boundary(store, block.parent_root, store.finalized_checkpoint.epoch) + `get_checkpoint_block(store, block.parent_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root` - _[REJECT]_ The block is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `parent_root`/`slot`). @@ -356,7 +356,7 @@ The following validations MUST pass before forwarding the `signed_aggregate_and_ (a client MAY queue aggregates for processing once block is retrieved). - _[REJECT]_ The block being voted for (`aggregate.data.beacon_block_root`) passes validation. - _[IGNORE]_ The current `finalized_checkpoint` is an ancestor of the `block` defined by `aggregate.data.beacon_block_root` -- i.e. - `get_ancestor_at_epoch_boundary(store, aggregate.data.beacon_block_root, finalized_checkpoint.epoch) + `get_checkpoint_block(store, aggregate.data.beacon_block_root, finalized_checkpoint.epoch) == store.finalized_checkpoint.root` @@ -425,9 +425,9 @@ The following validations MUST pass before forwarding the `attestation` on the s (a client MAY queue attestations for processing once block is retrieved). - _[REJECT]_ The block being voted for (`attestation.data.beacon_block_root`) passes validation. - _[REJECT]_ The attestation's target block is an ancestor of the block named in the LMD vote -- i.e. - `get_ancestor_at_epoch_boundary(store, attestation.data.beacon_block_root, attestation.data.target.epoch) == attestation.data.target.root` + `get_checkpoint_block(store, attestation.data.beacon_block_root, attestation.data.target.epoch) == attestation.data.target.root` - _[IGNORE]_ The current `finalized_checkpoint` is an ancestor of the `block` defined by `attestation.data.beacon_block_root` -- i.e. - `get_ancestor_at_epoch_boundary(store, attestation.data.beacon_block_root, store.finalized_checkpoint.epoch) + `get_checkpoint_block(store, attestation.data.beacon_block_root, store.finalized_checkpoint.epoch) == store.finalized_checkpoint.root` From 36fcb81b88c87f279cd4b46d094eb58514e9be8b Mon Sep 17 00:00:00 2001 From: Roberto Saltini Date: Tue, 18 Apr 2023 13:26:16 +1000 Subject: [PATCH 042/117] Break long statement into two statements --- specs/bellatrix/fork-choice.md | 3 ++- specs/deneb/fork-choice.md | 3 ++- specs/phase0/fork-choice.md | 15 +++++++++------ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/specs/bellatrix/fork-choice.md b/specs/bellatrix/fork-choice.md index 6c7a31508b..68519ff908 100644 --- a/specs/bellatrix/fork-choice.md +++ b/specs/bellatrix/fork-choice.md @@ -170,11 +170,12 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert store.finalized_checkpoint.root == get_checkpoint_block( + finalized_checkpoint_block = get_checkpoint_block( store, block.parent_root, store.finalized_checkpoint.epoch, ) + assert store.finalized_checkpoint.root == finalized_checkpoint_block # Check the block is valid and compute the post-state state = pre_state.copy() diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index 8a33fecc56..9faa11077f 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -82,11 +82,12 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert store.finalized_checkpoint.root == get_checkpoint_block( + finalized_checkpoint_block = get_checkpoint_block( store, block.parent_root, store.finalized_checkpoint.epoch, ) + assert store.finalized_checkpoint.root == finalized_checkpoint_block # [New in Deneb] # Check if blob data is available diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index a2bbb8f629..8582547fdb 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -290,13 +290,15 @@ def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconB voting_source.epoch + 2 >= current_epoch ) + finalized_checkpoint_block = get_checkpoint_block( + store, + block.parent_root, + store.finalized_checkpoint.epoch, + ) + correct_finalized = ( store.finalized_checkpoint.epoch == GENESIS_EPOCH - or store.finalized_checkpoint.root == get_checkpoint_block( - store, - block_root, - store.finalized_checkpoint.epoch, - ) + or store.finalized_checkpoint.root == finalized_checkpoint_block ) # If expected finalized/justified, add to viable block-tree and signal viability to parent. @@ -520,11 +522,12 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert store.finalized_checkpoint.root == get_checkpoint_block( + finalized_checkpoint_block = get_checkpoint_block( store, block.parent_root, store.finalized_checkpoint.epoch, ) + assert store.finalized_checkpoint.root == finalized_checkpoint_block # Check the block is valid and compute the post-state state = pre_state.copy() From c98560597351529f6f782fd434a2937d0f6c296e Mon Sep 17 00:00:00 2001 From: Roberto Saltini Date: Tue, 18 Apr 2023 13:49:08 +1000 Subject: [PATCH 043/117] Fix copy and past error --- specs/phase0/fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 8582547fdb..0d5bfb4d79 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -292,7 +292,7 @@ def filter_block_tree(store: Store, block_root: Root, blocks: Dict[Root, BeaconB finalized_checkpoint_block = get_checkpoint_block( store, - block.parent_root, + block_root, store.finalized_checkpoint.epoch, ) From b5bd90dd5f6028d59de1fb5c97c5da95ac53b3aa Mon Sep 17 00:00:00 2001 From: Roberto Saltini Date: Tue, 18 Apr 2023 13:51:13 +1000 Subject: [PATCH 044/117] Applied changes to tests --- .../test/phase0/fork_choice/test_get_head.py | 22 +++++++---- .../test/phase0/fork_choice/test_on_block.py | 38 +++++++++++++------ 2 files changed, 40 insertions(+), 20 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py index f5960ff703..f5c3aae15d 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py @@ -479,7 +479,7 @@ def test_voting_source_within_two_epoch(spec, state): - store.voting_source[block_root].epoch != store.justified_checkpoint.epoch, and - store.unrealized_justifications[block_root].epoch >= store.justified_checkpoint.epoch, and - store.voting_source[block_root].epoch + 2 >= current_epoch, and - - store.finalized_checkpoint.root == get_ancestor(store, block_root, finalized_slot) + - store.finalized_checkpoint.root == get_checkpoint_block(store, block_root, store.finalized_checkpoint.epoch) """ test_steps = [] # Initialization @@ -536,8 +536,11 @@ def test_voting_source_within_two_epoch(spec, state): assert store.unrealized_justifications[last_fork_block_root].epoch >= store.justified_checkpoint.epoch # assert store.voting_source[last_fork_block_root].epoch + 2 >= \ # spec.compute_epoch_at_slot(spec.get_current_slot(store)) - finalized_slot = spec.compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert store.finalized_checkpoint.root == spec.get_ancestor(store, last_fork_block_root, finalized_slot) + assert store.finalized_checkpoint.root == spec.get_checkpoint_block( + store, + last_fork_block_root, + store.finalized_checkpoint.epoch + ) assert spec.get_head(store) == last_fork_block_root yield 'steps', test_steps @@ -552,7 +555,7 @@ def test_voting_source_beyond_two_epoch(spec, state): - store.voting_source[block_root].epoch != store.justified_checkpoint.epoch, and - store.unrealized_justifications[block_root].epoch >= store.justified_checkpoint.epoch, and - store.voting_source[block_root].epoch + 2 < current_epoch, and - - store.finalized_checkpoint.root == get_ancestor(store, block_root, finalized_slot) + - store.finalized_checkpoint.root == get_checkpoint_block(store, block_root, store.finalized_checkpoint.epoch) """ test_steps = [] # Initialization @@ -617,8 +620,11 @@ def test_voting_source_beyond_two_epoch(spec, state): assert store.unrealized_justifications[last_fork_block_root].epoch >= store.justified_checkpoint.epoch # assert store.voting_source[last_fork_block_root].epoch + 2 < \ # spec.compute_epoch_at_slot(spec.get_current_slot(store)) - finalized_slot = spec.compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert store.finalized_checkpoint.root == spec.get_ancestor(store, last_fork_block_root, finalized_slot) + assert store.finalized_checkpoint.root == spec.get_checkpoint_block( + store, + last_fork_block_root, + store.finalized_checkpoint.epoch + ) assert spec.get_head(store) == correct_head yield 'steps', test_steps @@ -641,7 +647,7 @@ def test_incorrect_finalized(spec, state): # Check that the store doesn't allow for a head block that has: # - store.voting_source[block_root].epoch == store.justified_checkpoint.epoch, and # - store.finalized_checkpoint.epoch != GENESIS_EPOCH, and - # - store.finalized_checkpoint.root != get_ancestor(store, block_root, finalized_slot) + # - store.finalized_checkpoint.root != get_checkpoint_block(store, block_root, store.finalized_checkpoint.epoch) test_steps = [] # Initialization store, anchor_block = get_genesis_forkchoice_store_and_block(spec, state) @@ -718,7 +724,7 @@ def test_incorrect_finalized(spec, state): assert store.voting_source[last_fork_block_root].epoch == store.justified_checkpoint.epoch assert store.finalized_checkpoint.epoch != spec.GENESIS_EPOCH finalized_slot = spec.compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert store.finalized_checkpoint.root != spec.get_ancestor(store, last_fork_block_root, finalized_slot) + assert store.finalized_checkpoint.root != spec.get_checkpoint_block(store, block_root, store.finalized_checkpoint.epoch) assert spec.get_head(store) != last_fork_block_root assert spec.get_head(store) == head_root diff --git a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_on_block.py index 0af7753391..a3f09c7c96 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_on_block.py @@ -352,8 +352,7 @@ def test_new_finalized_slot_is_not_justified_checkpoint_ancestor(spec, state): # NOTE: Do not call `on_tick` here yield from add_block(spec, store, block, test_steps) - finalized_slot = spec.compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - ancestor_at_finalized_slot = spec.get_ancestor(store, pre_store_justified_checkpoint_root, finalized_slot) + ancestor_at_finalized_slot = spec.get_checkpoint_block(store, pre_store_justified_checkpoint_root, store.finalized_checkpoint.epoch) assert ancestor_at_finalized_slot != store.finalized_checkpoint.root assert store.finalized_checkpoint == another_state.finalized_checkpoint @@ -428,8 +427,7 @@ def test_new_finalized_slot_is_justified_checkpoint_ancestor(spec, state): for block in all_blocks: yield from tick_and_add_block(spec, store, block, test_steps) - finalized_slot = spec.compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - ancestor_at_finalized_slot = spec.get_ancestor(store, pre_store_justified_checkpoint_root, finalized_slot) + ancestor_at_finalized_slot = spec.get_checkpoint_block(store, pre_store_justified_checkpoint_root, store.finalized_checkpoint.epoch) assert ancestor_at_finalized_slot == store.finalized_checkpoint.root assert store.finalized_checkpoint == another_state.finalized_checkpoint @@ -857,10 +855,18 @@ def test_incompatible_justification_update_start_of_epoch(spec, state): # Now add the blocks & check that justification update was triggered for signed_block in signed_blocks: yield from tick_and_add_block(spec, store, signed_block, test_steps) - finalized_slot = spec.compute_start_slot_at_epoch(state.finalized_checkpoint.epoch) - assert spec.get_ancestor(store, last_block_root, finalized_slot) == state.finalized_checkpoint.root - justified_slot = spec.compute_start_slot_at_epoch(state.current_justified_checkpoint.epoch) - assert spec.get_ancestor(store, last_block_root, justified_slot) != state.current_justified_checkpoint.root + finalized_checkpoint_block = spec.get_checkpoint_block( + store, + last_block_root, + state.finalized_checkpoint.epoch, + ) + assert finalized_checkpoint_block == state.finalized_checkpoint.root + justified_checkpoint_block = spec.get_checkpoint_block( + store, + last_block_root, + state.current_justified_checkpoint.epoch, + ) + assert justified_checkpoint_block != state.current_justified_checkpoint.root assert store.finalized_checkpoint.epoch == 4 assert store.justified_checkpoint.epoch == 6 @@ -934,10 +940,18 @@ def test_incompatible_justification_update_end_of_epoch(spec, state): # Now add the blocks & check that justification update was triggered for signed_block in signed_blocks: yield from tick_and_add_block(spec, store, signed_block, test_steps) - finalized_slot = spec.compute_start_slot_at_epoch(state.finalized_checkpoint.epoch) - assert spec.get_ancestor(store, last_block_root, finalized_slot) == state.finalized_checkpoint.root - justified_slot = spec.compute_start_slot_at_epoch(state.current_justified_checkpoint.epoch) - assert spec.get_ancestor(store, last_block_root, justified_slot) != state.current_justified_checkpoint.root + finalized_checkpoint_block = spec.get_checkpoint_block( + store, + last_block_root, + state.finalized_checkpoint.epoch, + ) + assert finalized_checkpoint_block == state.finalized_checkpoint.root + justified_checkpoint_block = spec.get_checkpoint_block( + store, + last_block_root, + state.current_justified_checkpoint.epoch, + ) + assert justified_checkpoint_block != state.current_justified_checkpoint.root assert store.finalized_checkpoint.epoch == 4 assert store.justified_checkpoint.epoch == 6 From 313439a04b121a1c53585396dbea6df02026404f Mon Sep 17 00:00:00 2001 From: Roberto Saltini Date: Tue, 18 Apr 2023 13:54:31 +1000 Subject: [PATCH 045/117] Fix lint erorrs --- .../test/phase0/fork_choice/test_get_head.py | 6 +++++- .../test/phase0/fork_choice/test_on_block.py | 12 ++++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py index f5c3aae15d..30f94b854c 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py +++ b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_get_head.py @@ -724,7 +724,11 @@ def test_incorrect_finalized(spec, state): assert store.voting_source[last_fork_block_root].epoch == store.justified_checkpoint.epoch assert store.finalized_checkpoint.epoch != spec.GENESIS_EPOCH finalized_slot = spec.compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) - assert store.finalized_checkpoint.root != spec.get_checkpoint_block(store, block_root, store.finalized_checkpoint.epoch) + assert store.finalized_checkpoint.root != spec.get_checkpoint_block( + store, + block_root, + store.finalized_checkpoint.epoch + ) assert spec.get_head(store) != last_fork_block_root assert spec.get_head(store) == head_root diff --git a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_on_block.py index a3f09c7c96..840413a364 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/phase0/fork_choice/test_on_block.py @@ -352,7 +352,11 @@ def test_new_finalized_slot_is_not_justified_checkpoint_ancestor(spec, state): # NOTE: Do not call `on_tick` here yield from add_block(spec, store, block, test_steps) - ancestor_at_finalized_slot = spec.get_checkpoint_block(store, pre_store_justified_checkpoint_root, store.finalized_checkpoint.epoch) + ancestor_at_finalized_slot = spec.get_checkpoint_block( + store, + pre_store_justified_checkpoint_root, + store.finalized_checkpoint.epoch + ) assert ancestor_at_finalized_slot != store.finalized_checkpoint.root assert store.finalized_checkpoint == another_state.finalized_checkpoint @@ -427,7 +431,11 @@ def test_new_finalized_slot_is_justified_checkpoint_ancestor(spec, state): for block in all_blocks: yield from tick_and_add_block(spec, store, block, test_steps) - ancestor_at_finalized_slot = spec.get_checkpoint_block(store, pre_store_justified_checkpoint_root, store.finalized_checkpoint.epoch) + ancestor_at_finalized_slot = spec.get_checkpoint_block( + store, + pre_store_justified_checkpoint_root, + store.finalized_checkpoint.epoch + ) assert ancestor_at_finalized_slot == store.finalized_checkpoint.root assert store.finalized_checkpoint == another_state.finalized_checkpoint From ffb84598cf5f7d29bd6220e977f76d47c599bb4f Mon Sep 17 00:00:00 2001 From: Roberto Saltini Date: Tue, 18 Apr 2023 16:03:10 +1000 Subject: [PATCH 046/117] Fixed doc in get_checkpoint_block --- specs/phase0/fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 0d5bfb4d79..e25ae6e901 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -198,7 +198,7 @@ def get_ancestor(store: Store, root: Root, slot: Slot) -> Root: ```python def get_checkpoint_block(store: Store, root: Root, epoch: Epoch) -> Root: """ - Compute the epoch boundary block for epoch ``epoch`` in the chain of block ``root`` + Compute the checkpoint block for epoch ``epoch`` in the chain of block ``root`` """ epoch_first_slot = compute_start_slot_at_epoch(epoch) return get_ancestor(store, root, epoch_first_slot) From 87d42919b9f80e18ff2eba5354642d6b6c4829c6 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 18 Apr 2023 14:00:11 -0500 Subject: [PATCH 047/117] Add "commitment" to test names for consistency --- tests/generators/kzg_4844/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/generators/kzg_4844/main.py b/tests/generators/kzg_4844/main.py index 699d1f369a..b1391f6787 100644 --- a/tests/generators/kzg_4844/main.py +++ b/tests/generators/kzg_4844/main.py @@ -708,7 +708,7 @@ def case06_verify_blob_kzg_proof_batch(): # Edge case: Invalid commitment, too few bytes commitments_invalid_tooFewBytes = commitments[:3] + [commitments[3][:-1]] + commitments[4:] expect_exception(spec.verify_blob_kzg_proof_batch, VALID_BLOBS, commitments, commitments_invalid_tooFewBytes) - yield 'verify_blob_kzg_proof_batch_case_too_few_bytes', { + yield 'verify_blob_kzg_proof_batch_case_commitment_too_few_bytes', { 'input': { 'blobs': encode_hex_list(VALID_BLOBS), 'commitments': encode_hex_list(commitments_invalid_tooFewBytes), @@ -720,7 +720,7 @@ def case06_verify_blob_kzg_proof_batch(): # Edge case: Invalid commitment, too many bytes commitments_invalid_tooManyBytes = commitments[:3] + [commitments[3] + b"\x00"] + commitments[4:] expect_exception(spec.verify_blob_kzg_proof_batch, VALID_BLOBS, commitments, commitments_invalid_tooManyBytes) - yield 'verify_blob_kzg_proof_batch_case_too_many_bytes', { + yield 'verify_blob_kzg_proof_batch_case_commitment_too_many_bytes', { 'input': { 'blobs': encode_hex_list(VALID_BLOBS), 'commitments': encode_hex_list(commitments_invalid_tooManyBytes), From 03a3e4082a4cfcc984bf4b46d1c17cc08e61b576 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 19 Apr 2023 19:10:46 +0800 Subject: [PATCH 048/117] Fix p2p-interface.md ToC --- docs/README.md | 70 ------------ specs/altair/p2p-interface.md | 42 +++---- specs/bellatrix/p2p-interface.md | 64 +++++------ specs/capella/p2p-interface.md | 24 ++-- specs/deneb/p2p-interface.md | 79 ++++++------- specs/phase0/p2p-interface.md | 186 +++++++++++++++---------------- 6 files changed, 199 insertions(+), 266 deletions(-) delete mode 100644 docs/README.md diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index 9f2528263e..0000000000 --- a/docs/README.md +++ /dev/null @@ -1,70 +0,0 @@ -# Ethereum Proof-of-Stake Consensus Specifications - -[![Join the chat at https://discord.gg/qGpsxSA](https://img.shields.io/badge/chat-on%20discord-blue.svg)](https://discord.gg/qGpsxSA) [![Join the chat at https://gitter.im/ethereum/sharding](https://badges.gitter.im/ethereum/sharding.svg)](https://gitter.im/ethereum/sharding?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) - -To learn more about proof-of-stake and sharding, see the [PoS documentation](https://ethereum.org/en/developers/docs/consensus-mechanisms/pos/), [sharding documentation](https://ethereum.org/en/upgrades/sharding/) and the [research compendium](https://notes.ethereum.org/s/H1PGqDhpm). - -This repository hosts the current Ethereum proof-of-stake specifications. Discussions about design rationale and proposed changes can be brought up and discussed as issues. Solidified, agreed-upon changes to the spec can be made through pull requests. - -## Specs - -[![GitHub release](https://img.shields.io/github/v/release/ethereum/eth2.0-specs)](https://github.com/ethereum/eth2.0-specs/releases/) [![PyPI version](https://badge.fury.io/py/eth2spec.svg)](https://badge.fury.io/py/eth2spec) - -Core specifications for Ethereum proof-of-stake clients can be found in [specs](specs/). These are divided into features. -Features are researched and developed in parallel, and then consolidated into sequential upgrades when ready. - -### Stable Specifications - -| Seq. | Code Name | Fork Epoch | Specs | -| - | - | - | - | -| 0 | **Phase0** |`0` |
  • Core
    • [The beacon chain](specs/phase0/beacon-chain.md)
    • [Deposit contract](specs/phase0/deposit-contract.md)
    • [Beacon chain fork choice](specs/phase0/fork-choice.md)
  • Additions
    • [Honest validator guide](specs/phase0/validator.md)
    • [P2P networking](specs/phase0/p2p-interface.md)
    • [Weak subjectivity](specs/phase0/weak-subjectivity.md)
| -| 1 | **Altair** | `74240` |
  • Core
    • [Beacon chain changes](specs/altair/beacon-chain.md)
    • [Altair fork](specs/altair/fork.md)
  • Additions
    • [Light client sync protocol](specs/altair/light-client/sync-protocol.md) ([full node](specs/altair/light-client/full-node.md), [light client](specs/altair/light-client/light-client.md), [networking](specs/altair/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/altair/validator.md)
    • [P2P networking](specs/altair/p2p-interface.md)
| -| 2 | **Bellatrix**
(["The Merge"](https://ethereum.org/en/upgrades/merge/)) | `144896` |
  • Core
    • [Beacon Chain changes](specs/bellatrix/beacon-chain.md)
    • [Bellatrix fork](specs/bellatrix/fork.md)
    • [Fork choice changes](specs/bellatrix/fork-choice.md)
  • Additions
    • [Honest validator guide changes](specs/bellatrix/validator.md)
    • [P2P networking](specs/bellatrix/p2p-interface.md)
| -| 3 | **Capella** | `194048` |
  • Core
    • [Beacon chain changes](specs/capella/beacon-chain.md)
    • [Capella fork](specs/capella/fork.md)
  • Additions
    • [Light client sync protocol changes](specs/capella/light-client/sync-protocol.md) ([fork](specs/capella/light-client/fork.md), [full node](specs/capella/light-client/full-node.md), [networking](specs/capella/light-client/p2p-interface.md))
    • [Validator additions](specs/capella/validator.md)
    • [P2P networking](specs/capella/p2p-interface.md)
| - -### In-development Specifications -| Code Name or Topic | Specs | Notes | -| - | - | - | -| Deneb (tentative) |
  • Core
    • [Beacon Chain changes](specs/deneb/beacon-chain.md)
    • [Deneb fork](specs/deneb/fork.md)
    • [Polynomial commitments](specs/deneb/polynomial-commitments.md)
    • [Fork choice changes](specs/deneb/fork-choice.md)
  • Additions
    • [Light client sync protocol changes](specs/deneb/light-client/sync-protocol.md) ([fork](specs/deneb/light-client/fork.md), [full node](specs/deneb/light-client/full-node.md), [networking](specs/deneb/light-client/p2p-interface.md))
    • [Honest validator guide changes](specs/deneb/validator.md)
    • [P2P networking](specs/deneb/p2p-interface.md)
| -| Sharding (outdated) |
  • Core
    • [Beacon Chain changes](specs/_features/sharding/beacon-chain.md)
  • Additions
    • [P2P networking](specs/_features/sharding/p2p-interface.md)
| -| Custody Game (outdated) |
  • Core
    • [Beacon Chain changes](specs/_features/custody_game/beacon-chain.md)
  • Additions
    • [Honest validator guide changes](specs/_features/custody_game/validator.md)
| Dependent on sharding | -| Data Availability Sampling (outdated) |
  • Core
    • [Core types and functions](specs/_features/das/das-core.md)
    • [Fork choice changes](specs/_features/das/fork-choice.md)
  • Additions
    • [P2P Networking](specs/_features/das/p2p-interface.md)
    • [Sampling process](specs/_features/das/sampling.md)
|
  • Dependent on sharding
  • [Technical explainer](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD)
| -| EIP-6110 |
  • Core
    • [Beacon Chain changes](specs/_features/eip6110//beacon-chain.md)
    • [EIP-6110 fork](specs/_features/eip6110/fork.md)
  • Additions
    • [Honest validator guide changes](specs/_features/eip6110/validator.md)
| - -### Accompanying documents can be found in [specs](specs) and include: - -* [SimpleSerialize (SSZ) spec](ssz/simple-serialize.md) -* [Merkle proof formats](ssz/merkle-proofs.md) -* [General test format](tests/formats/README.md) - -## Additional specifications for client implementers - -Additional specifications and standards outside of requisite client functionality can be found in the following repos: - -* [Beacon APIs](https://github.com/ethereum/beacon-apis) -* [Beacon Metrics](https://github.com/ethereum/beacon-metrics/) - -## Design goals - -The following are the broad design goals for the Ethereum proof-of-stake consensus specifications: -* to minimize complexity, even at the cost of some losses in efficiency -* to remain live through major network partitions and when very large portions of nodes go offline -* to select all components such that they are either quantum secure or can be easily swapped out for quantum secure counterparts when available -* to utilize crypto and design techniques that allow for a large participation of validators in total and per unit time -* to allow for a typical consumer laptop with `O(C)` resources to process/validate `O(1)` shards (including any system level validation such as the beacon chain) - -## Useful external resources - -* [Design Rationale](https://notes.ethereum.org/s/rkhCgQteN#) -* [Phase 0 Onboarding Document](https://notes.ethereum.org/s/Bkn3zpwxB) -* [Combining GHOST and Casper paper](https://arxiv.org/abs/2003.03052) - -## For spec contributors - -Documentation on the different components used during spec writing can be found here: -* [YAML Test Generators](tests/generators/README.md) -* [Executable Python Spec, with Py-tests](tests/core/pyspec/README.md) - -## Consensus spec tests - -Conformance tests built from the executable python spec are available in the [Ethereum Proof-of-Stake Consensus Spec Tests](https://github.com/ethereum/consensus-spec-tests) repo. Compressed tarballs are available in [releases](https://github.com/ethereum/consensus-spec-tests/releases). diff --git a/specs/altair/p2p-interface.md b/specs/altair/p2p-interface.md index 8d6b1c433a..0f278b08c5 100644 --- a/specs/altair/p2p-interface.md +++ b/specs/altair/p2p-interface.md @@ -13,7 +13,7 @@ Altair adds new messages, topics and data to the Req-Resp, Gossip and Discovery - - [Warning](#warning) +- [Warning](#warning) - [Modifications in Altair](#modifications-in-altair) - [MetaData](#metadata) - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) @@ -43,9 +43,9 @@ Altair adds new messages, topics and data to the Req-Resp, Gossip and Discovery This document is currently illustrative for early Altair testnets and some parts are subject to change. Refer to the note in the [validator guide](./validator.md) for further details. -# Modifications in Altair +## Modifications in Altair -## MetaData +### MetaData The `MetaData` stored locally by clients is updated with an additional field to communicate the sync committee subnet subscriptions: @@ -62,12 +62,12 @@ Where - `seq_number` and `attnets` have the same meaning defined in the Phase 0 document. - `syncnets` is a `Bitvector` representing the node's sync committee subnet subscriptions. This field should mirror the data in the node's ENR as outlined in the [validator guide](./validator.md#sync-committee-subnet-stability). -## The gossip domain: gossipsub +### The gossip domain: gossipsub Gossip meshes are added in Altair to support the consensus activities of the sync committees. Validators use an aggregation scheme to balance the processing and networking load across all of the relevant actors. -### Topics and messages +#### Topics and messages Topics follow the same specification as in the Phase 0 document. New topics are added in Altair to support the sync committees and the beacon block topic is updated with the modified type. @@ -103,11 +103,11 @@ Definitions of these new types can be found in the [Altair validator guide](./va Note that the `ForkDigestValue` path segment of the topic separates the old and the new `beacon_block` topics. -#### Global topics +##### Global topics Altair changes the type of the global beacon block topic and adds one global topic to propagate partially aggregated sync committee messages to all potential proposers of beacon blocks. -##### `beacon_block` +###### `beacon_block` The existing specification for this topic does not change from the Phase 0 document, but the type of the payload does change to the (modified) `SignedBeaconBlock`. @@ -115,7 +115,7 @@ This type changes due to the inclusion of the inner `BeaconBlockBody` that is mo See the [state transition document](./beacon-chain.md#beaconblockbody) for Altair for further details. -##### `sync_committee_contribution_and_proof` +###### `sync_committee_contribution_and_proof` This topic is used to propagate partially aggregated sync committee messages to be included in future blocks. @@ -152,11 +152,11 @@ def get_sync_subcommittee_pubkeys(state: BeaconState, subcommittee_index: uint64 - _[REJECT]_ The aggregator signature, `signed_contribution_and_proof.signature`, is valid. - _[REJECT]_ The aggregate signature is valid for the message `beacon_block_root` and aggregate pubkey derived from the participation info in `aggregation_bits` for the subcommittee specified by the `contribution.subcommittee_index`. -#### Sync committee subnets +##### Sync committee subnets Sync committee subnets are used to propagate unaggregated sync committee messages to subsections of the network. -##### `sync_committee_{subnet_id}` +###### `sync_committee_{subnet_id}` The `sync_committee_{subnet_id}` topics are used to propagate unaggregated sync committee messages to the subnet `subnet_id` to be aggregated before being gossiped to the global `sync_committee_contribution_and_proof` topic. @@ -170,7 +170,7 @@ The following validations MUST pass before forwarding the `sync_committee_messag Note this validation is _per topic_ so that for a given `slot`, multiple messages could be forwarded with the same `validator_index` as long as the `subnet_id`s are distinct. - _[REJECT]_ The `signature` is valid for the message `beacon_block_root` for the validator referenced by `validator_index`. -#### Sync committees and aggregation +##### Sync committees and aggregation The aggregation scheme closely follows the design of the attestation aggregation scheme. Sync committee messages are broadcast into "subnets" defined by a topic. @@ -182,7 +182,7 @@ Unaggregated messages (along with metadata) are sent as `SyncCommitteeMessage`s Aggregated sync committee messages are packaged into (signed) `SyncCommitteeContribution` along with proofs and gossiped to the `sync_committee_contribution_and_proof` topic. -### Transitioning the gossip +#### Transitioning the gossip With any fork, the fork version, and thus the `ForkDigestValue`, change. Message types are unique per topic, and so for a smooth transition a node must temporarily subscribe to both the old and new topics. @@ -205,9 +205,9 @@ Post-fork: E.g. an attestation on the both the old and new topic is ignored like any duplicate. - Two epochs after the fork, pre-fork topics SHOULD be unsubscribed from. This is well after the configured `seen_ttl`. -## The Req/Resp domain +### The Req/Resp domain -### Req-Resp interaction +#### Req-Resp interaction An additional `` field is introduced to the `response_chunk` as defined in the Phase 0 document: @@ -221,7 +221,7 @@ On a non-zero `` with `ErrorMessage` payload, the `` is a In Altair and later forks, `` functions as a short meta-data, defined per req-resp method, and can parametrize the payload decoder. -#### `ForkDigest`-context +##### `ForkDigest`-context Starting with Altair, and in future forks, SSZ type definitions may change. For this common case, we define the `ForkDigest`-context: @@ -229,9 +229,9 @@ For this common case, we define the `ForkDigest`-context: A fixed-width 4 byte ``, set to the `ForkDigest` matching the chunk: `compute_fork_digest(fork_version, genesis_validators_root)`. -### Messages +#### Messages -#### BeaconBlocksByRange v2 +##### BeaconBlocksByRange v2 **Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_range/2/` @@ -246,7 +246,7 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: | `GENESIS_FORK_VERSION` | `phase0.SignedBeaconBlock` | | `ALTAIR_FORK_VERSION` | `altair.SignedBeaconBlock` | -#### BeaconBlocksByRoot v2 +##### BeaconBlocksByRoot v2 **Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_root/2/` @@ -261,7 +261,7 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: | `GENESIS_FORK_VERSION` | `phase0.SignedBeaconBlock` | | `ALTAIR_FORK_VERSION` | `altair.SignedBeaconBlock` | -#### GetMetaData v2 +##### GetMetaData v2 **Protocol ID:** `/eth2/beacon_chain/req/metadata/2/` @@ -279,7 +279,7 @@ Requests the MetaData of a peer, using the new `MetaData` definition given above that is extended from phase 0 in Altair. Other conditions for the `GetMetaData` protocol are unchanged from the phase 0 p2p networking document. -### Transitioning from v1 to v2 +#### Transitioning from v1 to v2 In advance of the fork, implementations can opt in to both run the v1 and v2 for a smooth transition. This is non-breaking, and is recommended as soon as the fork specification is stable. @@ -291,7 +291,7 @@ The v1 method MAY be unregistered at the fork boundary. In the event of a request on v1 for an Altair specific payload, the responder MUST return the **InvalidRequest** response code. -## The discovery domain: discv5 +### The discovery domain: discv5 The `attnets` key of the ENR is used as defined in the Phase 0 document. diff --git a/specs/bellatrix/p2p-interface.md b/specs/bellatrix/p2p-interface.md index 4d4044689b..b8b3a11d6b 100644 --- a/specs/bellatrix/p2p-interface.md +++ b/specs/bellatrix/p2p-interface.md @@ -13,23 +13,23 @@ Readers should understand the Phase 0 and Altair documents and use them as a bas - [Warning](#warning) -- [Modifications in Bellatrix](#modifications-in-bellatrix) - - [Configuration](#configuration) - - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) - - [Topics and messages](#topics-and-messages) - - [Global topics](#global-topics) - - [`beacon_block`](#beacon_block) - - [Transitioning the gossip](#transitioning-the-gossip) - - [The Req/Resp domain](#the-reqresp-domain) - - [Messages](#messages) - - [BeaconBlocksByRange v2](#beaconblocksbyrange-v2) - - [BeaconBlocksByRoot v2](#beaconblocksbyroot-v2) + - [Modifications in Bellatrix](#modifications-in-bellatrix) + - [Configuration](#configuration) + - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) + - [Topics and messages](#topics-and-messages) + - [Global topics](#global-topics) + - [`beacon_block`](#beacon_block) + - [Transitioning the gossip](#transitioning-the-gossip) + - [The Req/Resp domain](#the-reqresp-domain) + - [Messages](#messages) + - [BeaconBlocksByRange v2](#beaconblocksbyrange-v2) + - [BeaconBlocksByRoot v2](#beaconblocksbyroot-v2) - [Design decision rationale](#design-decision-rationale) - - [Gossipsub](#gossipsub) - - [Why was the max gossip message size increased at Bellatrix?](#why-was-the-max-gossip-message-size-increased-at-bellatrix) - - [Req/Resp](#reqresp) - - [Why was the max chunk response size increased at Bellatrix?](#why-was-the-max-chunk-response-size-increased-at-bellatrix) - - [Why allow invalid payloads on the P2P network?](#why-allow-invalid-payloads-on-the-p2p-network) + - [Gossipsub](#gossipsub) + - [Why was the max gossip message size increased at Bellatrix?](#why-was-the-max-gossip-message-size-increased-at-bellatrix) + - [Req/Resp](#reqresp) + - [Why was the max chunk response size increased at Bellatrix?](#why-was-the-max-chunk-response-size-increased-at-bellatrix) + - [Why allow invalid payloads on the P2P network?](#why-allow-invalid-payloads-on-the-p2p-network) @@ -39,9 +39,9 @@ Readers should understand the Phase 0 and Altair documents and use them as a bas This document is currently illustrative for early Bellatrix testnets and some parts are subject to change. Refer to the note in the [validator guide](./validator.md) for further details. -# Modifications in Bellatrix +## Modifications in Bellatrix -## Configuration +### Configuration This section outlines modifications constants that are used in this spec. @@ -50,11 +50,11 @@ This section outlines modifications constants that are used in this spec. | `GOSSIP_MAX_SIZE_BELLATRIX` | `10 * 2**20` (= 10,485,760, 10 MiB) | The maximum allowed size of uncompressed gossip messages starting at Bellatrix upgrade. | | `MAX_CHUNK_SIZE_BELLATRIX` | `10 * 2**20` (= 10,485,760, 10 MiB) | The maximum allowed size of uncompressed req/resp chunked responses starting at Bellatrix upgrade. | -## The gossip domain: gossipsub +### The gossip domain: gossipsub Some gossip meshes are upgraded in Bellatrix to support upgraded types. -### Topics and messages +#### Topics and messages Topics follow the same specification as in prior upgrades. All topics remain stable except the beacon block topic which is updated with the modified type. @@ -76,11 +76,11 @@ The new topics along with the type of the `data` field of a gossipsub message ar Note that the `ForkDigestValue` path segment of the topic separates the old and the new `beacon_block` topics. -#### Global topics +##### Global topics Bellatrix changes the type of the global beacon block topic. -##### `beacon_block` +###### `beacon_block` The *type* of the payload of this topic changes to the (modified) `SignedBeaconBlock` found in Bellatrix. Specifically, this type changes with the addition of `execution_payload` to the inner `BeaconBlockBody`. @@ -107,12 +107,12 @@ Alias `block = signed_beacon_block.message`, `execution_payload = block.body.exe The following gossip validation from prior specifications MUST NOT be applied if the execution is enabled for the block -- i.e. `is_execution_enabled(state, block.body)`: - [REJECT] The block's parent (defined by `block.parent_root`) passes validation. -### Transitioning the gossip +#### Transitioning the gossip See gossip transition details found in the [Altair document](../altair/p2p-interface.md#transitioning-the-gossip) for details on how to handle transitioning gossip topics. -## The Req/Resp domain +### The Req/Resp domain Non-faulty, [optimistic](/sync/optimistic.md) nodes may send blocks which result in an INVALID response from an execution engine. To prevent network @@ -122,9 +122,9 @@ down-scored or disconnected. Transmission of a block which is invalid due to any consensus layer rules (i.e., *not* execution layer rules) MAY result in down-scoring or disconnection. -### Messages +#### Messages -#### BeaconBlocksByRange v2 +##### BeaconBlocksByRange v2 **Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_range/2/` @@ -146,7 +146,7 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: | `ALTAIR_FORK_VERSION` | `altair.SignedBeaconBlock` | | `BELLATRIX_FORK_VERSION` | `bellatrix.SignedBeaconBlock` | -#### BeaconBlocksByRoot v2 +##### BeaconBlocksByRoot v2 **Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_root/2/` @@ -165,9 +165,9 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: # Design decision rationale -## Gossipsub +### Gossipsub -### Why was the max gossip message size increased at Bellatrix? +#### Why was the max gossip message size increased at Bellatrix? With the addition of `ExecutionPayload` to `BeaconBlock`s, there is a dynamic field -- `transactions` -- which can validly exceed the `GOSSIP_MAX_SIZE` limit (1 MiB) put in @@ -190,9 +190,9 @@ order of 128 KiB in the worst case and the current gas limit (~30M) bounds max b than 2 MiB today, this marginal difference in theoretical bounds will have zero impact on network functionality and security. -## Req/Resp +### Req/Resp -### Why was the max chunk response size increased at Bellatrix? +#### Why was the max chunk response size increased at Bellatrix? Similar to the discussion about the maximum gossip size increase, the `ExecutionPayload` type can cause `BeaconBlock`s to exceed the 1 MiB bounds put @@ -204,7 +204,7 @@ valid block sizes in the range of gas limits expected in the medium term. As with both gossip and req/rsp maximum values, type-specific limits should always by simultaneously respected. -### Why allow invalid payloads on the P2P network? +#### Why allow invalid payloads on the P2P network? The specification allows blocks with invalid execution payloads to propagate across gossip and via RPC calls. The reasoning for this is as follows: diff --git a/specs/capella/p2p-interface.md b/specs/capella/p2p-interface.md index 834fd44d88..a71b6479f1 100644 --- a/specs/capella/p2p-interface.md +++ b/specs/capella/p2p-interface.md @@ -4,7 +4,7 @@ This document contains the networking specification for Capella. The specification of these changes continues in the same format as the network specifications of previous upgrades, and assumes them as pre-requisite. -## Table of contents +### Table of contents @@ -26,13 +26,13 @@ The specification of these changes continues in the same format as the network s -# Modifications in Capella +## Modifications in Capella -## The gossip domain: gossipsub +### The gossip domain: gossipsub A new topic is added to support the gossip of withdrawal credential change messages. And an existing topic is upgraded for updated types in Capella. -### Topics and messages +#### Topics and messages Topics follow the same specification as in prior upgrades. All existing topics remain stable except the beacon block topic which is updated with the modified type. @@ -45,17 +45,17 @@ The new topics along with the type of the `data` field of a gossipsub message ar Note that the `ForkDigestValue` path segment of the topic separates the old and the new `beacon_block` topics. -#### Global topics +##### Global topics Capella changes the type of the global beacon block topic and adds one global topic to propagate withdrawal credential change messages to all potential proposers of beacon blocks. -##### `beacon_block` +###### `beacon_block` The *type* of the payload of this topic changes to the (modified) `SignedBeaconBlock` found in Capella. Specifically, this type changes with the addition of `bls_to_execution_changes` to the inner `BeaconBlockBody`. See Capella [state transition document](./beacon-chain.md#beaconblockbody) for further details. -##### `bls_to_execution_change` +###### `bls_to_execution_change` This topic is used to propagate signed bls to execution change messages to be included in future blocks. @@ -67,16 +67,16 @@ The following validations MUST pass before forwarding the `signed_bls_to_executi for the validator with index `signed_bls_to_execution_change.message.validator_index`. - _[REJECT]_ All of the conditions within `process_bls_to_execution_change` pass validation. -### Transitioning the gossip +#### Transitioning the gossip See gossip transition details found in the [Altair document](../altair/p2p-interface.md#transitioning-the-gossip) for details on how to handle transitioning gossip topics for Capella. -## The Req/Resp domain +### The Req/Resp domain -### Messages +#### Messages -#### BeaconBlocksByRange v2 +##### BeaconBlocksByRange v2 **Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_range/2/` @@ -93,7 +93,7 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: | `BELLATRIX_FORK_VERSION` | `bellatrix.SignedBeaconBlock` | | `CAPELLA_FORK_VERSION` | `capella.SignedBeaconBlock` | -#### BeaconBlocksByRoot v2 +##### BeaconBlocksByRoot v2 **Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_root/2/` diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 0b6381e20f..3c6f3c88a5 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -10,32 +10,35 @@ The specification of these changes continues in the same format as the network s -- [Configuration](#configuration) -- [Containers](#containers) - - [`BlobSidecar`](#blobsidecar) - - [`SignedBlobSidecar`](#signedblobsidecar) - - [`BlobIdentifier`](#blobidentifier) - - [Helpers](#helpers) - - [`verify_blob_sidecar_signature`](#verify_blob_sidecar_signature) -- [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) - - [Topics and messages](#topics-and-messages) - - [Global topics](#global-topics) - - [`beacon_block`](#beacon_block) - - [`blob_sidecar_{index}`](#blob_sidecar_index) - - [Transitioning the gossip](#transitioning-the-gossip) -- [The Req/Resp domain](#the-reqresp-domain) - - [Messages](#messages) - - [BeaconBlocksByRange v2](#beaconblocksbyrange-v2) - - [BeaconBlocksByRoot v2](#beaconblocksbyroot-v2) - - [BlobSidecarsByRoot v1](#blobsidecarsbyroot-v1) - - [BlobSidecarsByRange v1](#blobsidecarsbyrange-v1) +- [Modifications in Deneb](#modifications-in-deneb) + - [Configuration](#configuration) + - [Containers](#containers) + - [`BlobSidecar`](#blobsidecar) + - [`SignedBlobSidecar`](#signedblobsidecar) + - [`BlobIdentifier`](#blobidentifier) + - [Helpers](#helpers) + - [`verify_blob_sidecar_signature`](#verify_blob_sidecar_signature) + - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) + - [Topics and messages](#topics-and-messages) + - [Global topics](#global-topics) + - [`beacon_block`](#beacon_block) + - [`blob_sidecar_{index}`](#blob_sidecar_index) + - [Transitioning the gossip](#transitioning-the-gossip) + - [The Req/Resp domain](#the-reqresp-domain) + - [Messages](#messages) + - [BeaconBlocksByRange v2](#beaconblocksbyrange-v2) + - [BeaconBlocksByRoot v2](#beaconblocksbyroot-v2) + - [BlobSidecarsByRoot v1](#blobsidecarsbyroot-v1) + - [BlobSidecarsByRange v1](#blobsidecarsbyrange-v1) - [Design decision rationale](#design-decision-rationale) - [Why are blobs relayed as a sidecar, separate from beacon blocks?](#why-are-blobs-relayed-as-a-sidecar-separate-from-beacon-blocks) -## Configuration +## Modifications in Deneb + +### Configuration | Name | Value | Description | |------------------------------------------|-----------------------------------|---------------------------------------------------------------------| @@ -43,9 +46,9 @@ The specification of these changes continues in the same format as the network s | `MAX_REQUEST_BLOB_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK` | Maximum number of blob sidecars in a single request | | `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars | -## Containers +### Containers -### `BlobSidecar` +#### `BlobSidecar` ```python class BlobSidecar(Container): @@ -59,7 +62,7 @@ class BlobSidecar(Container): kzg_proof: KZGProof # Allows for quick verification of kzg_commitment ``` -### `SignedBlobSidecar` +#### `SignedBlobSidecar` ```python class SignedBlobSidecar(Container): @@ -67,7 +70,7 @@ class SignedBlobSidecar(Container): signature: BLSSignature ``` -### `BlobIdentifier` +#### `BlobIdentifier` ```python class BlobIdentifier(Container): @@ -75,9 +78,9 @@ class BlobIdentifier(Container): index: BlobIndex ``` -### Helpers +#### Helpers -#### `verify_blob_sidecar_signature` +##### `verify_blob_sidecar_signature` ```python def verify_blob_sidecar_signature(state: BeaconState, signed_blob_sidecar: SignedBlobSidecar) -> bool: @@ -86,11 +89,11 @@ def verify_blob_sidecar_signature(state: BeaconState, signed_blob_sidecar: Signe return bls.Verify(proposer.pubkey, signing_root, signed_blob_sidecar.signature) ``` -## The gossip domain: gossipsub +### The gossip domain: gossipsub Some gossip meshes are upgraded in the fork of Deneb to support upgraded types. -### Topics and messages +#### Topics and messages Topics follow the same specification as in prior upgrades. @@ -106,15 +109,15 @@ The new topics along with the type of the `data` field of a gossipsub message ar | - | - | | `blob_sidecar_{index}` | `SignedBlobSidecar` (new) | -#### Global topics +##### Global topics Deneb introduces new global topics for blob sidecars. -##### `beacon_block` +###### `beacon_block` The *type* of the payload of this topic changes to the (modified) `SignedBeaconBlock` found in deneb. -##### `blob_sidecar_{index}` +###### `blob_sidecar_{index}` This topic is used to propagate signed blob sidecars, one for each sidecar index. The number of indices is defined by `MAX_BLOBS_PER_BLOCK`. @@ -132,16 +135,16 @@ The following validations MUST pass before forwarding the `signed_blob_sidecar` If the `proposer_index` cannot immediately be verified against the expected shuffling, the sidecar MAY be queued for later processing while proposers for the block's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message. -### Transitioning the gossip +#### Transitioning the gossip See gossip transition details found in the [Altair document](../altair/p2p-interface.md#transitioning-the-gossip) for details on how to handle transitioning gossip topics for this upgrade. -## The Req/Resp domain +### The Req/Resp domain -### Messages +#### Messages -#### BeaconBlocksByRange v2 +##### BeaconBlocksByRange v2 **Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_range/2/` @@ -161,7 +164,7 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: No more than `MAX_REQUEST_BLOCKS_DENEB` may be requested at a time. -#### BeaconBlocksByRoot v2 +##### BeaconBlocksByRoot v2 **Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_root/2/` @@ -179,7 +182,7 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`: No more than `MAX_REQUEST_BLOCKS_DENEB` may be requested at a time. -#### BlobSidecarsByRoot v1 +##### BlobSidecarsByRoot v1 **Protocol ID:** `/eth2/beacon_chain/req/blob_sidecars_by_root/1/` @@ -228,7 +231,7 @@ Clients MUST support requesting sidecars since `minimum_request_epoch`, where `m Clients MUST respond with at least one sidecar, if they have it. Clients MAY limit the number of blocks and sidecars in the response. -#### BlobSidecarsByRange v1 +##### BlobSidecarsByRange v1 **Protocol ID:** `/eth2/beacon_chain/req/blob_sidecars_by_range/1/` diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index f527529316..f38601ed86 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -111,11 +111,11 @@ It consists of four main sections: -# Network fundamentals +## Network fundamentals This section outlines the specification for the networking stack in Ethereum consensus-layer clients. -## Transport +### Transport Even though libp2p is a multi-transport stack (designed to listen on multiple simultaneous transports and endpoints transparently), we hereby define a profile for basic interoperability. @@ -133,14 +133,14 @@ All listening endpoints must be publicly dialable, and thus not rely on libp2p c Nodes operating behind a NAT, or otherwise undialable by default (e.g. container runtime, firewall, etc.), MUST have their infrastructure configured to enable inbound traffic on the announced public listening endpoint. -## Encryption and identification +### Encryption and identification The [Libp2p-noise](https://github.com/libp2p/specs/tree/master/noise) secure channel handshake with `secp256k1` identities will be used for encryption. As specified in the libp2p specification, clients MUST support the `XX` handshake pattern. -## Protocol Negotiation +### Protocol Negotiation Clients MUST use exact equality when negotiating protocol versions to use and MAY use the version to give priority to higher version numbers. @@ -148,7 +148,7 @@ Clients MUST support [multistream-select 1.0](https://github.com/multiformats/mu and MAY support [multiselect 2.0](https://github.com/libp2p/specs/pull/95) when the spec solidifies. Once all clients have implementations for multiselect 2.0, multistream-select 1.0 MAY be phased out. -## Multiplexing +### Multiplexing During connection bootstrapping, libp2p dynamically negotiates a mutually supported multiplexing method to conduct parallel conversations. This applies to transports that are natively incapable of multiplexing (e.g. TCP, WebSockets, WebRTC), @@ -163,9 +163,9 @@ and MAY support [yamux](https://github.com/hashicorp/yamux/blob/master/spec.md). If both are supported by the client, yamux MUST take precedence during negotiation. See the [Rationale](#design-decision-rationale) section below for tradeoffs. -# Consensus-layer network interaction domains +## Consensus-layer network interaction domains -## Configuration +### Configuration This section outlines constants that are used in this spec. @@ -182,7 +182,7 @@ This section outlines constants that are used in this spec. | `MESSAGE_DOMAIN_INVALID_SNAPPY` | `0x00000000` | 4-byte domain for gossip message-id isolation of *invalid* snappy messages | | `MESSAGE_DOMAIN_VALID_SNAPPY` | `0x01000000` | 4-byte domain for gossip message-id isolation of *valid* snappy messages | -## MetaData +### MetaData Clients MUST locally store the following `MetaData`: @@ -203,7 +203,7 @@ Where is entirely independent of the ENR sequence number, and will in most cases be out of sync with the ENR sequence number. -## The gossip domain: gossipsub +### The gossip domain: gossipsub Clients MUST support the [gossipsub v1](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.0.md) libp2p Protocol including the [gossipsub v1.1](https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md) extension. @@ -229,7 +229,7 @@ The following gossipsub [parameters](https://github.com/libp2p/specs/blob/master for peer scoring and other attack mitigations. These are currently under investigation and will be spec'd and released to mainnet when they are ready. -### Topics and messages +#### Topics and messages Topics are plain UTF-8 strings and are encoded on the wire as determined by protobuf (gossipsub messages are enveloped in protobuf messages). Topic strings have form: `/eth2/ForkDigestValue/Name/Encoding`. @@ -289,7 +289,7 @@ We utilize `ACCEPT`, `REJECT`, and `IGNORE`. For each gossipsub topic, there are If all validations pass, return `ACCEPT`. If one or more validations fail while processing the items in order, return either `REJECT` or `IGNORE` as specified in the prefix of the particular condition. -#### Global topics +##### Global topics There are two primary global topics used to propagate beacon blocks (`beacon_block`) and aggregate attestations (`beacon_aggregate_and_proof`) to all nodes on the network. @@ -297,7 +297,7 @@ and aggregate attestations (`beacon_aggregate_and_proof`) to all nodes on the ne There are three additional global topics that are used to propagate lower frequency validator messages (`voluntary_exit`, `proposer_slashing`, and `attester_slashing`). -##### `beacon_block` +###### `beacon_block` The `beacon_block` topic is used solely for propagating new signed beacon blocks to all nodes on the networks. Signed blocks are sent in their entirety. @@ -325,7 +325,7 @@ The following validations MUST pass before forwarding the `signed_beacon_block` the block MAY be queued for later processing while proposers for the block's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message. -##### `beacon_aggregate_and_proof` +###### `beacon_aggregate_and_proof` The `beacon_aggregate_and_proof` topic is used to propagate aggregated attestations (as `SignedAggregateAndProof`s) to subscribing nodes (typically validators) to be included in future blocks. @@ -360,7 +360,7 @@ The following validations MUST pass before forwarding the `signed_aggregate_and_ == store.finalized_checkpoint.root` -##### `voluntary_exit` +###### `voluntary_exit` The `voluntary_exit` topic is used solely for propagating signed voluntary validator exits to proposers on the network. Signed voluntary exits are sent in their entirety. @@ -370,7 +370,7 @@ The following validations MUST pass before forwarding the `signed_voluntary_exit for the validator with index `signed_voluntary_exit.message.validator_index`. - _[REJECT]_ All of the conditions within `process_voluntary_exit` pass validation. -##### `proposer_slashing` +###### `proposer_slashing` The `proposer_slashing` topic is used solely for propagating proposer slashings to proposers on the network. Proposer slashings are sent in their entirety. @@ -380,7 +380,7 @@ The following validations MUST pass before forwarding the `proposer_slashing` on for the proposer with index `proposer_slashing.signed_header_1.message.proposer_index`. - _[REJECT]_ All of the conditions within `process_proposer_slashing` pass validation. -##### `attester_slashing` +###### `attester_slashing` The `attester_slashing` topic is used solely for propagating attester slashings to proposers on the network. Attester slashings are sent in their entirety. @@ -392,11 +392,11 @@ Clients who receive an attester slashing on this topic MUST validate the conditi verify if `any(attester_slashed_indices.difference(prior_seen_attester_slashed_indices))`). - _[REJECT]_ All of the conditions within `process_attester_slashing` pass validation. -#### Attestation subnets +##### Attestation subnets Attestation subnets are used to propagate unaggregated attestations to subsections of the network. -##### `beacon_attestation_{subnet_id}` +###### `beacon_attestation_{subnet_id}` The `beacon_attestation_{subnet_id}` topics are used to propagate unaggregated attestations to the subnet `subnet_id` (typically beacon and persistent committees) to be aggregated before being gossiped to `beacon_aggregate_and_proof`. @@ -432,7 +432,7 @@ The following validations MUST pass before forwarding the `attestation` on the s -#### Attestations and Aggregation +##### Attestations and Aggregation Attestation broadcasting is grouped into subnets defined by a topic. The number of subnets is defined via `ATTESTATION_SUBNET_COUNT`. @@ -445,7 +445,7 @@ Unaggregated attestations are sent as `Attestation`s to the subnet topic, Aggregated attestations are sent to the `beacon_aggregate_and_proof` topic as `AggregateAndProof`s. -### Encodings +#### Encodings Topics are post-fixed with an encoding. Encodings define how the payload of a gossipsub message is encoded. @@ -461,9 +461,9 @@ so [basic snappy block compression](https://github.com/google/snappy/blob/master Implementations MUST use a single encoding for gossip. Changing an encoding will require coordination between participating implementations. -## The Req/Resp domain +### The Req/Resp domain -### Protocol identification +#### Protocol identification Each message type is segregated into its own libp2p protocol ID, which is a case-sensitive UTF-8 string of the form: @@ -485,7 +485,7 @@ With: This protocol segregation allows libp2p `multistream-select 1.0` / `multiselect 2.0` to handle the request type, version, and encoding negotiation before establishing the underlying streams. -### Req/Resp interaction +#### Req/Resp interaction We use ONE stream PER request/response interaction. Streams are closed when the interaction finishes, whether in success or in error. @@ -515,7 +515,7 @@ Regardless of these type specific bounds, a global maximum uncompressed byte siz Clients MUST ensure that lengths are within these bounds; if not, they SHOULD reset the stream immediately. Clients tracking peer reputation MAY decrement the score of the misbehaving peer under this circumstance. -#### Requesting side +##### Requesting side Once a new stream with the protocol ID for the request type has been negotiated, the full request message SHOULD be sent immediately. The request MUST be encoded according to the encoding strategy. @@ -537,7 +537,7 @@ A requester SHOULD read from the stream until either: For requests consisting of a single valid `response_chunk`, the requester SHOULD read the chunk fully, as defined by the `encoding-dependent-header`, before closing the stream. -#### Responding side +##### Responding side Once a new stream with the protocol ID for the request type has been negotiated, the responder SHOULD process the incoming request and MUST validate it before processing it. @@ -588,7 +588,7 @@ The `ErrorMessage` schema is: *Note*: By convention, the `error_message` is a sequence of bytes that MAY be interpreted as a UTF-8 string (for debugging purposes). Clients MUST treat as valid any byte sequences. -### Encoding strategies +#### Encoding strategies The token of the negotiated protocol ID specifies the type of encoding to be used for the req/resp interaction. Only one value is possible at this time: @@ -599,7 +599,7 @@ Only one value is possible at this time: For example, the `BeaconBlocksByRoot` request is an SSZ-encoded list of `Root`'s. This encoding type MUST be supported by all clients. -#### SSZ-snappy encoding strategy +##### SSZ-snappy encoding strategy The [SimpleSerialize (SSZ) specification](../../ssz/simple-serialize.md) outlines how objects are SSZ-encoded. @@ -646,9 +646,9 @@ constituents individually as `response_chunk`s. For example, the `List[SignedBeaconBlock, ...]` response type sends zero or more `response_chunk`s. Each _successful_ `response_chunk` contains a single `SignedBeaconBlock` payload. -### Messages +#### Messages -#### Status +##### Status **Protocol ID:** ``/eth2/beacon_chain/req/status/1/`` @@ -694,7 +694,7 @@ SHOULD request beacon blocks from its counterparty via the `BeaconBlocksByRange` the client might need to send `Status` request again to learn if the peer has a higher head. Implementers are free to implement such behavior in their own way. -#### Goodbye +##### Goodbye **Protocol ID:** ``/eth2/beacon_chain/req/goodbye/1/`` @@ -718,7 +718,7 @@ The request/response MUST be encoded as a single SSZ-field. The response MUST consist of a single `response_chunk`. -#### BeaconBlocksByRange +##### BeaconBlocksByRange **Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_range/1/` @@ -795,7 +795,7 @@ In particular when `step == 1`, each `parent_root` MUST match the `hash_tree_roo After the initial block, clients MAY stop in the process of responding if their fork choice changes the view of the chain in the context of the request. -#### BeaconBlocksByRoot +##### BeaconBlocksByRoot **Protocol ID:** `/eth2/beacon_chain/req/beacon_blocks_by_root/1/` @@ -835,7 +835,7 @@ Clients MAY limit the number of blocks in the response. `/eth2/beacon_chain/req/beacon_blocks_by_root/1/` is deprecated. Clients MAY respond with an empty list during the deprecation transition period. -#### Ping +##### Ping **Protocol ID:** `/eth2/beacon_chain/req/ping/1/` @@ -867,7 +867,7 @@ The request MUST be encoded as an SSZ-field. The response MUST consist of a single `response_chunk`. -#### GetMetaData +##### GetMetaData **Protocol ID:** `/eth2/beacon_chain/req/metadata/1/` @@ -890,14 +890,14 @@ The response MUST be encoded as an SSZ-container. The response MUST consist of a single `response_chunk`. -## The discovery domain: discv5 +### The discovery domain: discv5 Discovery Version 5 ([discv5](https://github.com/ethereum/devp2p/blob/master/discv5/discv5.md)) (Protocol version v5.1) is used for peer discovery. `discv5` is a standalone protocol, running on UDP on a dedicated port, meant for peer discovery only. `discv5` supports self-certified, flexible peer records (ENRs) and topic-based advertisement, both of which are (or will be) requirements in this context. -### Integration into libp2p stacks +#### Integration into libp2p stacks `discv5` SHOULD be integrated into the client’s libp2p stack by implementing an adaptor to make it conform to the [service discovery](https://github.com/libp2p/go-libp2p-core/blob/master/discovery/discovery.go) @@ -908,7 +908,7 @@ and the outputs will be multiaddrs converted from the ENR records returned by th This integration enables the libp2p stack to subsequently form connections and streams with discovered peers. -### ENR structure +#### ENR structure The Ethereum Node Record (ENR) for an Ethereum consensus client MUST contain the following entries (exclusive of the sequence number and signature, which MUST be present in an ENR): @@ -923,7 +923,7 @@ The ENR MAY contain the following entries: Specifications of these parameters can be found in the [ENR Specification](http://eips.ethereum.org/EIPS/eip-778). -#### Attestation subnet bitfield +##### Attestation subnet bitfield The ENR `attnets` entry signifies the attestation subnet bitfield with the following form to more easily discover peers participating in particular attestation gossip subnets. @@ -936,7 +936,7 @@ If a node's `MetaData.attnets` has any non-zero bit, the ENR MUST include the `a If a node's `MetaData.attnets` is composed of all zeros, the ENR MAY optionally include the `attnets` entry or leave it out entirely. -#### `eth2` field +##### `eth2` field ENRs MUST carry a generic `eth2` key with an 16-byte value of the node's current fork digest, next fork version, and next fork epoch to ensure connections are made with peers on the intended Ethereum network. @@ -979,11 +979,11 @@ Clients MAY connect to peers with the same `fork_digest` but a different `next_f Unless `ENRForkID` is manually updated to matching prior to the earlier `next_fork_epoch` of the two clients, these connecting clients will be unable to successfully interact starting at the earlier `next_fork_epoch`. -# Design decision rationale +## Design decision rationale -## Transport +### Transport -### Why are we defining specific transports? +#### Why are we defining specific transports? libp2p peers can listen on multiple transports concurrently, and these can change over time. Multiaddrs encode not only the address but also the transport to be used to dial. @@ -992,7 +992,7 @@ Due to this dynamic nature, agreeing on specific transports like TCP, QUIC, or W However, it is useful to define a minimum baseline for interoperability purposes. -### Can clients support other transports/handshakes than the ones mandated by the spec? +#### Can clients support other transports/handshakes than the ones mandated by the spec? Clients may support other transports such as libp2p QUIC, WebSockets, and WebRTC transports, if available in the language of choice. While interoperability shall not be harmed by lack of such support, the advantages are desirable: @@ -1007,7 +1007,7 @@ and the accompanying [QUIC-TLS document](https://tools.ietf.org/html/draft-ietf- The usage of one handshake procedure or the other shall be transparent to the application layer, once the libp2p Host/Node object has been configured appropriately. -### What are the advantages of using TCP/QUIC/Websockets? +#### What are the advantages of using TCP/QUIC/Websockets? TCP is a reliable, ordered, full-duplex, congestion-controlled network protocol that powers much of the Internet as we know it today. HTTP/1.1 and HTTP/2 run atop TCP. @@ -1027,7 +1027,7 @@ and we may only become subject to standard IP-based firewall filtering—somethi WebSockets and/or WebRTC transports are necessary for interaction with browsers, and will become increasingly important as we incorporate browser-based light clients to the Ethereum network. -### Why do we not just support a single transport? +#### Why do we not just support a single transport? Networks evolve. Hardcoding design decisions leads to ossification, preventing the evolution of networks alongside the state of the art. @@ -1039,7 +1039,7 @@ Clients can adopt new transports without breaking old ones, and the multi-transp (e.g. browsers, embedded devices) to interact with the network as first-class citizens via suitable/native transports (e.g. WSS), without the need for proxying or trust delegation to servers. -### Why are we not using QUIC from the start? +#### Why are we not using QUIC from the start? The QUIC standard is still not finalized (at working draft 22 at the time of writing), and not all mainstream runtimes/languages have mature, standard, and/or fully-interoperable [QUIC support](https://github.com/quicwg/base-drafts/wiki/Implementations). @@ -1052,9 +1052,9 @@ On the other hand, TLS 1.3 is the newest, simplified iteration of TLS. Old, insecure, obsolete ciphers and algorithms have been removed, adopting Ed25519 as the sole ECDH key agreement function. Handshakes are faster, 1-RTT data is supported, and session resumption is a reality, amongst other features. -## Multiplexing +### Multiplexing -### Why are we using mplex/yamux? +#### Why are we using mplex/yamux? [Yamux](https://github.com/hashicorp/yamux/blob/master/spec.md) is a multiplexer invented by Hashicorp that supports stream-level congestion control. Implementations exist in a limited set of languages, and it’s not a trivial piece to develop. @@ -1066,9 +1066,9 @@ It does not support stream-level congestion control and is subject to head-of-li Overlay multiplexers are not necessary with QUIC since the protocol provides native multiplexing, but they need to be layered atop TCP, WebSockets, and other transports that lack such support. -## Protocol Negotiation +### Protocol Negotiation -### When is multiselect 2.0 due and why do we plan to migrate to it? +#### When is multiselect 2.0 due and why do we plan to migrate to it? multiselect 2.0 is currently being conceptualized. The debate started [on this issue](https://github.com/libp2p/specs/pull/95), @@ -1084,7 +1084,7 @@ We plan to eventually migrate to multiselect 2.0 because it will: 3. Leverage *push data* mechanisms of underlying protocols to expedite negotiation. 4. Provide the building blocks for enhanced censorship resistance. -### What is the difference between connection-level and stream-level protocol negotiation? +#### What is the difference between connection-level and stream-level protocol negotiation? All libp2p connections must be authenticated, encrypted, and multiplexed. Connections using network transports unsupportive of native authentication/encryption and multiplexing (e.g. TCP) need to undergo protocol negotiation to agree on a mutually supported: @@ -1101,9 +1101,9 @@ When opening streams, peers pin a protocol to that stream, by conducting *stream At present, multistream-select 1.0 is used for both types of negotiation, but multiselect 2.0 will use dedicated mechanisms for connection bootstrapping process and stream protocol negotiation. -## Encryption +### Encryption -### Why are we not supporting SecIO? +#### Why are we not supporting SecIO? SecIO has been the default encryption layer for libp2p for years. It is used in IPFS and Filecoin. And although it will be superseded shortly, it is proven to work at scale. @@ -1114,7 +1114,7 @@ a mechanism that multiselect 2.0 will leverage to reduce round trips during conn SecIO is not considered secure for the purposes of this spec. -### Why are we using Noise? +#### Why are we using Noise? Copied from the Noise Protocol Framework [website](http://www.noiseprotocol.org): @@ -1129,7 +1129,7 @@ and are used in major cryptographic-centric projects like WireGuard, I2P, and Li [Various](https://www.wireguard.com/papers/kobeissi-bhargavan-noise-explorer-2018.pdf) [studies](https://eprint.iacr.org/2019/436.pdf) have assessed the stated security goals of several Noise handshakes with positive results. -### Why are we using encryption at all? +#### Why are we using encryption at all? Transport level encryption secures message exchange and provides properties that are useful for privacy, safety, and censorship resistance. These properties are derived from the following security guarantees that apply to the entire communication between two peers: @@ -1146,9 +1146,9 @@ Note that transport-level encryption is not exclusive of application-level encry Transport-level encryption secures the communication itself, while application-level cryptography is necessary for the application’s use cases (e.g. signatures, randomness, etc.). -## Gossipsub +### Gossipsub -### Why are we using a pub/sub algorithm for block and attestation propagation? +#### Why are we using a pub/sub algorithm for block and attestation propagation? Pubsub is a technique to broadcast/disseminate data across a network rapidly. Such data is packaged in fire-and-forget messages that do not require a response from every recipient. @@ -1156,18 +1156,18 @@ Peers subscribed to a topic participate in the propagation of messages in that t The alternative is to maintain a fully connected mesh (all peers connected to each other 1:1), which scales poorly (O(n^2)). -### Why are we using topics to segregate encodings, yet only support one encoding? +#### Why are we using topics to segregate encodings, yet only support one encoding? For future extensibility with almost zero overhead now (besides the extra bytes in the topic name). -### How do we upgrade gossip channels (e.g. changes in encoding, compression)? +#### How do we upgrade gossip channels (e.g. changes in encoding, compression)? Changing gossipsub/broadcasts requires a coordinated upgrade where all clients start publishing to the new topic together, during a hard fork. When a node is preparing for upcoming tasks (e.g. validator duty lookahead) on a gossipsub topic, the node should join the topic of the future epoch in which the task is to occur in addition to listening to the topics for the current epoch. -### Why must all clients use the same gossip topic instead of one negotiated between each peer pair? +#### Why must all clients use the same gossip topic instead of one negotiated between each peer pair? Supporting multiple topics/encodings would require the presence of relayers to translate between encodings and topics so as to avoid network fragmentation where participants have diverging views on the gossiped state, @@ -1182,7 +1182,7 @@ but the price here is pretty high in terms of overhead -- both computational and It is permitted for clients to publish data on alternative topics as long as they also publish on the network-wide mandatory topic. -### Why are the topics strings and not hashes? +#### Why are the topics strings and not hashes? Topic names have a hierarchical structure. In the future, gossipsub may support wildcard subscriptions @@ -1195,14 +1195,14 @@ since the domain is finite anyway, and calculating a digest's preimage would be Furthermore, the topic names are shorter than their digest equivalents (assuming SHA-256 hash), so hashing topics would bloat messages unnecessarily. -### Why are we using the `StrictNoSign` signature policy? +#### Why are we using the `StrictNoSign` signature policy? The policy omits the `from` (1), `seqno` (3), `signature` (5) and `key` (6) fields. These fields would: - Expose origin of sender (`from`), type of sender (based on `seqno`) - Add extra unused data to the gossip, since message IDs are based on `data`, not on the `from` and `seqno`. - Introduce more message validation than necessary, e.g. no `signature`. -### Why are we overriding the default libp2p pubsub `message-id`? +#### Why are we overriding the default libp2p pubsub `message-id`? For our current purposes, there is no need to address messages based on source peer, or track a message `seqno`. By overriding the default `message-id` to use content-addressing we can filter unnecessary duplicates before hitting the application layer. @@ -1214,7 +1214,7 @@ Some examples of where messages could be duplicated: Partial aggregates could be duplicated * Clients re-publishing seen messages -### Why are these specific gossip parameters chosen? +#### Why are these specific gossip parameters chosen? - `D`, `D_low`, `D_high`, `D_lazy`: recommended defaults. - `heartbeat_interval`: 0.7 seconds, recommended for the beacon chain in the [GossipSub evaluation report by Protocol Labs](https://gateway.ipfs.io/ipfs/QmRAFP5DBnvNjdYSbWhEhVRJJDFCLpPyvew5GwCCB4VxM4). @@ -1233,7 +1233,7 @@ Some examples of where messages could be duplicated: Attestation gossip validity is bounded by an epoch, so this is the safe max bound. -### Why is there `MAXIMUM_GOSSIP_CLOCK_DISPARITY` when validating slot ranges of messages in gossip subnets? +#### Why is there `MAXIMUM_GOSSIP_CLOCK_DISPARITY` when validating slot ranges of messages in gossip subnets? For some gossip channels (e.g. those for Attestations and BeaconBlocks), there are designated ranges of slots during which particular messages can be sent, @@ -1247,14 +1247,14 @@ For minimum and maximum allowable slot broadcast times, Although messages can at times be eagerly gossiped to the network, the node's fork choice prevents integration of these messages into the actual consensus until the _actual local start_ of the designated slot. -### Why are there `ATTESTATION_SUBNET_COUNT` attestation subnets? +#### Why are there `ATTESTATION_SUBNET_COUNT` attestation subnets? Depending on the number of validators, it may be more efficient to group shard subnets and might provide better stability for the gossipsub channel. The exact grouping will be dependent on more involved network tests. This constant allows for more flexibility in setting up the network topology for attestation aggregation (as aggregation should happen on each subnet). The value is currently set to be equal to `MAX_COMMITTEES_PER_SLOT` if/until network tests indicate otherwise. -### Why are attestations limited to be broadcast on gossip channels within `SLOTS_PER_EPOCH` slots? +#### Why are attestations limited to be broadcast on gossip channels within `SLOTS_PER_EPOCH` slots? Attestations can only be included on chain within an epoch's worth of slots so this is the natural cutoff. There is no utility to the chain to broadcast attestations older than one epoch, @@ -1265,7 +1265,7 @@ In addition to this, relaying attestations requires validating the attestation i Thus, validating arbitrarily old attestations would put additional requirements on which states need to be readily available to the node. This would result in a higher resource burden and could serve as a DoS vector. -### Why are aggregate attestations broadcast to the global topic as `AggregateAndProof`s rather than just as `Attestation`s? +#### Why are aggregate attestations broadcast to the global topic as `AggregateAndProof`s rather than just as `Attestation`s? The dominant strategy for an individual validator is to always broadcast an aggregate containing their own attestation to the global channel to ensure that proposers see their attestation for inclusion. @@ -1275,19 +1275,19 @@ the gossiped aggregate ensures that this dominant strategy will not flood the gl Also, an attacker can create any number of honest-looking aggregates and broadcast them to the global pubsub channel. Thus without some sort of proof of selection as an aggregator, the global channel can trivially be spammed. -### Why are we sending entire objects in the pubsub and not just hashes? +#### Why are we sending entire objects in the pubsub and not just hashes? Entire objects should be sent to get the greatest propagation speeds. If only hashes are sent, then block and attestation propagation is dependent on recursive requests from each peer. In a hash-only scenario, peers could receive hashes without knowing who to download the actual contents from. Sending entire objects ensures that they get propagated through the entire network. -### Should clients gossip blocks if they *cannot* validate the proposer signature due to not yet being synced, not knowing the head block, etc? +#### Should clients gossip blocks if they *cannot* validate the proposer signature due to not yet being synced, not knowing the head block, etc? The prohibition of unverified-block-gossiping extends to nodes that cannot verify a signature due to not being fully synced to ensure that such (amplified) DOS attacks are not possible. -### How are we going to discover peers in a gossipsub topic? +#### How are we going to discover peers in a gossipsub topic? In Phase 0, peers for attestation subnets will be found using the `attnets` entry in the ENR. @@ -1295,7 +1295,7 @@ Although this method will be sufficient for early upgrade of the beacon chain, w ENRs should ultimately not be used for this purpose. They are best suited to store identity, location, and capability information, rather than more volatile advertisements. -### How should fork version be used in practice? +#### How should fork version be used in practice? Fork versions are to be manually updated (likely via incrementing) at each hard fork. This is to provide native domain separation for signatures as well as to aid in usefulness for identitying peers (via ENRs) @@ -1308,9 +1308,9 @@ In these cases, extra care should be taken to isolate fork versions (e.g. flip a A node locally stores all previous and future planned fork versions along with the each fork epoch. This allows for handling sync and processing messages starting from past forks/epochs. -## Req/Resp +### Req/Resp -### Why segregate requests into dedicated protocol IDs? +#### Why segregate requests into dedicated protocol IDs? Requests are segregated by protocol ID to: @@ -1343,7 +1343,7 @@ Multiselect 2.0 will eventually remove this overhead by memoizing previously sel Fortunately, this req/resp protocol is not the expected network bottleneck in the protocol so the additional overhead is not expected to significantly hinder this domain. -### Why are messages length-prefixed with a protobuf varint in the SSZ-encoding? +#### Why are messages length-prefixed with a protobuf varint in the SSZ-encoding? We are using single-use streams where each stream is closed at the end of the message. Thus, libp2p transparently handles message delimiting in the underlying stream. @@ -1361,7 +1361,7 @@ Nevertheless, in the case of `ssz_snappy`, messages are still length-prefixed wi [Protobuf varint](https://developers.google.com/protocol-buffers/docs/encoding#varints) is an efficient technique to encode variable-length (unsigned here) ints. Instead of reserving a fixed-size field of as many bytes as necessary to convey the maximum possible value, this field is elastic in exchange for 1-bit overhead per byte. -### Why do we version protocol strings with ordinals instead of semver? +#### Why do we version protocol strings with ordinals instead of semver? Using semver for network protocols is confusing. It is never clear what a change in a field, even if backwards compatible on deserialization, actually implies. @@ -1382,11 +1382,11 @@ because it's unclear if "backwards compatibility" and "breaking change" apply on For this reason, we remove and replace semver with ordinals that require explicit agreement and do not mandate a specific policy for changes. -### Why is it called Req/Resp and not RPC? +#### Why is it called Req/Resp and not RPC? Req/Resp is used to avoid confusion with JSON-RPC and similar user-client interaction mechanisms. -### Why do we allow empty responses in block requests? +#### Why do we allow empty responses in block requests? When requesting blocks by range or root, it may happen that there are no blocks in the selected range or the responding node does not have the requested blocks. @@ -1413,7 +1413,7 @@ Failing to provide blocks that nodes "should" have is reason to trust a peer les -- for example, if a particular peer gossips a block, it should have access to its parent. If a request for the parent fails, it's indicative of poor peer quality since peers should validate blocks before gossiping them. -### Why does `BeaconBlocksByRange` let the server choose which branch to send blocks from? +#### Why does `BeaconBlocksByRange` let the server choose which branch to send blocks from? When connecting, the `Status` message gives an idea about the sync status of a particular peer, but this changes over time. By the time a subsequent `BeaconBlockByRange` request is processed, the information may be stale, @@ -1423,7 +1423,7 @@ To avoid this race condition, we allow the responding side to choose which branc The requesting client then goes on to validate the blocks and incorporate them in their own database -- because they follow the same rules, they should at this point arrive at the same canonical chain. -### Why are `BlocksByRange` requests only required to be served for the latest `MIN_EPOCHS_FOR_BLOCK_REQUESTS` epochs? +#### Why are `BlocksByRange` requests only required to be served for the latest `MIN_EPOCHS_FOR_BLOCK_REQUESTS` epochs? Due to economic finality and weak subjectivity requirements of a proof-of-stake blockchain, for a new node to safely join the network the node must provide a recent checkpoint found out-of-band. This checkpoint can be in the form of a `root` & `epoch` or it can be the entire @@ -1447,7 +1447,7 @@ MIN_EPOCHS_FOR_BLOCK_REQUESTS = ( Where `MAX_SAFETY_DECAY = 100` and thus `MIN_EPOCHS_FOR_BLOCK_REQUESTS = 33024` (~5 months). -### Why must the proposer signature be checked when backfilling blocks in the database? +#### Why must the proposer signature be checked when backfilling blocks in the database? When backfilling blocks in a database from a know safe block/state (e.g. when starting from a weak subjectivity state), the node not only must ensure the `BeaconBlock`s form a chain to the known safe block, @@ -1462,7 +1462,7 @@ Although in this particular use case this does not represent a decay in safety would represent invalid historic data and could be unwittingly transmitted to additional nodes. -### What's the effect of empty slots on the sync algorithm? +#### What's the effect of empty slots on the sync algorithm? When syncing one can only tell that a slot has been skipped on a particular branch by examining subsequent blocks and analyzing the graph formed by the parent root. @@ -1472,9 +1472,9 @@ For example, if a peer responds with blocks [2, 3] when asked for [2, 3, 4], cli -- it merely means that the responding peer did not send it (they may not have it yet or may maliciously be trying to hide it) and successive blocks will be needed to determine if there exists a block at slot 4 in this particular branch. -## Discovery +### Discovery -### Why are we using discv5 and not libp2p Kademlia DHT? +#### Why are we using discv5 and not libp2p Kademlia DHT? discv5 is a standalone protocol, running on UDP on a dedicated port, meant for peer and service discovery only. discv5 supports self-certified, flexible peer records (ENRs) and topic-based advertisement, both of which are, or will be, requirements in this context. @@ -1490,7 +1490,7 @@ It should also help light clients of both networks find nodes with specific capa discv5 is in the process of being audited. -### What is the difference between an ENR and a multiaddr, and why are we using ENRs? +#### What is the difference between an ENR and a multiaddr, and why are we using ENRs? Ethereum Node Records are self-certified node records. Nodes craft and disseminate ENRs for themselves, proving authorship via a cryptographic signature. @@ -1510,7 +1510,7 @@ discv5 uses ENRs and we will presumably need to: 2. Define a bi-directional conversion function between multiaddrs and the corresponding denormalized fields in an ENR (ip, ip6, tcp, tcp6, etc.), for compatibility with nodes that do not support multiaddr natively (e.g. Ethereum execution-layer nodes). -### Why do we not form ENRs and find peers until genesis block/state is known? +#### Why do we not form ENRs and find peers until genesis block/state is known? Although client software might very well be running locally prior to the solidification of the beacon chain genesis state and block, clients cannot form valid ENRs prior to this point. @@ -1521,9 +1521,9 @@ Once genesis data is known, we can then form ENRs and safely find peers. When using a proof-of-work deposit contract for deposits, `fork_digest` will be known `GENESIS_DELAY` (7 days in mainnet configuration) before `genesis_time`, providing ample time to find peers and form initial connections and gossip subnets prior to genesis. -## Compression/Encoding +### Compression/Encoding -### Why are we using SSZ for encoding? +#### Why are we using SSZ for encoding? SSZ is used at the consensus layer, and all implementations should have support for SSZ-encoding/decoding, requiring no further dependencies to be added to client implementations. @@ -1533,7 +1533,7 @@ The actual data in most protocols will be further compressed for efficiency. SSZ has well-defined schemas for consensus objects (typically sent across the wire) reducing any serialization schema data that needs to be sent. It also has defined all required types that are required for this network specification. -### Why are we compressing, and at which layers? +#### Why are we compressing, and at which layers? We compress on the wire to achieve smaller payloads per-message, which, in aggregate, result in higher efficiency, better utilization of available bandwidth, and overall reduction in network-wide traffic overhead. @@ -1563,13 +1563,13 @@ This looks different depending on the interaction layer: implementers are encouraged to encapsulate the encoding and compression logic behind MessageReader and MessageWriter components/strategies that can be layered on top of the raw byte streams. -### Why are we using Snappy for compression? +#### Why are we using Snappy for compression? Snappy is used in Ethereum 1.0. It is well maintained by Google, has good benchmarks, and can calculate the size of the uncompressed object without inflating it in memory. This prevents DOS vectors where large uncompressed data is sent. -### Can I get access to unencrypted bytes on the wire for debugging purposes? +#### Can I get access to unencrypted bytes on the wire for debugging purposes? Yes, you can add loggers in your libp2p protocol handlers to log incoming and outgoing messages. It is recommended to use programming design patterns to encapsulate the logging logic cleanly. @@ -1580,7 +1580,7 @@ you can use logging facilities in those frameworks/runtimes to enable message tr For specific ad-hoc testing scenarios, you can use the [plaintext/2.0.0 secure channel](https://github.com/libp2p/specs/blob/master/plaintext/README.md) (which is essentially no-op encryption or message authentication), in combination with tcpdump or Wireshark to inspect the wire. -### What are SSZ type size bounds? +#### What are SSZ type size bounds? The SSZ encoding outputs of each type have size bounds: each dynamic type, such as a list, has a "limit", which can be used to compute the maximum valid output size. Note that for some more complex dynamic-length objects, element offsets (4 bytes each) may need to be included. @@ -1589,7 +1589,7 @@ Other types are static, they have a fixed size: no dynamic-length content is inv For reference, the type bounds can be computed ahead of time, [as per this example](https://gist.github.com/protolambda/db75c7faa1e94f2464787a480e5d613e). It is advisable to derive these lengths from the SSZ type definitions in use, to ensure that version changes do not cause out-of-sync type bounds. -# libp2p implementations matrix +## libp2p implementations matrix This section will soon contain a matrix showing the maturity/state of the libp2p features required by this spec across the languages in which clients are being developed. From accf99fba3b05e6a7c505e4f5bb07168e0921cbb Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 20 Apr 2023 08:47:49 +0900 Subject: [PATCH 049/117] rename to eip6914 --- specs/_features/{reuse_indices => eip6914}/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename specs/_features/{reuse_indices => eip6914}/beacon-chain.md (93%) diff --git a/specs/_features/reuse_indices/beacon-chain.md b/specs/_features/eip6914/beacon-chain.md similarity index 93% rename from specs/_features/reuse_indices/beacon-chain.md rename to specs/_features/eip6914/beacon-chain.md index 6dd71e36fe..5f75895941 100644 --- a/specs/_features/reuse_indices/beacon-chain.md +++ b/specs/_features/eip6914/beacon-chain.md @@ -1,4 +1,4 @@ -# Reuse indices -- The Beacon Chain +EIP-6914 -- The Beacon Chain ## Table of contents @@ -21,7 +21,7 @@ ## Introduction -This is the beacon chain specification to assign new deposits to existing validator records that have withdrawn long ago. +This is the beacon chain specification to assign new deposits to existing validator records. Refers to [EIP-6914](https://github.com/ethereum/EIPs/pull/6914). *Note:* This specification is built upon [Capella](../../capella/beacon_chain.md) and is under active development. From 498fbd04a23de5a21e4544e203cb7157679c5282 Mon Sep 17 00:00:00 2001 From: dapplion <35266934+dapplion@users.noreply.github.com> Date: Thu, 20 Apr 2023 08:52:25 +0900 Subject: [PATCH 050/117] rename SAFE_EPOCHS_TO_REUSE_INDEX --- specs/_features/eip6914/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/_features/eip6914/beacon-chain.md b/specs/_features/eip6914/beacon-chain.md index 5f75895941..2c60c9bdb4 100644 --- a/specs/_features/eip6914/beacon-chain.md +++ b/specs/_features/eip6914/beacon-chain.md @@ -31,7 +31,7 @@ This is the beacon chain specification to assign new deposits to existing valida | Name | Value | Unit | Duration | | - | - | - | - | -| `REUSE_VALIDATOR_INDEX_DELAY` | `uint64(2**16)` (= 65,536) | epochs | ~0.8 year | +| `SAFE_EPOCHS_TO_REUSE_INDEX` | `uint64(2**16)` (= 65,536) | epochs | ~0.8 year | ## Helper functions @@ -45,7 +45,7 @@ def is_reusable_validator(validator: Validator, balance: Gwei, epoch: Epoch) -> Check if ``validator`` index can be re-assigned to a new deposit. """ return ( - epoch > validator.withdrawable_epoch + REUSE_VALIDATOR_INDEX_DELAY + epoch > validator.withdrawable_epoch + SAFE_EPOCHS_TO_REUSE_INDEX and balance == 0 ) ``` From b7f3d37a528b8b8e3c2ce250eded3855d0b04552 Mon Sep 17 00:00:00 2001 From: Ben Edgington Date: Fri, 21 Apr 2023 09:16:51 +0100 Subject: [PATCH 051/117] Move is_previous_epoch_justified --- specs/phase0/fork-choice.md | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/specs/phase0/fork-choice.md b/specs/phase0/fork-choice.md index 6e281d5c3d..f39a9234c0 100644 --- a/specs/phase0/fork-choice.md +++ b/specs/phase0/fork-choice.md @@ -11,8 +11,8 @@ - [Configuration](#configuration) - [Helpers](#helpers) - [`LatestMessage`](#latestmessage) - - [`is_previous_epoch_justified`](#is_previous_epoch_justified) - [`Store`](#store) + - [`is_previous_epoch_justified`](#is_previous_epoch_justified) - [`get_forkchoice_store`](#get_forkchoice_store) - [`get_slots_since_genesis`](#get_slots_since_genesis) - [`get_current_slot`](#get_current_slot) @@ -92,17 +92,6 @@ class LatestMessage(object): root: Root ``` - -### `is_previous_epoch_justified` - -```python -def is_previous_epoch_justified(store: Store) -> bool: - current_slot = get_current_slot(store) - current_epoch = compute_epoch_at_slot(current_slot) - return store.justified_checkpoint.epoch + 1 == current_epoch -``` - - #### `Store` The `Store` is responsible for tracking information required for the fork choice algorithm. The important fields being tracked are described below: @@ -130,6 +119,15 @@ class Store(object): unrealized_justifications: Dict[Root, Checkpoint] = field(default_factory=dict) ``` +#### `is_previous_epoch_justified` + +```python +def is_previous_epoch_justified(store: Store) -> bool: + current_slot = get_current_slot(store) + current_epoch = compute_epoch_at_slot(current_slot) + return store.justified_checkpoint.epoch + 1 == current_epoch +``` + #### `get_forkchoice_store` The provided anchor-state will be regarded as a trusted state, to not roll back beyond. From 745d529598632029fee9820590b033b7e0e23935 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 25 Apr 2023 12:57:42 +0800 Subject: [PATCH 052/117] Add `compute_subscribed_subnets` unittests and fix typing errors --- specs/phase0/validator.md | 10 +++-- .../validator/test_validator_unittest.py | 39 ++++++++++++++++++- 2 files changed, 45 insertions(+), 4 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 56ca50732c..5266fec7ae 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -616,10 +616,14 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th ```python def compute_subscribed_subnet(node_id: int, epoch: Epoch, index: int) -> int: - node_id_prefix = node_id >> (256 - ATTESTATION_SUBNET_PREFIX_BITS) + node_id_prefix = node_id >> (256 - int(ATTESTATION_SUBNET_PREFIX_BITS)) node_offset = node_id % EPOCHS_PER_SUBNET_SUBSCRIPTION - permutation_seed = hash(uint_to_bytes((epoch + node_offset) // EPOCHS_PER_SUBNET_SUBSCRIPTION)) - permutated_prefix = compute_shuffled_index(node_id_prefix, 1 << ATTESTATION_SUBNET_PREFIX_BITS, permutation_seed) + permutation_seed = hash(uint_to_bytes(uint64((epoch + node_offset) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) + permutated_prefix = compute_shuffled_index( + node_id_prefix, + 1 << int(ATTESTATION_SUBNET_PREFIX_BITS), + permutation_seed, + ) return (permutated_prefix + index) % ATTESTATION_SUBNET_COUNT ``` diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py index cf7ef392f1..177748eacd 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py @@ -1,6 +1,12 @@ +import random + from eth2spec.test.context import ( + single_phase, spec_state_test, - always_bls, with_phases, with_all_phases, + spec_test, + always_bls, + with_phases, + with_all_phases, ) from eth2spec.test.helpers.constants import PHASE0 from eth2spec.test.helpers.attestations import build_attestation_data, get_valid_attestation @@ -476,3 +482,34 @@ def test_get_aggregate_and_proof_signature(spec, state): privkey=privkey, pubkey=pubkey, ) + + +def run_compute_subscribed_subnets_arguments(spec, rng=random.Random(1111)): + node_id = rng.randint(0, 2**40 - 1) # try VALIDATOR_REGISTRY_LIMIT + epoch = rng.randint(0, 2**64 - 1) + subnets = spec.compute_subscribed_subnets(node_id, epoch) + assert len(subnets) == spec.SUBNETS_PER_NODE + + +@with_all_phases +@spec_test +@single_phase +def test_compute_subscribed_subnets_random_1(spec): + rng = random.Random(1111) + run_compute_subscribed_subnets_arguments(spec, rng) + + +@with_all_phases +@spec_test +@single_phase +def test_compute_subscribed_subnets_random_2(spec): + rng = random.Random(2222) + run_compute_subscribed_subnets_arguments(spec, rng) + + +@with_all_phases +@spec_test +@single_phase +def test_compute_subscribed_subnets_random_3(spec): + rng = random.Random(3333) + run_compute_subscribed_subnets_arguments(spec, rng) From 655094ee4a7026185f1962fc245dfa763735828a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 28 Apr 2023 17:28:11 +0800 Subject: [PATCH 053/117] Add `test_zero_blob` test case --- .../eth2spec/test/deneb/sanity/test_blocks.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 5e65dbd4ef..9a6c6a45bc 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -16,13 +16,11 @@ ) -@with_deneb_and_later -@spec_state_test -def test_one_blob(spec, state): +def run_block_with_blobs(spec, state, blob_count): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=blob_count) block.body.blob_kzg_commitments = blob_kzg_commitments block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) @@ -34,15 +32,17 @@ def test_one_blob(spec, state): @with_deneb_and_later @spec_state_test -def test_max_blobs(spec, state): - yield 'pre', state +def test_zero_blob(spec, state): + yield from run_block_with_blobs(spec, state, blob_count=0) - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=spec.MAX_BLOBS_PER_BLOCK) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) - yield 'blocks', [signed_block] - yield 'post', state +@with_deneb_and_later +@spec_state_test +def test_one_blob(spec, state): + yield from run_block_with_blobs(spec, state, blob_count=1) + + +@with_deneb_and_later +@spec_state_test +def test_max_blobs(spec, state): + yield from run_block_with_blobs(spec, state, blob_count=spec.MAX_BLOBS_PER_BLOCK) From 327ef095d0fe8f309dc42126a9b3abf64f31175d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 28 Apr 2023 17:37:50 +0800 Subject: [PATCH 054/117] Add `test_incorrect_blob_tx_type` --- .../eth2spec/test/deneb/sanity/test_blocks.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 9a6c6a45bc..50b975d3bc 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -46,3 +46,20 @@ def test_one_blob(spec, state): @spec_state_test def test_max_blobs(spec, state): yield from run_block_with_blobs(spec, state, blob_count=spec.MAX_BLOBS_PER_BLOCK) + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_blob_tx_type(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + block.body.blob_kzg_commitments = blob_kzg_commitments + opaque_tx[0] == spec.uint8(0x04) # incorrect tx type + block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state From 22da92e5979b72d6b2c37719c84ee66b9d909cc8 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 28 Apr 2023 18:26:42 +0800 Subject: [PATCH 055/117] Refactor the spec and add more test cases --- specs/deneb/beacon-chain.md | 7 +- .../eth2spec/test/deneb/sanity/test_blocks.py | 90 ++++++++++++++++++- 2 files changed, 92 insertions(+), 5 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index df1da8e2a5..359c7fc95b 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -173,6 +173,8 @@ def tx_peek_blob_versioned_hashes(opaque_tx: Transaction) -> Sequence[VersionedH message_offset + uint32.decode_bytes(opaque_tx[(message_offset + 188):(message_offset + 192)]) ) + # `VersionedHash` is a 32-byte object + assert (len(opaque_tx) - blob_versioned_hashes_offset) % 32 == 0 return [ VersionedHash(opaque_tx[x:(x + 32)]) for x in range(blob_versioned_hashes_offset, len(opaque_tx), 32) @@ -205,7 +207,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_eth1_data(state, block.body) process_operations(state, block.body) process_sync_aggregate(state, block.body.sync_aggregate) - process_blob_kzg_commitments(state, block.body) # [New in Deneb] + process_blob_kzg_commitments(block.body) # [New in Deneb] ``` #### Execution payload @@ -248,8 +250,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe #### Blob KZG commitments ```python -def process_blob_kzg_commitments(state: BeaconState, body: BeaconBlockBody) -> None: - # pylint: disable=unused-argument +def process_blob_kzg_commitments(body: BeaconBlockBody) -> None: assert verify_kzg_commitments_against_transactions(body.execution_payload.transactions, body.blob_kzg_commitments) ``` diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 50b975d3bc..0f8f8b3a4d 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -1,3 +1,5 @@ +import random + from eth2spec.test.helpers.state import ( state_transition_and_sign_block ) @@ -50,15 +52,99 @@ def test_max_blobs(spec, state): @with_deneb_and_later @spec_state_test -def test_incorrect_blob_tx_type(spec, state): +def test_invalid_incorrect_blob_tx_type(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + block.body.blob_kzg_commitments = blob_kzg_commitments + opaque_tx = b'\x04' + opaque_tx[1:] # incorrect tx type + block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +@with_deneb_and_later +@spec_state_test +def test_invalid_incorrect_transaction_length_1_byte(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + block.body.blob_kzg_commitments = blob_kzg_commitments + opaque_tx = opaque_tx + b'\x12' # incorrect tx length + block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +@with_deneb_and_later +@spec_state_test +def test_invalid_incorrect_transaction_length_32_bytes(spec, state): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) block.body.blob_kzg_commitments = blob_kzg_commitments - opaque_tx[0] == spec.uint8(0x04) # incorrect tx type + opaque_tx = opaque_tx + b'\x12' * 32 # incorrect tx length block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +@with_deneb_and_later +@spec_state_test +def test_invalid_incorrect_commitment(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + blob_kzg_commitments[0] = b'\x12' * 48 # incorrect commitment + block.body.blob_kzg_commitments = blob_kzg_commitments + block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +@with_deneb_and_later +@spec_state_test +def test_invalid_incorrect_commitments_order(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=2, rng=random.Random(1111)) + block.body.blob_kzg_commitments = [blob_kzg_commitments[1], blob_kzg_commitments[0]] # incorrect order + block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_block_hash(spec, state): + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + block.body.blob_kzg_commitments = blob_kzg_commitments + block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.block_hash = b'\x12' * 32 # incorrect block hash + # CL itself doesn't verify EL block hash signed_block = state_transition_and_sign_block(spec, state, block) yield 'blocks', [signed_block] From 2a1d998594ba795c22ffd657292e2b0b87fe8850 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 28 Apr 2023 19:01:19 +0800 Subject: [PATCH 056/117] Test non zero `excess_data_gas` --- tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py | 1 + tests/core/pyspec/eth2spec/test/helpers/execution_payload.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 0f8f8b3a4d..2b99d6a725 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -25,6 +25,7 @@ def run_block_with_blobs(spec, state, blob_count): opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=blob_count) block.body.blob_kzg_commitments = blob_kzg_commitments block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.excess_data_gas = 2 block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) signed_block = state_transition_and_sign_block(spec, state, block) diff --git a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py index c0a70aca1d..c8ef1cbf00 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/helpers/execution_payload.py @@ -165,6 +165,8 @@ def build_empty_execution_payload(spec, state, randao_mix=None): ) if is_post_capella(spec): payload.withdrawals = spec.get_expected_withdrawals(state) + if is_post_deneb(spec): + payload.excess_data_gas = 0 payload.block_hash = compute_el_block_hash(spec, payload) From e31fcbd6a9f795100ec6f1de434ffd4555a0f0e2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 28 Apr 2023 23:09:10 +0800 Subject: [PATCH 057/117] Add `GetPayloadResponse` for `get_payload` API --- setup.py | 2 +- specs/_features/eip4788/validator.md | 4 +-- specs/bellatrix/validator.md | 19 +++++++++--- specs/capella/validator.md | 14 +++++++-- specs/deneb/validator.md | 46 ++++++++++++++++++++++------ tests/formats/fork_choice/README.md | 4 +-- 6 files changed, 68 insertions(+), 21 deletions(-) diff --git a/setup.py b/setup.py index fc3acb8062..a3e94642e0 100644 --- a/setup.py +++ b/setup.py @@ -588,7 +588,7 @@ def notify_forkchoice_updated(self: ExecutionEngine, payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]: pass - def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> ExecutionPayload: + def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: # pylint: disable=unused-argument raise NotImplementedError("no default block production") diff --git a/specs/_features/eip4788/validator.md b/specs/_features/eip4788/validator.md index 421e297ce9..3140cdb213 100644 --- a/specs/_features/eip4788/validator.md +++ b/specs/_features/eip4788/validator.md @@ -13,7 +13,7 @@ - [Helpers](#helpers) - [Protocols](#protocols) - [`ExecutionEngine`](#executionengine) - - [`get_payload`](#get_payload) + - [Modified `get_payload`](#modified-get_payload) - [Beacon chain responsibilities](#beacon-chain-responsibilities) - [Block proposal](#block-proposal) - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) @@ -40,7 +40,7 @@ Please see related Beacon Chain doc before continuing and use them as a referenc ### `ExecutionEngine` -#### `get_payload` +#### Modified `get_payload` `get_payload` returns the upgraded EIP-4788 `ExecutionPayload` type. diff --git a/specs/bellatrix/validator.md b/specs/bellatrix/validator.md index a176d7534e..dea763cde8 100644 --- a/specs/bellatrix/validator.md +++ b/specs/bellatrix/validator.md @@ -9,6 +9,7 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Helpers](#helpers) + - [`GetPayloadResponse`](#getpayloadresponse) - [`get_pow_block_at_terminal_total_difficulty`](#get_pow_block_at_terminal_total_difficulty) - [`get_terminal_pow_block`](#get_terminal_pow_block) - [Protocols](#protocols) @@ -36,6 +37,14 @@ Please see related Beacon Chain doc before continuing and use them as a referenc ## Helpers +### `GetPayloadResponse` + +```python +@dataclass +class GetPayloadResponse(object): + execution_payload: ExecutionPayload +``` + ### `get_pow_block_at_terminal_total_difficulty` ```python @@ -83,13 +92,13 @@ The Engine API may be used to implement it with an external execution engine. #### `get_payload` -Given the `payload_id`, `get_payload` returns the most recent version of the execution payload that -has been built since the corresponding call to `notify_forkchoice_updated` method. +Given the `payload_id`, `get_payload` returns `GetPayloadResponse` with the most recent version of +the execution payload that has been built since the corresponding call to `notify_forkchoice_updated` method. ```python -def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> ExecutionPayload: +def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: """ - Return ``execution_payload`` object. + Return ``GetPayloadResponse`` object. """ ... ``` @@ -162,7 +171,7 @@ def get_execution_payload(payload_id: Optional[PayloadId], execution_engine: Exe # Pre-merge, empty payload return ExecutionPayload() else: - return execution_engine.get_payload(payload_id) + return execution_engine.get_payload(payload_id).execution_payload ``` *Note*: It is recommended for a validator to call `prepare_execution_payload` as soon as input parameters become known, diff --git a/specs/capella/validator.md b/specs/capella/validator.md index 644ee476f9..29cff8c611 100644 --- a/specs/capella/validator.md +++ b/specs/capella/validator.md @@ -11,9 +11,10 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Helpers](#helpers) + - [Modified `GetPayloadResponse`](#modified-getpayloadresponse) - [Protocols](#protocols) - [`ExecutionEngine`](#executionengine) - - [`get_payload`](#get_payload) + - [Modified `get_payload`](#modified-get_payload) - [Beacon chain responsibilities](#beacon-chain-responsibilities) - [Block proposal](#block-proposal) - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) @@ -39,11 +40,20 @@ Please see related Beacon Chain doc before continuing and use them as a referenc ## Helpers +### Modified `GetPayloadResponse` + +```python +@dataclass +class GetPayloadResponse(object): + execution_payload: ExecutionPayload + block_value: uint256 +``` + ## Protocols ### `ExecutionEngine` -#### `get_payload` +#### Modified `get_payload` `get_payload` returns the upgraded Capella `ExecutionPayload` type. diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index b627de023e..6562c91ddd 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -11,7 +11,11 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) - [Helpers](#helpers) - - [`get_blobs_and_kzg_commitments`](#get_blobs_and_kzg_commitments) + - [`BlobsBundle`](#blobsbundle) + - [Modified `GetPayloadResponse`](#modified-getpayloadresponse) +- [Protocol](#protocol) + - [`ExecutionEngine`](#executionengine) + - [Modified `get_payload`](#modified-get_payload) - [Beacon chain responsibilities](#beacon-chain-responsibilities) - [Block and sidecar proposal](#block-and-sidecar-proposal) - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody) @@ -36,17 +40,40 @@ Please see related Beacon Chain doc before continuing and use them as a referenc ## Helpers -### `get_blobs_and_kzg_commitments` +### `BlobsBundle` -The interface to retrieve blobs and corresponding kzg commitments. +```python +@dataclass +class BlobsBundle(object): + commitments: Sequence[KZGCommitment] + proofs: Sequence[KZGProof] + blobs: Sequence[Blob] +``` + +### Modified `GetPayloadResponse` + +```python +@dataclass +class GetPayloadResponse(object): + execution_payload: ExecutionPayload + block_value: uint256 + blobs_bundle: BlobsBundle +``` + +## Protocol + +### `ExecutionEngine` + +#### Modified `get_payload` -Note: This API is *unstable*. `get_blobs_and_kzg_commitments` and `get_payload` may be unified. -Implementers may also retrieve blobs individually per transaction. +Given the `payload_id`, `get_payload` returns the most recent version of the execution payload that +has been built since the corresponding call to `notify_forkchoice_updated` method. ```python -def get_blobs_and_kzg_commitments( - payload_id: PayloadId -) -> Tuple[Sequence[Blob], Sequence[KZGCommitment], Sequence[KZGProof]]: +def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: + """ + Return ExecutionPayload, uint256, BlobsBundle objects. + """ # pylint: disable=unused-argument ... ``` @@ -62,7 +89,8 @@ All validator responsibilities remain unchanged other than those noted below. ##### Blob KZG commitments 1. After retrieving the execution payload from the execution engine as specified in Capella, -use the `payload_id` to retrieve `blobs` and `blob_kzg_commitments` via `get_blobs_and_kzg_commitments(payload_id)`. +use the `payload_id` to retrieve `blobs`, `blob_kzg_commitments`, and `blob_kzg_proofs` +via `get_payload(payload_id).blobs_bundle`. 2. Validate `blobs` and `blob_kzg_commitments`: ```python diff --git a/tests/formats/fork_choice/README.md b/tests/formats/fork_choice/README.md index c94b959338..3b28837de7 100644 --- a/tests/formats/fork_choice/README.md +++ b/tests/formats/fork_choice/README.md @@ -114,8 +114,8 @@ Optional step for optimistic sync tests. This step sets the [`payloadStatus`](https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md#PayloadStatusV1) value that Execution Layer client mock returns in responses to the following Engine API calls: -* [`engine_newPayloadV1(payload)`](https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md#engine_newpayloadv1) if `payload.blockHash == payload_info.block_hash` -* [`engine_forkchoiceUpdatedV1(forkchoiceState, ...)`](https://github.com/ethereum/execution-apis/blob/main/src/engine/specification.md#engine_forkchoiceupdatedv1) if `forkchoiceState.headBlockHash == payload_info.block_hash` +* [`engine_newPayloadV1(payload)`](https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#engine_newpayloadv1) if `payload.blockHash == payload_info.block_hash` +* [`engine_forkchoiceUpdatedV1(forkchoiceState, ...)`](https://github.com/ethereum/execution-apis/blob/main/src/engine/paris.md#engine_forkchoiceupdatedv1) if `forkchoiceState.headBlockHash == payload_info.block_hash` *Note:* Status of a payload must be *initialized* via `on_payload_info` before the corresponding `on_block` execution step. From 7570445e983d9e63e204be9182c9e8682be58391 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 2 May 2023 23:45:21 +0800 Subject: [PATCH 058/117] Fix sync testgen --- tests/generators/sync/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/generators/sync/main.py b/tests/generators/sync/main.py index 68e38cebe6..5563e6f8c3 100644 --- a/tests/generators/sync/main.py +++ b/tests/generators/sync/main.py @@ -1,5 +1,5 @@ from eth2spec.gen_helpers.gen_from_tests.gen import run_state_test_generators -from eth2spec.test.helpers.constants import BELLATRIX, CAPELLA, DENEB +from eth2spec.test.helpers.constants import BELLATRIX, CAPELLA, DENEB, EIP6110 if __name__ == "__main__": From ddf7e8013bd717fd01faa484f63a24f362063f05 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 2 May 2023 10:57:49 -0500 Subject: [PATCH 059/117] Add group of invalid G1 points --- tests/generators/kzg_4844/main.py | 481 +++++++----------------------- 1 file changed, 114 insertions(+), 367 deletions(-) diff --git a/tests/generators/kzg_4844/main.py b/tests/generators/kzg_4844/main.py index b1391f6787..7ddc3471cd 100644 --- a/tests/generators/kzg_4844/main.py +++ b/tests/generators/kzg_4844/main.py @@ -54,12 +54,17 @@ def evaluate_blob_at(blob, z): ) +BLS_MODULUS_BYTES = spec.BLS_MODULUS.to_bytes(32, spec.ENDIANNESS) + G1 = bls.G1_to_bytes48(bls.G1()) -P1_NOT_IN_G1 = bytes.fromhex("8123456789abcdef0123456789abcdef0123456789abcdef" + +G1_INVALID_TOO_FEW_BYTES = G1[:-1] +G1_INVALID_TOO_MANY_BYTES = G1 + b"\x00" +G1_INVALID_P1_NOT_IN_G1 = bytes.fromhex("8123456789abcdef0123456789abcdef0123456789abcdef" + "0123456789abcdef0123456789abcdef0123456789abcdef") -P1_NOT_ON_CURVE = bytes.fromhex("8123456789abcdef0123456789abcdef0123456789abcdef" + +G1_INVALID_P1_NOT_ON_CURVE = bytes.fromhex("8123456789abcdef0123456789abcdef0123456789abcdef" + "0123456789abcdef0123456789abcdef0123456789abcde0") -BLS_MODULUS_BYTES = spec.BLS_MODULUS.to_bytes(32, spec.ENDIANNESS) +INVALID_G1_POINTS = [G1_INVALID_TOO_FEW_BYTES, G1_INVALID_TOO_MANY_BYTES, + G1_INVALID_P1_NOT_IN_G1, G1_INVALID_P1_NOT_ON_CURVE] BLOB_ALL_ZEROS = spec.Blob() BLOB_RANDOM_VALID1 = spec.Blob(b''.join([field_element_bytes(pow(2, n + 256, spec.BLS_MODULUS)) for n in range(4096)])) @@ -150,7 +155,7 @@ def case02_compute_kzg_proof(): for blob in INVALID_BLOBS: z = VALID_FIELD_ELEMENTS[0] expect_exception(spec.compute_kzg_proof, blob, z) - identifier = f'{encode_hex(hash(blob))}_{encode_hex(z)}' + identifier = f'{encode_hex(hash(blob))}' yield f'compute_kzg_proof_case_invalid_blob_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'blob': encode_hex(blob), @@ -163,7 +168,7 @@ def case02_compute_kzg_proof(): for z in INVALID_FIELD_ELEMENTS: blob = VALID_BLOBS[4] expect_exception(spec.compute_kzg_proof, blob, z) - identifier = f'{encode_hex(hash(blob))}_{encode_hex(z)}' + identifier = f'{encode_hex(hash(z))}' yield f'compute_kzg_proof_case_invalid_z_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'blob': encode_hex(blob), @@ -209,13 +214,29 @@ def case03_verify_kzg_proof(): 'output': False } + # Edge case: Invalid commitment + for commitment in INVALID_G1_POINTS: + blob, z = VALID_BLOBS[2], VALID_FIELD_ELEMENTS[1] + proof, y = spec.compute_kzg_proof(blob, z) + expect_exception(spec.verify_kzg_proof, commitment, z, y, proof) + identifier = f'{encode_hex(commitment)}' + yield f'verify_kzg_proof_case_invalid_commitment_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { + 'input': { + 'commitment': encode_hex(commitment), + 'z': encode_hex(z), + 'y': encode_hex(y), + 'proof': encode_hex(proof), + }, + 'output': None + } + # Edge case: Invalid z for z in INVALID_FIELD_ELEMENTS: blob, validz = VALID_BLOBS[4], VALID_FIELD_ELEMENTS[1] proof, y = spec.compute_kzg_proof(blob, validz) commitment = spec.blob_to_kzg_commitment(blob) expect_exception(spec.verify_kzg_proof, commitment, z, y, proof) - identifier = f'{encode_hex(hash(blob))}_{encode_hex(z)}' + identifier = f'{encode_hex(z)}' yield f'verify_kzg_proof_case_invalid_z_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'commitment': encode_hex(commitment), @@ -232,7 +253,7 @@ def case03_verify_kzg_proof(): proof, _ = spec.compute_kzg_proof(blob, z) commitment = spec.blob_to_kzg_commitment(blob) expect_exception(spec.verify_kzg_proof, commitment, z, y, proof) - identifier = f'{encode_hex(hash(blob))}_{encode_hex(y)}' + identifier = f'{encode_hex(y)}' yield f'verify_kzg_proof_case_invalid_y_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { 'input': { 'commitment': encode_hex(commitment), @@ -243,133 +264,22 @@ def case03_verify_kzg_proof(): 'output': None } - # Edge case: Invalid proof, not in G1 - blob, z = VALID_BLOBS[2], VALID_FIELD_ELEMENTS[0] - proof = P1_NOT_IN_G1 - commitment = spec.blob_to_kzg_commitment(blob) - y = VALID_FIELD_ELEMENTS[1] - expect_exception(spec.verify_kzg_proof, commitment, z, y, proof) - yield 'verify_kzg_proof_case_proof_not_in_G1', { - 'input': { - 'commitment': encode_hex(commitment), - 'z': encode_hex(z), - 'y': encode_hex(y), - 'proof': encode_hex(proof), - }, - 'output': None - } - - # Edge case: Invalid proof, not on curve - blob, z = VALID_BLOBS[3], VALID_FIELD_ELEMENTS[1] - proof = P1_NOT_ON_CURVE - commitment = spec.blob_to_kzg_commitment(blob) - y = VALID_FIELD_ELEMENTS[1] - expect_exception(spec.verify_kzg_proof, commitment, z, y, proof) - yield 'verify_kzg_proof_case_proof_not_on_curve', { - 'input': { - 'commitment': encode_hex(commitment), - 'z': encode_hex(z), - 'y': encode_hex(y), - 'proof': encode_hex(proof), - }, - 'output': None - } - - # Edge case: Invalid proof, too few bytes - blob = VALID_BLOBS[1] - commitment = spec.blob_to_kzg_commitment(blob) - z = VALID_FIELD_ELEMENTS[4] - proof, y = spec.compute_kzg_proof(blob, z) - proof = proof[:-1] - expect_exception(spec.verify_kzg_proof, commitment, z, y, proof) - yield 'verify_kzg_proof_case_proof_too_few_bytes', { - 'input': { - 'commitment': encode_hex(commitment), - 'z': encode_hex(z), - 'y': encode_hex(y), - 'proof': encode_hex(proof), - }, - 'output': None - } - - # Edge case: Invalid proof, too many bytes - blob = VALID_BLOBS[1] - commitment = spec.blob_to_kzg_commitment(blob) - z = VALID_FIELD_ELEMENTS[4] - proof, y = spec.compute_kzg_proof(blob, z) - proof = proof + b"\x00" - expect_exception(spec.verify_kzg_proof, commitment, z, y, proof) - yield 'verify_kzg_proof_case_proof_too_many_bytes', { - 'input': { - 'commitment': encode_hex(commitment), - 'z': encode_hex(z), - 'y': encode_hex(y), - 'proof': encode_hex(proof), - }, - 'output': None - } - - # Edge case: Invalid commitment, not in G1 - blob, z = VALID_BLOBS[4], VALID_FIELD_ELEMENTS[3] - proof, y = spec.compute_kzg_proof(blob, z) - commitment = P1_NOT_IN_G1 - expect_exception(spec.verify_kzg_proof, commitment, z, y, proof) - yield 'verify_kzg_proof_case_commitment_not_in_G1', { - 'input': { - 'commitment': encode_hex(commitment), - 'z': encode_hex(z), - 'y': encode_hex(y), - 'proof': encode_hex(proof), - }, - 'output': None - } - - # Edge case: Invalid commitment, not on curve - blob, z = VALID_BLOBS[1], VALID_FIELD_ELEMENTS[4] - proof, y = spec.compute_kzg_proof(blob, z) - commitment = P1_NOT_ON_CURVE - expect_exception(spec.verify_kzg_proof, commitment, z, y, proof) - yield 'verify_kzg_proof_case_commitment_not_on_curve', { - 'input': { - 'commitment': encode_hex(commitment), - 'z': encode_hex(z), - 'y': encode_hex(y), - 'proof': encode_hex(proof), - }, - 'output': None - } - - # Edge case: Invalid commitment, too few bytes - blob = VALID_BLOBS[1] - commitment = spec.blob_to_kzg_commitment(blob)[:-1] - z = VALID_FIELD_ELEMENTS[4] - proof, y = spec.compute_kzg_proof(blob, z) - expect_exception(spec.verify_kzg_proof, commitment, z, y, proof) - yield 'verify_kzg_proof_case_commitment_too_few_bytes', { - 'input': { - 'commitment': encode_hex(commitment), - 'z': encode_hex(z), - 'y': encode_hex(y), - 'proof': encode_hex(proof), - }, - 'output': None - } - - # Edge case: Invalid commitment, too many bytes - blob = VALID_BLOBS[1] - commitment = spec.blob_to_kzg_commitment(blob) + b"\x00" - z = VALID_FIELD_ELEMENTS[4] - proof, y = spec.compute_kzg_proof(blob, z) - expect_exception(spec.verify_kzg_proof, commitment, z, y, proof) - yield 'verify_kzg_proof_case_commitment_too_many_bytes', { - 'input': { - 'commitment': encode_hex(commitment), - 'z': encode_hex(z), - 'y': encode_hex(y), - 'proof': encode_hex(proof), - }, - 'output': None - } + # Edge case: Invalid proof + for proof in INVALID_G1_POINTS: + blob, z = VALID_BLOBS[2], VALID_FIELD_ELEMENTS[1] + _, y = spec.compute_kzg_proof(blob, z) + commitment = spec.blob_to_kzg_commitment(blob) + expect_exception(spec.verify_kzg_proof, commitment, z, y, proof) + identifier = f'{encode_hex(proof)}' + yield f'verify_kzg_proof_case_invalid_proof_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { + 'input': { + 'commitment': encode_hex(commitment), + 'z': encode_hex(z), + 'y': encode_hex(y), + 'proof': encode_hex(proof), + }, + 'output': None + } def case04_compute_blob_kzg_proof(): @@ -399,31 +309,18 @@ def case04_compute_blob_kzg_proof(): 'output': None } - # Edge case: Invalid commitment, not in G1 - commitment = P1_NOT_IN_G1 - blob = VALID_BLOBS[1] - expect_exception(spec.compute_blob_kzg_proof, blob, commitment) - identifier = f'{encode_hex(hash(blob))}' - yield 'compute_blob_kzg_proof_case_invalid_commitment_not_in_G1', { - 'input': { - 'blob': encode_hex(blob), - 'commitment': encode_hex(commitment), - }, - 'output': None - } - - # Edge case: Invalid commitment, not on curve - commitment = P1_NOT_ON_CURVE - blob = VALID_BLOBS[1] - expect_exception(spec.compute_blob_kzg_proof, blob, commitment) - identifier = f'{encode_hex(hash(blob))}' - yield 'compute_blob_kzg_proof_case_invalid_commitment_not_on_curve', { - 'input': { - 'blob': encode_hex(blob), - 'commitment': encode_hex(commitment), - }, - 'output': None - } + # Edge case: Invalid commitment + for commitment in INVALID_G1_POINTS: + blob = VALID_BLOBS[1] + expect_exception(spec.compute_blob_kzg_proof, blob, commitment) + identifier = f'{encode_hex(hash(commitment))}' + yield f'compute_blob_kzg_proof_case_invalid_commitment_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { + 'input': { + 'blob': encode_hex(blob), + 'commitment': encode_hex(commitment), + }, + 'output': None + } def case05_verify_blob_kzg_proof(): @@ -457,120 +354,6 @@ def case05_verify_blob_kzg_proof(): 'output': False } - # Edge case: Invalid proof, not in G1 - blob = VALID_BLOBS[2] - proof = P1_NOT_IN_G1 - commitment = G1 - expect_exception(spec.verify_blob_kzg_proof, blob, commitment, proof) - yield 'verify_blob_kzg_proof_case_proof_not_in_G1', { - 'input': { - 'blob': encode_hex(blob), - 'commitment': encode_hex(commitment), - 'proof': encode_hex(proof), - }, - 'output': None - } - - # Edge case: Invalid proof, not on curve - blob = VALID_BLOBS[1] - proof = P1_NOT_ON_CURVE - commitment = G1 - expect_exception(spec.verify_blob_kzg_proof, blob, commitment, proof) - yield 'verify_blob_kzg_proof_case_proof_not_on_curve', { - 'input': { - 'blob': encode_hex(blob), - 'commitment': encode_hex(commitment), - 'proof': encode_hex(proof), - }, - 'output': None - } - - # Edge case: Invalid proof, too few bytes - blob = VALID_BLOBS[1] - commitment = spec.blob_to_kzg_commitment(blob) - proof = spec.compute_blob_kzg_proof(blob, commitment)[:-1] - expect_exception(spec.verify_blob_kzg_proof, blob, commitment, proof) - yield 'verify_blob_kzg_proof_case_proof_too_few_bytes', { - 'input': { - 'blob': encode_hex(blob), - 'commitment': encode_hex(commitment), - 'proof': encode_hex(proof), - }, - 'output': None - } - - # Edge case: Invalid proof, too many bytes - blob = VALID_BLOBS[1] - commitment = spec.blob_to_kzg_commitment(blob) - proof = spec.compute_blob_kzg_proof(blob, commitment) + b"\x00" - expect_exception(spec.verify_blob_kzg_proof, blob, commitment, proof) - yield 'verify_blob_kzg_proof_case_proof_too_many_bytes', { - 'input': { - 'blob': encode_hex(blob), - 'commitment': encode_hex(commitment), - 'proof': encode_hex(proof), - }, - 'output': None - } - - # Edge case: Invalid commitment, not in G1 - blob = VALID_BLOBS[0] - proof = G1 - commitment = P1_NOT_IN_G1 - expect_exception(spec.verify_blob_kzg_proof, blob, commitment, proof) - yield 'verify_blob_kzg_proof_case_commitment_not_in_G1', { - 'input': { - 'blob': encode_hex(blob), - 'commitment': encode_hex(commitment), - 'proof': encode_hex(proof), - }, - 'output': None - } - - # Edge case: Invalid commitment, not on curve - blob = VALID_BLOBS[2] - proof = G1 - commitment = P1_NOT_ON_CURVE - expect_exception(spec.verify_blob_kzg_proof, blob, commitment, proof) - yield 'verify_blob_kzg_proof_case_commitment_not_on_curve', { - 'input': { - 'blob': encode_hex(blob), - 'commitment': encode_hex(commitment), - 'proof': encode_hex(proof), - }, - 'output': None - } - - # Edge case: Invalid commitment, too few bytes - blob = VALID_BLOBS[1] - commitment = spec.blob_to_kzg_commitment(blob) - proof = spec.compute_blob_kzg_proof(blob, commitment) - commitment = commitment[:-1] - expect_exception(spec.verify_blob_kzg_proof, blob, commitment, proof) - yield 'verify_blob_kzg_proof_case_commitment_too_few_bytes', { - 'input': { - 'blob': encode_hex(blob), - 'commitment': encode_hex(commitment), - 'proof': encode_hex(proof), - }, - 'output': None - } - - # Edge case: Invalid commitment, too many bytes - blob = VALID_BLOBS[1] - commitment = spec.blob_to_kzg_commitment(blob) - proof = spec.compute_blob_kzg_proof(blob, commitment) - commitment = commitment + b"\x00" - expect_exception(spec.verify_blob_kzg_proof, blob, commitment, proof) - yield 'verify_blob_kzg_proof_case_commitment_too_many_bytes', { - 'input': { - 'blob': encode_hex(blob), - 'commitment': encode_hex(commitment), - 'proof': encode_hex(proof), - }, - 'output': None - } - # Edge case: Invalid blob for blob in INVALID_BLOBS: proof = G1 @@ -586,6 +369,36 @@ def case05_verify_blob_kzg_proof(): 'output': None } + # Edge case: Invalid commitment + for commitment in INVALID_G1_POINTS: + blob = VALID_BLOBS[1] + proof = G1 + expect_exception(spec.verify_blob_kzg_proof, blob, commitment, proof) + identifier = f'{encode_hex(hash(commitment))}' + yield f'verify_blob_kzg_proof_case_invalid_commitment_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { + 'input': { + 'blob': encode_hex(blob), + 'commitment': encode_hex(commitment), + 'proof': encode_hex(proof), + }, + 'output': None + } + + # Edge case: Invalid proof + for proof in INVALID_G1_POINTS: + blob = VALID_BLOBS[1] + commitment = G1 + expect_exception(spec.verify_blob_kzg_proof, blob, commitment, proof) + identifier = f'{encode_hex(hash(proof))}' + yield f'verify_blob_kzg_proof_case_invalid_proof_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { + 'input': { + 'blob': encode_hex(blob), + 'commitment': encode_hex(commitment), + 'proof': encode_hex(proof), + }, + 'output': None + } + def case06_verify_blob_kzg_proof_batch(): # Valid cases @@ -633,101 +446,35 @@ def case06_verify_blob_kzg_proof_batch(): 'output': None } - # Edge case: Invalid proof, not in G1 - proofs_invalid_notG1 = [P1_NOT_IN_G1] + proofs[1:] - expect_exception(spec.verify_blob_kzg_proof_batch, VALID_BLOBS, commitments, proofs_invalid_notG1) - yield 'verify_blob_kzg_proof_batch_case_proof_not_in_G1', { - 'input': { - 'blobs': encode_hex_list(VALID_BLOBS), - 'commitments': encode_hex_list(commitments), - 'proofs': encode_hex_list(proofs_invalid_notG1), - }, - 'output': None - } - - # Edge case: Invalid proof, not on curve - proofs_invalid_notCurve = proofs[:1] + [P1_NOT_ON_CURVE] + proofs[2:] - expect_exception(spec.verify_blob_kzg_proof_batch, VALID_BLOBS, commitments, proofs_invalid_notCurve) - yield 'verify_blob_kzg_proof_batch_case_proof_not_on_curve', { - 'input': { - 'blobs': encode_hex_list(VALID_BLOBS), - 'commitments': encode_hex_list(commitments), - 'proofs': encode_hex_list(proofs_invalid_notCurve), - }, - 'output': None - } - - # Edge case: Invalid proof, too few bytes - proofs_invalid_tooFewBytes = proofs[:1] + [proofs[1][:-1]] + proofs[2:] - expect_exception(spec.verify_blob_kzg_proof_batch, VALID_BLOBS, commitments, proofs_invalid_tooFewBytes) - yield 'verify_blob_kzg_proof_batch_case_proof_too_few_bytes', { - 'input': { - 'blobs': encode_hex_list(VALID_BLOBS), - 'commitments': encode_hex_list(commitments), - 'proofs': encode_hex_list(proofs_invalid_tooFewBytes), - }, - 'output': None - } - - # Edge case: Invalid proof, too many bytes - proofs_invalid_tooManyBytes = proofs[:1] + [proofs[1] + b"\x00"] + proofs[2:] - expect_exception(spec.verify_blob_kzg_proof_batch, VALID_BLOBS, commitments, proofs_invalid_tooManyBytes) - yield 'verify_blob_kzg_proof_batch_case_proof_too_many_bytes', { - 'input': { - 'blobs': encode_hex_list(VALID_BLOBS), - 'commitments': encode_hex_list(commitments), - 'proofs': encode_hex_list(proofs_invalid_tooManyBytes), - }, - 'output': None - } - - # Edge case: Invalid commitment, not in G1 - commitments_invalid_notG1 = commitments[:2] + [P1_NOT_IN_G1] + commitments[3:] - expect_exception(spec.verify_blob_kzg_proof_batch, VALID_BLOBS, commitments, commitments_invalid_notG1) - yield 'verify_blob_kzg_proof_batch_case_commitment_not_in_G1', { - 'input': { - 'blobs': encode_hex_list(VALID_BLOBS), - 'commitments': encode_hex_list(commitments_invalid_notG1), - 'proofs': encode_hex_list(proofs), - }, - 'output': None - } - - # Edge case: Invalid commitment, not on curve - commitments_invalid_notCurve = commitments[:3] + [P1_NOT_ON_CURVE] + commitments[4:] - expect_exception(spec.verify_blob_kzg_proof_batch, VALID_BLOBS, commitments, commitments_invalid_notCurve) - yield 'verify_blob_kzg_proof_batch_case_not_on_curve', { - 'input': { - 'blobs': encode_hex_list(VALID_BLOBS), - 'commitments': encode_hex_list(commitments_invalid_notCurve), - 'proofs': encode_hex_list(proofs), - }, - 'output': None - } - - # Edge case: Invalid commitment, too few bytes - commitments_invalid_tooFewBytes = commitments[:3] + [commitments[3][:-1]] + commitments[4:] - expect_exception(spec.verify_blob_kzg_proof_batch, VALID_BLOBS, commitments, commitments_invalid_tooFewBytes) - yield 'verify_blob_kzg_proof_batch_case_commitment_too_few_bytes', { - 'input': { - 'blobs': encode_hex_list(VALID_BLOBS), - 'commitments': encode_hex_list(commitments_invalid_tooFewBytes), - 'proofs': encode_hex_list(proofs), - }, - 'output': None - } + # Edge case: Invalid commitment + for commitment in INVALID_G1_POINTS: + blobs = VALID_BLOBS + commitments_invalid = [commitment] + commitments[1:] + expect_exception(spec.verify_blob_kzg_proof_batch, blobs, commitments_invalid, proofs) + identifier = f'{encode_hex(hash(commitment))}' + yield f'verify_blob_kzg_proof_batch_case_invalid_commitment_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { + 'input': { + 'blobs': encode_hex_list(blobs), + 'commitments': encode_hex_list(commitments_invalid), + 'proofs': encode_hex_list(proofs), + }, + 'output': None + } - # Edge case: Invalid commitment, too many bytes - commitments_invalid_tooManyBytes = commitments[:3] + [commitments[3] + b"\x00"] + commitments[4:] - expect_exception(spec.verify_blob_kzg_proof_batch, VALID_BLOBS, commitments, commitments_invalid_tooManyBytes) - yield 'verify_blob_kzg_proof_batch_case_commitment_too_many_bytes', { - 'input': { - 'blobs': encode_hex_list(VALID_BLOBS), - 'commitments': encode_hex_list(commitments_invalid_tooManyBytes), - 'proofs': encode_hex_list(proofs), - }, - 'output': None - } + # Edge case: Invalid proof + for proof in INVALID_G1_POINTS: + blobs = VALID_BLOBS + proofs_invalid = [proof] + proofs[1:] + expect_exception(spec.verify_blob_kzg_proof_batch, blobs, commitments, proofs_invalid) + identifier = f'{encode_hex(hash(proof))}' + yield f'verify_blob_kzg_proof_batch_case_invalid_proof_{(hash(bytes(identifier, "utf-8"))[:8]).hex()}', { + 'input': { + 'blobs': encode_hex_list(blobs), + 'commitments': encode_hex_list(commitments), + 'proofs': encode_hex_list(proofs_invalid), + }, + 'output': None + } # Edge case: Blob length different expect_exception(spec.verify_blob_kzg_proof_batch, VALID_BLOBS[:-1], commitments, proofs) From dfdbe15e42ece956e6525b4cb4c26227083227db Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Tue, 2 May 2023 13:05:41 -0500 Subject: [PATCH 060/117] Fix linter issues --- tests/generators/kzg_4844/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/generators/kzg_4844/main.py b/tests/generators/kzg_4844/main.py index 7ddc3471cd..2f3efeb21a 100644 --- a/tests/generators/kzg_4844/main.py +++ b/tests/generators/kzg_4844/main.py @@ -60,9 +60,9 @@ def evaluate_blob_at(blob, z): G1_INVALID_TOO_FEW_BYTES = G1[:-1] G1_INVALID_TOO_MANY_BYTES = G1 + b"\x00" G1_INVALID_P1_NOT_IN_G1 = bytes.fromhex("8123456789abcdef0123456789abcdef0123456789abcdef" + - "0123456789abcdef0123456789abcdef0123456789abcdef") + "0123456789abcdef0123456789abcdef0123456789abcdef") G1_INVALID_P1_NOT_ON_CURVE = bytes.fromhex("8123456789abcdef0123456789abcdef0123456789abcdef" + - "0123456789abcdef0123456789abcdef0123456789abcde0") + "0123456789abcdef0123456789abcdef0123456789abcde0") INVALID_G1_POINTS = [G1_INVALID_TOO_FEW_BYTES, G1_INVALID_TOO_MANY_BYTES, G1_INVALID_P1_NOT_IN_G1, G1_INVALID_P1_NOT_ON_CURVE] From 057517526e7f24a11483e5028128614ad055f822 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 3 May 2023 17:25:04 +0800 Subject: [PATCH 061/117] Set python_requires=">=3.9" (#2964) --- .circleci/config.yml | 22 +++++++++++----------- setup.py | 2 +- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 1d5b098111..5958a2fc69 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -60,7 +60,7 @@ commands: jobs: checkout_specs: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.9 working_directory: ~/specs-repo steps: # Restore git repo at point close to target branch/revision, to speed up checkout @@ -80,7 +80,7 @@ jobs: - ~/specs-repo install_pyspec_test: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.9 working_directory: ~/specs-repo steps: - restore_cache: @@ -92,7 +92,7 @@ jobs: - save_pyspec_cached_venv test-phase0: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.9 working_directory: ~/specs-repo steps: - restore_cache: @@ -105,7 +105,7 @@ jobs: path: tests/core/pyspec/test-reports test-altair: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.9 working_directory: ~/specs-repo steps: - restore_cache: @@ -118,7 +118,7 @@ jobs: path: tests/core/pyspec/test-reports test-bellatrix: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.9 working_directory: ~/specs-repo steps: - restore_cache: @@ -131,7 +131,7 @@ jobs: path: tests/core/pyspec/test-reports test-capella: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.9 working_directory: ~/specs-repo steps: - restore_cache: @@ -144,7 +144,7 @@ jobs: path: tests/core/pyspec/test-reports test-deneb: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.9 working_directory: ~/specs-repo steps: - restore_cache: @@ -179,7 +179,7 @@ jobs: command: sudo npm install -g doctoc@2 && make check_toc codespell: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.9 working_directory: ~/specs-repo steps: - checkout @@ -188,7 +188,7 @@ jobs: command: pip install 'codespell<3.0.0,>=2.0.0' --user && make codespell lint: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.9 working_directory: ~/specs-repo steps: - restore_cache: @@ -244,7 +244,7 @@ jobs: - /nix install_deposit_contract_web3_tester: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.9 working_directory: ~/specs-repo steps: - restore_cache: @@ -256,7 +256,7 @@ jobs: - save_deposit_contract_tester_cached_venv test_deposit_contract_web3_tests: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.9 working_directory: ~/specs-repo steps: - restore_cache: diff --git a/setup.py b/setup.py index a7fe7d9e02..5d27369794 100644 --- a/setup.py +++ b/setup.py @@ -1180,7 +1180,7 @@ def run(self): packages=find_packages(where='tests/core/pyspec') + ['configs', 'specs'], py_modules=["eth2spec"], cmdclass=commands, - python_requires=">=3.8, <4", + python_requires=">=3.9, <4", extras_require={ "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"], "lint": ["flake8==5.0.4", "mypy==0.981", "pylint==2.15.3"], From 4244dc8e139f299b0b0a2cdaf1f045b69a9b0917 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 May 2023 17:55:14 +0800 Subject: [PATCH 062/117] Make `excess_data_gas` a param of `run_block_with_blobs` --- tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 2b99d6a725..111565cce2 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -18,14 +18,14 @@ ) -def run_block_with_blobs(spec, state, blob_count): +def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=blob_count) block.body.blob_kzg_commitments = blob_kzg_commitments block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.excess_data_gas = 2 + block.body.execution_payload.excess_data_gas = excess_data_gas block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) signed_block = state_transition_and_sign_block(spec, state, block) From 79b8a9abecded921179d2b2854d8dc7b8c570d5d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 May 2023 18:09:01 +0800 Subject: [PATCH 063/117] Apply suggestions from code review --- specs/phase0/validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 5266fec7ae..92eadde5f1 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -612,7 +612,7 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th * Remain subscribed to `SUBNETS_PER_NODE` for `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs. * Maintain advertisement of the selected subnets in their node's ENR `attnets` entry by setting the selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets. -* Select these subnets based on their node-id as specified by the following `compute_subscribed_subnets(node_id,epoch)` function. +* Select these subnets based on their node-id as specified by the following `compute_subscribed_subnets(node_id, epoch)` function. ```python def compute_subscribed_subnet(node_id: int, epoch: Epoch, index: int) -> int: @@ -629,7 +629,7 @@ def compute_subscribed_subnet(node_id: int, epoch: Epoch, index: int) -> int: ```python def compute_subscribed_subnets(node_id: int, epoch: Epoch) -> Sequence[int]: - return [compute_subscribed_subnet(node_id, epoch, idx) for idx in range(SUBNETS_PER_NODE)] + return [compute_subscribed_subnet(node_id, epoch, index) for index in range(SUBNETS_PER_NODE)] ``` *Note*: When preparing for a hard fork, a validator must select and subscribe to subnets of the future fork versioning at least `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. From 5cb2733ed5271c582fb2235367558ff8950dd7a2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 4 May 2023 18:50:13 +0800 Subject: [PATCH 064/117] Add custom types `NodeID` and `SubnetID` and constant `NODE_ID_BITS` --- setup.py | 4 +-- specs/phase0/validator.md | 25 ++++++++++++++----- .../unittests/test_config_invariants.py | 2 ++ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index 52bad2b71b..f1130eb586 100644 --- a/setup.py +++ b/setup.py @@ -383,7 +383,7 @@ def imports(cls, preset_name: str) -> str: from eth2spec.utils.ssz.ssz_impl import hash_tree_root, copy, uint_to_bytes from eth2spec.utils.ssz.ssz_typing import ( - View, boolean, Container, List, Vector, uint8, uint32, uint64, + View, boolean, Container, List, Vector, uint8, uint32, uint64, uint256, Bytes1, Bytes4, Bytes32, Bytes48, Bytes96, Bitlist) from eth2spec.utils.ssz.ssz_typing import Bitvector # noqa: F401 from eth2spec.utils import bls @@ -551,7 +551,7 @@ def imports(cls, preset_name: str): return super().imports(preset_name) + f''' from typing import Protocol from eth2spec.altair import {preset_name} as altair -from eth2spec.utils.ssz.ssz_typing import Bytes8, Bytes20, ByteList, ByteVector, uint256 +from eth2spec.utils.ssz.ssz_typing import Bytes8, Bytes20, ByteList, ByteVector ''' @classmethod diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index 92eadde5f1..604350ed80 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -10,6 +10,7 @@ This is an accompanying document to [Phase 0 -- The Beacon Chain](./beacon-chain - [Introduction](#introduction) - [Prerequisites](#prerequisites) +- [Custom types](#custom-types) - [Constants](#constants) - [Misc](#misc) - [Containers](#containers) @@ -82,6 +83,15 @@ A validator is an entity that participates in the consensus of the Ethereum proo All terminology, constants, functions, and protocol mechanics defined in the [Phase 0 -- The Beacon Chain](./beacon-chain.md) and [Phase 0 -- Deposit Contract](./deposit-contract.md) doc are requisite for this document and used throughout. Please see the Phase 0 doc before continuing and use as a reference throughout. +## Custom types + +We define the following Python custom types for type hinting and readability: + +| Name | SSZ equivalent | Description | +| - | - | - | +| `NodeID` | `uint256` | node identifier | +| `SubnetID` | `uint64` | subnet identifier | + ## Constants ### Misc @@ -94,6 +104,7 @@ All terminology, constants, functions, and protocol mechanics defined in the [Ph | `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | | `SUBNETS_PER_NODE` | `2` | The number of long-lived subnets a beacon node should be subscribed to. | | `ATTESTATION_SUBNET_PREFIX_BITS` | `(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | | +| `NODE_ID_BITS` | `256` | The bit length of uint256 is 256 | ## Containers @@ -515,7 +526,9 @@ The `subnet_id` for the `attestation` is calculated with: - Let `subnet_id = compute_subnet_for_attestation(committees_per_slot, attestation.data.slot, attestation.data.index)`. ```python -def compute_subnet_for_attestation(committees_per_slot: uint64, slot: Slot, committee_index: CommitteeIndex) -> uint64: +def compute_subnet_for_attestation(committees_per_slot: uint64, + slot: Slot, + committee_index: CommitteeIndex) -> SubnetID: """ Compute the correct subnet for an attestation for Phase 0. Note, this mimics expected future behavior where attestations will be mapped to their shard subnet. @@ -523,7 +536,7 @@ def compute_subnet_for_attestation(committees_per_slot: uint64, slot: Slot, comm slots_since_epoch_start = uint64(slot % SLOTS_PER_EPOCH) committees_since_epoch_start = committees_per_slot * slots_since_epoch_start - return uint64((committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT) + return SubnetID((committees_since_epoch_start + committee_index) % ATTESTATION_SUBNET_COUNT) ``` ### Attestation aggregation @@ -615,8 +628,8 @@ Because Phase 0 does not have shards and thus does not have Shard Committees, th * Select these subnets based on their node-id as specified by the following `compute_subscribed_subnets(node_id, epoch)` function. ```python -def compute_subscribed_subnet(node_id: int, epoch: Epoch, index: int) -> int: - node_id_prefix = node_id >> (256 - int(ATTESTATION_SUBNET_PREFIX_BITS)) +def compute_subscribed_subnet(node_id: NodeID, epoch: Epoch, index: int) -> SubnetID: + node_id_prefix = node_id >> (NODE_ID_BITS - int(ATTESTATION_SUBNET_PREFIX_BITS)) node_offset = node_id % EPOCHS_PER_SUBNET_SUBSCRIPTION permutation_seed = hash(uint_to_bytes(uint64((epoch + node_offset) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) permutated_prefix = compute_shuffled_index( @@ -624,11 +637,11 @@ def compute_subscribed_subnet(node_id: int, epoch: Epoch, index: int) -> int: 1 << int(ATTESTATION_SUBNET_PREFIX_BITS), permutation_seed, ) - return (permutated_prefix + index) % ATTESTATION_SUBNET_COUNT + return SubnetID((permutated_prefix + index) % ATTESTATION_SUBNET_COUNT) ``` ```python -def compute_subscribed_subnets(node_id: int, epoch: Epoch) -> Sequence[int]: +def compute_subscribed_subnets(node_id: NodeID, epoch: Epoch) -> Sequence[SubnetID]: return [compute_subscribed_subnet(node_id, epoch, index) for index in range(SUBNETS_PER_NODE)] ``` diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py index 69aa3eb2a5..b0fd06374d 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py @@ -76,6 +76,8 @@ def test_time(spec, state): @spec_state_test def test_networking(spec, state): assert spec.SUBNETS_PER_NODE <= spec.ATTESTATION_SUBNET_COUNT + node_id_length = spec.NodeID(1).type_byte_length() # in bytes + assert node_id_length * 8 == spec.NODE_ID_BITS # in bits @with_all_phases From 9f5bb03cb4fea394a0d64963c3e3b820a5da5813 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 10 Nov 2022 16:33:50 -0500 Subject: [PATCH 065/117] Refactor `run_generator` --- .../gen_helpers/gen_base/gen_runner.py | 298 +++++++++++------- 1 file changed, 176 insertions(+), 122 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 0a831a5921..5e6ea93d34 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -1,4 +1,7 @@ -from eth_utils import encode_hex +from dataclasses import ( + dataclass, + field, +) import os import time import shutil @@ -15,6 +18,8 @@ from filelock import FileLock from snappy import compress +from eth_utils import encode_hex + from eth2spec.test import context from eth2spec.test.exceptions import SkippedTest @@ -28,6 +33,14 @@ TIME_THRESHOLD_TO_PRINT = 1.0 # seconds +@dataclass +class Diagnostics(object): + collected_test_count: int = 0 + generated_test_count: int = 0 + skipped_test_count: int = 0 + test_identifiers: list = field(default_factory=list) + + def validate_output_dir(path_str): path = Path(path_str) @@ -40,6 +53,47 @@ def validate_output_dir(path_str): return path +def get_test_case_dir(test_case, output_dir): + return ( + Path(output_dir) / Path(test_case.preset_name) / Path(test_case.fork_name) + / Path(test_case.runner_name) / Path(test_case.handler_name) + / Path(test_case.suite_name) / Path(test_case.case_name) + ) + + +def get_test_identifier(test_case): + return "::".join([ + test_case.preset_name, + test_case.fork_name, + test_case.runner_name, + test_case.handler_name, + test_case.suite_name, + test_case.case_name + ]) + + +def get_incomplete_tag_file(case_dir): + return case_dir / "INCOMPLETE" + + +def should_skip_case_dir(case_dir, is_force, diagnostics_obj): + is_skip = False + incomplete_tag_file = get_incomplete_tag_file(case_dir) + + if case_dir.exists(): + if not is_force and not incomplete_tag_file.exists(): + diagnostics_obj.skipped_test_count += 1 + print(f'Skipping already existing test: {case_dir}') + is_skip = True + else: + print(f'Warning, output directory {case_dir} already exist,' + ' old files will be deleted and it will generate test vector files with the latest version') + # Clear the existing case_dir folder + shutil.rmtree(case_dir) + + return is_skip, diagnostics_obj + + def run_generator(generator_name, test_providers: Iterable[TestProvider]): """ Implementation for a general test generator. @@ -129,10 +183,8 @@ def cfg_represent_quoted_str(self, data): print(f"Filtering test-generator runs to only include presets: {', '.join(presets)}") collect_only = args.collect_only - collected_test_count = 0 - generated_test_count = 0 - skipped_test_count = 0 - test_identifiers = [] + + diagnostics_obj = Diagnostics() provider_start = time.time() for tprov in test_providers: @@ -145,146 +197,114 @@ def cfg_represent_quoted_str(self, data): if len(presets) != 0 and test_case.preset_name not in presets: continue - case_dir = ( - Path(output_dir) / Path(test_case.preset_name) / Path(test_case.fork_name) - / Path(test_case.runner_name) / Path(test_case.handler_name) - / Path(test_case.suite_name) / Path(test_case.case_name) - ) - collected_test_count += 1 + case_dir = get_test_case_dir(test_case, output_dir) print(f"Collected test at: {case_dir}") + diagnostics_obj.collected_test_count += 1 - incomplete_tag_file = case_dir / "INCOMPLETE" - - if case_dir.exists(): - if not args.force and not incomplete_tag_file.exists(): - skipped_test_count += 1 - print(f'Skipping already existing test: {case_dir}') - continue - else: - print(f'Warning, output directory {case_dir} already exist,' - f' old files will be deleted and it will generate test vector files with the latest version') - # Clear the existing case_dir folder - shutil.rmtree(case_dir) - - print(f'Generating test: {case_dir}') - test_start = time.time() - - written_part = False - - # Add `INCOMPLETE` tag file to indicate that the test generation has not completed. - case_dir.mkdir(parents=True, exist_ok=True) - with incomplete_tag_file.open("w") as f: - f.write("\n") - - try: - def output_part(out_kind: str, name: str, fn: Callable[[Path, ], None]): - # make sure the test case directory is created before any test part is written. - case_dir.mkdir(parents=True, exist_ok=True) - try: - fn(case_dir) - except IOError as e: - error_message = ( - f'[Error] error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}' - ) - # Write to error log file - with log_file.open("a+") as f: - f.write(error_message) - traceback.print_exc(file=f) - f.write('\n') - - sys.exit(error_message) - - meta = dict() - - try: - for (name, out_kind, data) in test_case.case_fn(): - written_part = True - if out_kind == "meta": - meta[name] = data - elif out_kind == "cfg": - output_part(out_kind, name, dump_yaml_fn(data, name, file_mode, cfg_yaml)) - elif out_kind == "data": - output_part(out_kind, name, dump_yaml_fn(data, name, file_mode, yaml)) - elif out_kind == "ssz": - output_part(out_kind, name, dump_ssz_fn(data, name, file_mode)) - else: - assert False # Unknown kind - except SkippedTest as e: - print(e) - skipped_test_count += 1 - shutil.rmtree(case_dir) - continue - - # Once all meta data is collected (if any), write it to a meta data file. - if len(meta) != 0: - written_part = True - output_part("data", "meta", dump_yaml_fn(meta, "meta", file_mode, yaml)) - - if not written_part: - print(f"test case {case_dir} did not produce any test case parts") - except Exception as e: - error_message = f"[ERROR] failed to generate vector(s) for test {case_dir}: {e}" - # Write to error log file - with log_file.open("a+") as f: - f.write(error_message) - traceback.print_exc(file=f) - f.write('\n') - traceback.print_exc() - else: - # If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder. - if not written_part: - shutil.rmtree(case_dir) - else: - generated_test_count += 1 - test_identifier = "::".join([ - test_case.preset_name, - test_case.fork_name, - test_case.runner_name, - test_case.handler_name, - test_case.suite_name, - test_case.case_name - ]) - test_identifiers.append(test_identifier) - # Only remove `INCOMPLETE` tag file - os.remove(incomplete_tag_file) - test_end = time.time() - span = round(test_end - test_start, 2) - if span > TIME_THRESHOLD_TO_PRINT: - print(f' - generated in {span} seconds') + is_skip, diagnostics_obj = should_skip_case_dir(case_dir, args.force, diagnostics_obj) + if is_skip: + continue + + # generate test vector + is_skip, diagnostics_obj = generate_test_vector_and_diagnose( + test_case, case_dir, log_file, file_mode, + cfg_yaml, yaml, diagnostics_obj, + ) + if is_skip: + continue provider_end = time.time() span = round(provider_end - provider_start, 2) if collect_only: - print(f"Collected {collected_test_count} tests in total") + print(f"Collected {diagnostics_obj.collected_test_count} tests in total") else: - summary_message = f"completed generation of {generator_name} with {generated_test_count} tests" - summary_message += f" ({skipped_test_count} skipped tests)" + summary_message = f"completed generation of {generator_name} with {diagnostics_obj.generated_test_count} tests" + summary_message += f" ({diagnostics_obj.skipped_test_count} skipped tests)" if span > TIME_THRESHOLD_TO_PRINT: summary_message += f" in {span} seconds" print(summary_message) - diagnostics = { - "collected_test_count": collected_test_count, - "generated_test_count": generated_test_count, - "skipped_test_count": skipped_test_count, - "test_identifiers": test_identifiers, + + diagnostics_output = { + "collected_test_count": diagnostics_obj.collected_test_count, + "generated_test_count": diagnostics_obj.generated_test_count, + "skipped_test_count": diagnostics_obj.skipped_test_count, + "test_identifiers": diagnostics_obj.test_identifiers, "durations": [f"{span} seconds"], } - diagnostics_path = Path(os.path.join(output_dir, "diagnostics.json")) - diagnostics_lock = FileLock(os.path.join(output_dir, "diagnostics.json.lock")) + diagnostics_path = Path(os.path.join(output_dir, "diagnostics_obj.json")) + diagnostics_lock = FileLock(os.path.join(output_dir, "diagnostics_obj.json.lock")) with diagnostics_lock: diagnostics_path.touch(exist_ok=True) if os.path.getsize(diagnostics_path) == 0: with open(diagnostics_path, "w+") as f: - json.dump(diagnostics, f) + json.dump(diagnostics_output, f) else: with open(diagnostics_path, "r+") as f: existing_diagnostics = json.load(f) - for k, v in diagnostics.items(): + for k, v in diagnostics_output.items(): existing_diagnostics[k] += v with open(diagnostics_path, "w+") as f: json.dump(existing_diagnostics, f) - print(f"wrote diagnostics to {diagnostics_path}") + print(f"wrote diagnostics_obj to {diagnostics_path}") + + +def generate_test_vector_and_diagnose(test_case, case_dir, log_file, file_mode, cfg_yaml, yaml, diagnostics_obj): + is_skip = False + + print(f'Generating test: {case_dir}') + test_start = time.time() + + written_part = False + + # Add `INCOMPLETE` tag file to indicate that the test generation has not completed. + incomplete_tag_file = get_incomplete_tag_file(case_dir) + case_dir.mkdir(parents=True, exist_ok=True) + with incomplete_tag_file.open("w") as f: + f.write("\n") + + try: + meta = dict() + try: + written_part, meta = execute_test(test_case, case_dir, meta, log_file, file_mode, cfg_yaml, yaml) + except SkippedTest as e: + print(e) + diagnostics_obj.skipped_test_count += 1 + shutil.rmtree(case_dir) + is_skip = True + return is_skip, diagnostics_obj + + # Once all meta data is collected (if any), write it to a meta data file. + if len(meta) != 0: + written_part = True + output_part(case_dir, log_file, "data", "meta", dump_yaml_fn(meta, "meta", file_mode, yaml)) + + if not written_part: + print(f"test case {case_dir} did not produce any test case parts") + except Exception as e: + error_message = f"[ERROR] failed to generate vector(s) for test {case_dir}: {e}" + # Write to error log file + with log_file.open("a+") as f: + f.write(error_message) + traceback.print_exc(file=f) + f.write('\n') + traceback.print_exc() + else: + # If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder. + if not written_part: + shutil.rmtree(case_dir) + else: + diagnostics_obj.generated_test_count += 1 + test_identifier = get_test_identifier(test_case) + diagnostics_obj.test_identifiers.append(test_identifier) + # Only remove `INCOMPLETE` tag file + os.remove(incomplete_tag_file) + test_end = time.time() + span = round(test_end - test_start, 2) + if span > TIME_THRESHOLD_TO_PRINT: + print(f' - generated in {span} seconds') + + return is_skip, diagnostics_obj def dump_yaml_fn(data: Any, name: str, file_mode: str, yaml_encoder: YAML): @@ -295,6 +315,40 @@ def dump(case_path: Path): return dump +def output_part(case_dir, log_file, out_kind: str, name: str, fn: Callable[[Path, ], None]): + # make sure the test case directory is created before any test part is written. + case_dir.mkdir(parents=True, exist_ok=True) + try: + fn(case_dir) + except IOError as e: + error_message = f'[Error] error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}' + # Write to error log file + with log_file.open("a+") as f: + f.write(error_message) + traceback.print_exc(file=f) + f.write('\n') + + sys.exit(error_message) + + +def execute_test(test_case, case_dir, meta, log_file, file_mode, cfg_yaml, yaml): + result = test_case.case_fn() + for (name, out_kind, data) in result: + written_part = True + if out_kind == "meta": + meta[name] = data + elif out_kind == "cfg": + output_part(case_dir, log_file, out_kind, name, dump_yaml_fn(data, name, file_mode, cfg_yaml)) + elif out_kind == "data": + output_part(case_dir, log_file, out_kind, name, dump_yaml_fn(data, name, file_mode, yaml)) + elif out_kind == "ssz": + output_part(case_dir, log_file, out_kind, name, dump_ssz_fn(data, name, file_mode)) + else: + raise ValueError("Unknown out_kind %s" % out_kind) + + return written_part, meta + + def dump_ssz_fn(data: AnyStr, name: str, file_mode: str): def dump(case_path: Path): out_path = case_path / Path(name + '.ssz_snappy') From aeccd20fd102021984073162e48fc84abc900601 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Fri, 5 May 2023 23:03:25 +0800 Subject: [PATCH 066/117] Try multiprocessing --- setup.py | 2 +- .../gen_helpers/gen_base/gen_runner.py | 141 ++++++++++++------ tests/core/pyspec/eth2spec/test/context.py | 6 +- 3 files changed, 100 insertions(+), 49 deletions(-) diff --git a/setup.py b/setup.py index 5d27369794..a82f55e02c 100644 --- a/setup.py +++ b/setup.py @@ -1184,7 +1184,7 @@ def run(self): extras_require={ "test": ["pytest>=4.4", "pytest-cov", "pytest-xdist"], "lint": ["flake8==5.0.4", "mypy==0.981", "pylint==2.15.3"], - "generator": ["python-snappy==0.6.1", "filelock"], + "generator": ["python-snappy==0.6.1", "filelock", "pathos==0.3.0"], "docs": ["mkdocs==1.4.2", "mkdocs-material==9.1.5", "mdx-truly-sane-lists==1.3", "mkdocs-awesome-pages-plugin==2.8.0"] }, install_requires=[ diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 5e6ea93d34..56b1bcab18 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -11,12 +11,16 @@ import json from typing import Iterable, AnyStr, Any, Callable import traceback +import multiprocessing +from collections import namedtuple + from ruamel.yaml import ( YAML, ) from filelock import FileLock from snappy import compress +from pathos.multiprocessing import ProcessingPool as Pool from eth_utils import encode_hex @@ -32,6 +36,12 @@ TIME_THRESHOLD_TO_PRINT = 1.0 # seconds +# Generator mode setting +MODE_SINGLE_PROCESS = 'MODE_SINGLE_PROCESS' +MODE_MULTIPROCESSING = 'MODE_MULTIPROCESSING' + +GENERATOR_MODE = MODE_SINGLE_PROCESS + @dataclass class Diagnostics(object): @@ -41,6 +51,45 @@ class Diagnostics(object): test_identifiers: list = field(default_factory=list) +TestCaseParams = namedtuple( + 'TestCaseParams', [ + 'test_case', 'case_dir', 'log_file', 'file_mode', + ]) + + +def worker_function(item): + return generate_test_vector(*item) + + +def get_default_yaml(): + yaml = YAML(pure=True) + yaml.default_flow_style = None + + def _represent_none(self, _): + return self.represent_scalar('tag:yaml.org,2002:null', 'null') + + yaml.representer.add_representer(type(None), _represent_none) + + return yaml + + +def get_cfg_yaml(): + # Spec config is using a YAML subset + cfg_yaml = YAML(pure=True) + cfg_yaml.default_flow_style = False # Emit separate line for each key + + def cfg_represent_bytes(self, data): + return self.represent_int(encode_hex(data)) + + cfg_yaml.representer.add_representer(bytes, cfg_represent_bytes) + + def cfg_represent_quoted_str(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:str', data, style="'") + + cfg_yaml.representer.add_representer(context.quoted_str, cfg_represent_quoted_str) + return cfg_yaml + + def validate_output_dir(path_str): path = Path(path_str) @@ -148,28 +197,6 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): else: file_mode = "w" - yaml = YAML(pure=True) - yaml.default_flow_style = None - - def _represent_none(self, _): - return self.represent_scalar('tag:yaml.org,2002:null', 'null') - - yaml.representer.add_representer(type(None), _represent_none) - - # Spec config is using a YAML subset - cfg_yaml = YAML(pure=True) - cfg_yaml.default_flow_style = False # Emit separate line for each key - - def cfg_represent_bytes(self, data): - return self.represent_int(encode_hex(data)) - - cfg_yaml.representer.add_representer(bytes, cfg_represent_bytes) - - def cfg_represent_quoted_str(self, data): - return self.represent_scalar(u'tag:yaml.org,2002:str', data, style="'") - - cfg_yaml.representer.add_representer(context.quoted_str, cfg_represent_quoted_str) - log_file = Path(output_dir) / 'testgen_error_log.txt' print(f"Generating tests into {output_dir}") @@ -185,8 +212,11 @@ def cfg_represent_quoted_str(self, data): collect_only = args.collect_only diagnostics_obj = Diagnostics() - provider_start = time.time() + + if GENERATOR_MODE == MODE_MULTIPROCESSING: + all_test_case_params = [] + for tprov in test_providers: if not collect_only: # runs anything that we don't want to repeat for every test case. @@ -205,13 +235,20 @@ def cfg_represent_quoted_str(self, data): if is_skip: continue - # generate test vector - is_skip, diagnostics_obj = generate_test_vector_and_diagnose( - test_case, case_dir, log_file, file_mode, - cfg_yaml, yaml, diagnostics_obj, - ) - if is_skip: - continue + if GENERATOR_MODE == MODE_SINGLE_PROCESS: + result = generate_test_vector(test_case, case_dir, log_file, file_mode) + write_result_into_diagnostics_obj(result, diagnostics_obj) + elif GENERATOR_MODE == MODE_MULTIPROCESSING: + item = TestCaseParams(test_case, case_dir, log_file, file_mode) + all_test_case_params.append(item) + + if GENERATOR_MODE == MODE_MULTIPROCESSING: + num_process = multiprocessing.cpu_count() // 2 - 1 + with Pool(processes=num_process) as pool: + results = pool.map(worker_function, iter(all_test_case_params)) + + for result in results: + write_result_into_diagnostics_obj(result, diagnostics_obj) provider_end = time.time() span = round(provider_end - provider_start, 2) @@ -249,54 +286,55 @@ def cfg_represent_quoted_str(self, data): print(f"wrote diagnostics_obj to {diagnostics_path}") -def generate_test_vector_and_diagnose(test_case, case_dir, log_file, file_mode, cfg_yaml, yaml, diagnostics_obj): - is_skip = False +def generate_test_vector(test_case, case_dir, log_file, file_mode): + cfg_yaml = get_cfg_yaml() + yaml = get_default_yaml() + + written_part = False print(f'Generating test: {case_dir}') test_start = time.time() - written_part = False - # Add `INCOMPLETE` tag file to indicate that the test generation has not completed. incomplete_tag_file = get_incomplete_tag_file(case_dir) case_dir.mkdir(parents=True, exist_ok=True) with incomplete_tag_file.open("w") as f: f.write("\n") + result = None try: meta = dict() try: written_part, meta = execute_test(test_case, case_dir, meta, log_file, file_mode, cfg_yaml, yaml) except SkippedTest as e: + result = 0 # 0 means skipped print(e) - diagnostics_obj.skipped_test_count += 1 shutil.rmtree(case_dir) - is_skip = True - return is_skip, diagnostics_obj + return result # Once all meta data is collected (if any), write it to a meta data file. if len(meta) != 0: written_part = True output_part(case_dir, log_file, "data", "meta", dump_yaml_fn(meta, "meta", file_mode, yaml)) - if not written_part: - print(f"test case {case_dir} did not produce any test case parts") except Exception as e: + result = -1 # -1 means error error_message = f"[ERROR] failed to generate vector(s) for test {case_dir}: {e}" # Write to error log file with log_file.open("a+") as f: f.write(error_message) traceback.print_exc(file=f) f.write('\n') + print(error_message) traceback.print_exc() else: # If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder. if not written_part: + print(f"test case {case_dir} did not produce any written_part") shutil.rmtree(case_dir) + result = -1 else: - diagnostics_obj.generated_test_count += 1 - test_identifier = get_test_identifier(test_case) - diagnostics_obj.test_identifiers.append(test_identifier) + result = get_test_identifier(test_case) # Only remove `INCOMPLETE` tag file os.remove(incomplete_tag_file) test_end = time.time() @@ -304,7 +342,19 @@ def generate_test_vector_and_diagnose(test_case, case_dir, log_file, file_mode, if span > TIME_THRESHOLD_TO_PRINT: print(f' - generated in {span} seconds') - return is_skip, diagnostics_obj + return result + + +def write_result_into_diagnostics_obj(result, diagnostics_obj): + if result == -1: # error + pass + elif result == 0: + diagnostics_obj.skipped_test_count += 1 + elif result is not None: + diagnostics_obj.generated_test_count += 1 + diagnostics_obj.test_identifiers.append(result) + else: + raise Exception(f"Unexpected result: {result}") def dump_yaml_fn(data: Any, name: str, file_mode: str, yaml_encoder: YAML): @@ -312,6 +362,7 @@ def dump(case_path: Path): out_path = case_path / Path(name + '.yaml') with out_path.open(file_mode) as f: yaml_encoder.dump(data, f) + f.close() return dump @@ -320,14 +371,14 @@ def output_part(case_dir, log_file, out_kind: str, name: str, fn: Callable[[Path case_dir.mkdir(parents=True, exist_ok=True) try: fn(case_dir) - except IOError as e: + except (IOError, ValueError) as e: error_message = f'[Error] error when dumping test "{case_dir}", part "{name}", kind "{out_kind}": {e}' # Write to error log file with log_file.open("a+") as f: f.write(error_message) traceback.print_exc(file=f) f.write('\n') - + print(error_message) sys.exit(error_message) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index 901fd273a8..626ffc1dbc 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -560,7 +560,7 @@ def _get_basic_dict(ssz_dict: Dict[str, Any]) -> Dict[str, Any]: return result -def _get_copy_of_spec(spec): +def get_copy_of_spec(spec): fork = spec.fork preset = spec.config.PRESET_BASE module_path = f"eth2spec.{fork}.{preset}" @@ -601,14 +601,14 @@ def with_config_overrides(config_overrides, emitted_fork=None, emit=True): def decorator(fn): def wrapper(*args, spec: Spec, **kw): # Apply config overrides to spec - spec, output_config = spec_with_config_overrides(_get_copy_of_spec(spec), config_overrides) + spec, output_config = spec_with_config_overrides(get_copy_of_spec(spec), config_overrides) # Apply config overrides to additional phases, if present if 'phases' in kw: phases = {} for fork in kw['phases']: phases[fork], output = spec_with_config_overrides( - _get_copy_of_spec(kw['phases'][fork]), config_overrides) + get_copy_of_spec(kw['phases'][fork]), config_overrides) if emitted_fork == fork: output_config = output kw['phases'] = phases From 98d0ca48b8730f05ed180182971f629bfedd8d0d Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 6 May 2023 17:45:22 +0800 Subject: [PATCH 067/117] Fix `test_randomized_state` and `test_randomized_state_leaking` --- .../test_process_inactivity_updates.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_inactivity_updates.py b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_inactivity_updates.py index 0816dfad62..57fe8b9ca3 100644 --- a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_inactivity_updates.py +++ b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_inactivity_updates.py @@ -54,7 +54,15 @@ def test_genesis_random_scores(spec, state): # def run_inactivity_scores_test(spec, state, participation_fn=None, inactivity_scores_fn=None, rng=Random(10101)): - next_epoch_via_block(spec, state) + while True: + try: + next_epoch_via_block(spec, state) + except AssertionError: + # If the proposer is slashed, we skip this epoch and try to propose block at the next epoch + next_epoch(spec, state) + else: + break + if participation_fn is not None: participation_fn(spec, state, rng=rng) if inactivity_scores_fn is not None: @@ -363,7 +371,7 @@ def test_randomized_state(spec, state): their inactivity score does not change. """ rng = Random(10011001) - _run_randomized_state_test_for_inactivity_updates(spec, state, rng=rng) + yield from _run_randomized_state_test_for_inactivity_updates(spec, state, rng=rng) @with_altair_and_later @@ -377,6 +385,6 @@ def test_randomized_state_leaking(spec, state): (refer ``get_eligible_validator_indices`). """ rng = Random(10011002) - _run_randomized_state_test_for_inactivity_updates(spec, state, rng=rng) + yield from _run_randomized_state_test_for_inactivity_updates(spec, state, rng=rng) # Check still in leak assert spec.is_in_inactivity_leak(state) From 3ae4bf14f1d607a19f5b3bd694b841fff0545d98 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 6 May 2023 20:11:50 +0800 Subject: [PATCH 068/117] Fix and set to `MODE_MULTIPROCESSING` --- .../core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index 56b1bcab18..c8b7f0c039 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -40,7 +40,7 @@ MODE_SINGLE_PROCESS = 'MODE_SINGLE_PROCESS' MODE_MULTIPROCESSING = 'MODE_MULTIPROCESSING' -GENERATOR_MODE = MODE_SINGLE_PROCESS +GENERATOR_MODE = MODE_MULTIPROCESSING @dataclass @@ -330,7 +330,7 @@ def generate_test_vector(test_case, case_dir, log_file, file_mode): else: # If no written_part, the only file was incomplete_tag_file. Clear the existing case_dir folder. if not written_part: - print(f"test case {case_dir} did not produce any written_part") + print(f"[Error] test case {case_dir} did not produce any written_part") shutil.rmtree(case_dir) result = -1 else: @@ -384,6 +384,7 @@ def output_part(case_dir, log_file, out_kind: str, name: str, fn: Callable[[Path def execute_test(test_case, case_dir, meta, log_file, file_mode, cfg_yaml, yaml): result = test_case.case_fn() + written_part = False for (name, out_kind, data) in result: written_part = True if out_kind == "meta": From 5e2a18a319bfe0be233444d202b6306c81dc0f64 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 6 May 2023 17:45:22 +0800 Subject: [PATCH 069/117] Fix `test_randomized_state` and `test_randomized_state_leaking` --- .../test_process_inactivity_updates.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_inactivity_updates.py b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_inactivity_updates.py index 0816dfad62..57fe8b9ca3 100644 --- a/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_inactivity_updates.py +++ b/tests/core/pyspec/eth2spec/test/altair/epoch_processing/test_process_inactivity_updates.py @@ -54,7 +54,15 @@ def test_genesis_random_scores(spec, state): # def run_inactivity_scores_test(spec, state, participation_fn=None, inactivity_scores_fn=None, rng=Random(10101)): - next_epoch_via_block(spec, state) + while True: + try: + next_epoch_via_block(spec, state) + except AssertionError: + # If the proposer is slashed, we skip this epoch and try to propose block at the next epoch + next_epoch(spec, state) + else: + break + if participation_fn is not None: participation_fn(spec, state, rng=rng) if inactivity_scores_fn is not None: @@ -363,7 +371,7 @@ def test_randomized_state(spec, state): their inactivity score does not change. """ rng = Random(10011001) - _run_randomized_state_test_for_inactivity_updates(spec, state, rng=rng) + yield from _run_randomized_state_test_for_inactivity_updates(spec, state, rng=rng) @with_altair_and_later @@ -377,6 +385,6 @@ def test_randomized_state_leaking(spec, state): (refer ``get_eligible_validator_indices`). """ rng = Random(10011002) - _run_randomized_state_test_for_inactivity_updates(spec, state, rng=rng) + yield from _run_randomized_state_test_for_inactivity_updates(spec, state, rng=rng) # Check still in leak assert spec.is_in_inactivity_leak(state) From 1008714e545b5a1e60cca772271f7df0592db394 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 9 May 2023 21:49:41 +0800 Subject: [PATCH 070/117] Add `settings.py` of testgen --- .../gen_helpers/gen_base/gen_runner.py | 20 ++++++++----------- .../eth2spec/gen_helpers/gen_base/settings.py | 13 ++++++++++++ 2 files changed, 21 insertions(+), 12 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py index c8b7f0c039..2562c7fad9 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py @@ -11,7 +11,6 @@ import json from typing import Iterable, AnyStr, Any, Callable import traceback -import multiprocessing from collections import namedtuple from ruamel.yaml import ( @@ -28,21 +27,19 @@ from eth2spec.test.exceptions import SkippedTest from .gen_typing import TestProvider +from .settings import ( + GENERATOR_MODE, + MODE_MULTIPROCESSING, + MODE_SINGLE_PROCESS, + NUM_PROCESS, + TIME_THRESHOLD_TO_PRINT, +) # Flag that the runner does NOT run test via pytest context.is_pytest = False -TIME_THRESHOLD_TO_PRINT = 1.0 # seconds - -# Generator mode setting -MODE_SINGLE_PROCESS = 'MODE_SINGLE_PROCESS' -MODE_MULTIPROCESSING = 'MODE_MULTIPROCESSING' - -GENERATOR_MODE = MODE_MULTIPROCESSING - - @dataclass class Diagnostics(object): collected_test_count: int = 0 @@ -243,8 +240,7 @@ def run_generator(generator_name, test_providers: Iterable[TestProvider]): all_test_case_params.append(item) if GENERATOR_MODE == MODE_MULTIPROCESSING: - num_process = multiprocessing.cpu_count() // 2 - 1 - with Pool(processes=num_process) as pool: + with Pool(processes=NUM_PROCESS) as pool: results = pool.map(worker_function, iter(all_test_case_params)) for result in results: diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py new file mode 100644 index 0000000000..b8c3e591fb --- /dev/null +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py @@ -0,0 +1,13 @@ +import multiprocessing + + +# Generator mode setting +MODE_SINGLE_PROCESS = 'MODE_SINGLE_PROCESS' +MODE_MULTIPROCESSING = 'MODE_MULTIPROCESSING' +# Test generator mode +GENERATOR_MODE = MODE_MULTIPROCESSING +# Number of subprocesses when using MODE_MULTIPROCESSING +NUM_PROCESS = multiprocessing.cpu_count() // 2 - 1 + +# Diagnostics +TIME_THRESHOLD_TO_PRINT = 1.0 # seconds From 0f5ac1186e29ec883abd11842530f91c74d5bc52 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 10 May 2023 02:29:00 +0800 Subject: [PATCH 071/117] Remove `is_execution_enabled` condition since Capella --- specs/_features/eip6110/beacon-chain.md | 8 +++---- specs/_features/sharding/beacon-chain.md | 3 +-- specs/capella/beacon-chain.md | 6 ++--- specs/deneb/beacon-chain.md | 5 ++-- .../test/bellatrix/sanity/test_blocks.py | 7 ++++-- .../test/capella/sanity/test_blocks.py | 23 +++++++++++++++++++ 6 files changed, 37 insertions(+), 15 deletions(-) diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index 708418e1cd..4d69fb4e05 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -176,9 +176,8 @@ class BeaconState(Container): ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) - if is_execution_enabled(state, block.body): - process_withdrawals(state, block.body.execution_payload) - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in EIP6110] + process_withdrawals(state, block.body.execution_payload) + process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in EIP6110] process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) # [Modified in EIP6110] @@ -212,8 +211,7 @@ def process_operations(state: BeaconState, body: BeaconBlockBody) -> None: for_ops(body.bls_to_execution_changes, process_bls_to_execution_change) # [New in EIP6110] - if is_execution_enabled(state, body): - for_ops(body.execution_payload.deposit_receipts, process_deposit_receipt) + for_ops(body.execution_payload.deposit_receipts, process_deposit_receipt) ``` #### New `process_deposit_receipt` diff --git a/specs/_features/sharding/beacon-chain.md b/specs/_features/sharding/beacon-chain.md index 7d6df51aa8..f7de7af656 100644 --- a/specs/_features/sharding/beacon-chain.md +++ b/specs/_features/sharding/beacon-chain.md @@ -236,8 +236,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) verify_builder_block_bid(state, block) process_sharded_data(state, block) - if is_execution_enabled(state, block.body): - process_execution_payload(state, block, EXECUTION_ENGINE) + process_execution_payload(state, block, EXECUTION_ENGINE) if not is_builder_block_slot(block.slot): process_randao(state, block.body) diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index 1df617daaf..72260108f9 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -331,9 +331,9 @@ def process_historical_summaries_update(state: BeaconState) -> None: ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) - if is_execution_enabled(state, block.body): - process_withdrawals(state, block.body.execution_payload) # [New in Capella] - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Capella] + # Removed `is_execution_enabled` check + process_withdrawals(state, block.body.execution_payload) # [New in Capella] + process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Capella] process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) # [Modified in Capella] diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 359c7fc95b..6aaa2567f6 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -200,9 +200,8 @@ def verify_kzg_commitments_against_transactions(transactions: Sequence[Transacti ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) - if is_execution_enabled(state, block.body): - process_withdrawals(state, block.body.execution_payload) - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Deneb] + process_withdrawals(state, block.body.execution_payload) + process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Deneb] process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/bellatrix/sanity/test_blocks.py index ef6bb75a9c..75dfa0c9cf 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/sanity/test_blocks.py @@ -10,7 +10,10 @@ build_randomized_execution_payload ) from eth2spec.test.context import ( - with_bellatrix_and_later, spec_state_test + BELLATRIX, + with_bellatrix_and_later, + with_phases, + spec_state_test, ) @@ -44,7 +47,7 @@ def test_empty_block_transition_randomized_payload(spec, state): yield 'post', state -@with_bellatrix_and_later +@with_phases([BELLATRIX]) @spec_state_test def test_is_execution_enabled_false(spec, state): # Set `latest_execution_payload_header` to empty diff --git a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py index d62e458be6..11d6f4e915 100644 --- a/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/capella/sanity/test_blocks.py @@ -31,6 +31,29 @@ from eth2spec.test.helpers.voluntary_exits import prepare_signed_exits +# +# `is_execution_enabled` has been removed from Capella +# + +@with_capella_and_later +@spec_state_test +def test_invalid_is_execution_enabled_false(spec, state): + # Set `latest_execution_payload_header` to empty + state.latest_execution_payload_header = spec.ExecutionPayloadHeader() + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + + # Set `execution_payload` to empty + block.body.execution_payload = spec.ExecutionPayload() + assert len(block.body.execution_payload.transactions) == 0 + + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + + yield 'blocks', [signed_block] + yield 'post', None + + # # BLSToExecutionChange # From 5b983f40972e44a07a9022390898a48fab37e599 Mon Sep 17 00:00:00 2001 From: gajinder Date: Fri, 12 May 2023 21:44:43 +0530 Subject: [PATCH 072/117] Update the endianess of the polynomial commitments to be big endian --- specs/deneb/polynomial-commitments.md | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/specs/deneb/polynomial-commitments.md b/specs/deneb/polynomial-commitments.md index e23c31fab8..fb2d702372 100644 --- a/specs/deneb/polynomial-commitments.md +++ b/specs/deneb/polynomial-commitments.md @@ -77,6 +77,7 @@ Public functions MUST accept raw bytes as input and perform the required cryptog | `BYTES_PER_FIELD_ELEMENT` | `uint64(32)` | Bytes used to encode a BLS scalar field element | | `BYTES_PER_BLOB` | `uint64(BYTES_PER_FIELD_ELEMENT * FIELD_ELEMENTS_PER_BLOB)` | The number of bytes in a blob | | `G1_POINT_AT_INFINITY` | `Bytes48(b'\xc0' + b'\x00' * 47)` | Serialized form of the point at infinity on the G1 group | +| `KZG_ENDIANNESS` | `'big'` | The endianess of the field elements including blobs | ## Preset @@ -161,7 +162,7 @@ def hash_to_bls_field(data: bytes) -> BLSFieldElement: The output is not uniform over the BLS field. """ hashed_data = hash(data) - return BLSFieldElement(int.from_bytes(hashed_data, ENDIANNESS) % BLS_MODULUS) + return BLSFieldElement(int.from_bytes(hashed_data, KZG_ENDIANNESS) % BLS_MODULUS) ``` #### `bytes_to_bls_field` @@ -172,7 +173,7 @@ def bytes_to_bls_field(b: Bytes32) -> BLSFieldElement: Convert untrusted bytes to a trusted and validated BLS scalar field element. This function does not accept inputs greater than the BLS modulus. """ - field_element = int.from_bytes(b, ENDIANNESS) + field_element = int.from_bytes(b, KZG_ENDIANNESS) assert field_element < BLS_MODULUS return BLSFieldElement(field_element) ``` @@ -237,7 +238,7 @@ def compute_challenge(blob: Blob, """ # Append the degree of the polynomial as a domain separator - degree_poly = int.to_bytes(FIELD_ELEMENTS_PER_BLOB, 16, ENDIANNESS) + degree_poly = int.to_bytes(FIELD_ELEMENTS_PER_BLOB, 16, KZG_ENDIANNESS) data = FIAT_SHAMIR_PROTOCOL_DOMAIN + degree_poly data += blob @@ -406,15 +407,15 @@ def verify_kzg_proof_batch(commitments: Sequence[KZGCommitment], # Compute a random challenge. Note that it does not have to be computed from a hash, # r just has to be random. - degree_poly = int.to_bytes(FIELD_ELEMENTS_PER_BLOB, 8, ENDIANNESS) - num_commitments = int.to_bytes(len(commitments), 8, ENDIANNESS) + degree_poly = int.to_bytes(FIELD_ELEMENTS_PER_BLOB, 8, KZG_ENDIANNESS) + num_commitments = int.to_bytes(len(commitments), 8, KZG_ENDIANNESS) data = RANDOM_CHALLENGE_KZG_BATCH_DOMAIN + degree_poly + num_commitments # Append all inputs to the transcript before we hash for commitment, z, y, proof in zip(commitments, zs, ys, proofs): data += commitment \ - + int.to_bytes(z, BYTES_PER_FIELD_ELEMENT, ENDIANNESS) \ - + int.to_bytes(y, BYTES_PER_FIELD_ELEMENT, ENDIANNESS) \ + + int.to_bytes(z, BYTES_PER_FIELD_ELEMENT, KZG_ENDIANNESS) \ + + int.to_bytes(y, BYTES_PER_FIELD_ELEMENT, KZG_ENDIANNESS) \ + proof r = hash_to_bls_field(data) @@ -451,7 +452,7 @@ def compute_kzg_proof(blob: Blob, z_bytes: Bytes32) -> Tuple[KZGProof, Bytes32]: assert len(z_bytes) == BYTES_PER_FIELD_ELEMENT polynomial = blob_to_polynomial(blob) proof, y = compute_kzg_proof_impl(polynomial, bytes_to_bls_field(z_bytes)) - return proof, y.to_bytes(BYTES_PER_FIELD_ELEMENT, ENDIANNESS) + return proof, y.to_bytes(BYTES_PER_FIELD_ELEMENT, KZG_ENDIANNESS) ``` #### `compute_quotient_eval_within_domain` From 847553783b786686e7f24c69d104c055cbf9508b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 May 2023 17:11:50 +0800 Subject: [PATCH 073/117] Add `test_zeroed_commitment` --- .../eth2spec/test/deneb/sanity/test_blocks.py | 20 +++++++++++++++++++ .../pyspec/eth2spec/test/helpers/sharding.py | 19 +++++++++--------- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 111565cce2..b518f2b55a 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -150,3 +150,23 @@ def test_incorrect_block_hash(spec, state): yield 'blocks', [signed_block] yield 'post', state + + +@with_deneb_and_later +@spec_state_test +def test_zeroed_commitment(spec, state): + """ + The blob is invalid, but the commitment is in correct form. + """ + yield 'pre', state + + block = build_empty_block_for_next_slot(spec, state) + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=1, is_valid_blob=False) + assert all(commitment == b'\x00' * 48 for commitment in blob_kzg_commitments) + block.body.blob_kzg_commitments = blob_kzg_commitments + block.body.execution_payload.transactions = [opaque_tx] + block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) + signed_block = state_transition_and_sign_block(spec, state, block) + + yield 'blocks', [signed_block] + yield 'post', state diff --git a/tests/core/pyspec/eth2spec/test/helpers/sharding.py b/tests/core/pyspec/eth2spec/test/helpers/sharding.py index 6b913b90ec..16b5a02341 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/sharding.py +++ b/tests/core/pyspec/eth2spec/test/helpers/sharding.py @@ -50,12 +50,9 @@ class SignedBlobTransaction(Container): signature: ECDSASignature -def get_sample_blob(spec, rng=None): - if rng is None: - rng = random.Random(5566) - +def get_sample_blob(spec, rng=random.Random(5566), is_valid_blob=True): values = [ - rng.randint(0, spec.BLS_MODULUS - 1) + rng.randint(0, spec.BLS_MODULUS - 1) if is_valid_blob else spec.BLS_MODULUS for _ in range(spec.FIELD_ELEMENTS_PER_BLOB) ] @@ -98,15 +95,19 @@ def get_poly_in_both_forms(spec, rng=None): return coeffs, evals -def get_sample_opaque_tx(spec, blob_count=1, rng=None): +def get_sample_opaque_tx(spec, blob_count=1, rng=random.Random(5566), is_valid_blob=True): blobs = [] blob_kzg_commitments = [] blob_kzg_proofs = [] blob_versioned_hashes = [] for _ in range(blob_count): - blob = get_sample_blob(spec, rng) - blob_commitment = spec.KZGCommitment(spec.blob_to_kzg_commitment(blob)) - blob_kzg_proof = spec.compute_blob_kzg_proof(blob, blob_commitment) + blob = get_sample_blob(spec, rng, is_valid_blob=is_valid_blob) + if is_valid_blob: + blob_commitment = spec.KZGCommitment(spec.blob_to_kzg_commitment(blob)) + blob_kzg_proof = spec.compute_blob_kzg_proof(blob, blob_commitment) + else: + blob_commitment = spec.KZGCommitment() + blob_kzg_proof = spec.KZGProof() blob_versioned_hash = spec.kzg_commitment_to_versioned_hash(blob_commitment) blobs.append(blob) blob_kzg_commitments.append(blob_commitment) From eea04704d48e8ddb7626f9835d9d81c548f15498 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 May 2023 17:16:41 +0800 Subject: [PATCH 074/117] Fix tests --- setup.py | 2 +- .../polynomial_commitments/test_polynomial_commitments.py | 6 +++--- tests/core/pyspec/eth2spec/test/helpers/sharding.py | 2 +- tests/generators/kzg_4844/main.py | 6 +++--- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index b2316ed958..c6a9ebaa92 100644 --- a/setup.py +++ b/setup.py @@ -279,7 +279,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr elif name in config: config_vars[name] = VariableDefinition(value_def.type_name, config[name], value_def.comment, None) else: - if name == 'ENDIANNESS': + if name in ('ENDIANNESS', 'KZG_ENDIANNESS'): # Deal with mypy Literal typing check value_def = _parse_value(name, value, type_hint='Final') constant_vars[name] = value_def diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py index 6d3f377a33..5ee8097c82 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/polynomial_commitments/test_polynomial_commitments.py @@ -33,7 +33,7 @@ def bls_add_one(x): def field_element_bytes(x): - return int.to_bytes(x % BLS_MODULUS, 32, "little") + return int.to_bytes(x % BLS_MODULUS, 32, "big") @with_deneb_and_later @@ -304,7 +304,7 @@ def test_bytes_to_bls_field_modulus_minus_one(spec): Verify that `bytes_to_bls_field` handles modulus minus one """ - spec.bytes_to_bls_field((BLS_MODULUS - 1).to_bytes(spec.BYTES_PER_FIELD_ELEMENT, spec.ENDIANNESS)) + spec.bytes_to_bls_field((BLS_MODULUS - 1).to_bytes(spec.BYTES_PER_FIELD_ELEMENT, spec.KZG_ENDIANNESS)) @with_deneb_and_later @@ -316,7 +316,7 @@ def test_bytes_to_bls_field_modulus(spec): """ expect_assertion_error(lambda: spec.bytes_to_bls_field( - BLS_MODULUS.to_bytes(spec.BYTES_PER_FIELD_ELEMENT, spec.ENDIANNESS) + BLS_MODULUS.to_bytes(spec.BYTES_PER_FIELD_ELEMENT, spec.KZG_ENDIANNESS) )) diff --git a/tests/core/pyspec/eth2spec/test/helpers/sharding.py b/tests/core/pyspec/eth2spec/test/helpers/sharding.py index 6b913b90ec..cd88ae3ea8 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/sharding.py +++ b/tests/core/pyspec/eth2spec/test/helpers/sharding.py @@ -61,7 +61,7 @@ def get_sample_blob(spec, rng=None): b = bytes() for v in values: - b += v.to_bytes(32, spec.ENDIANNESS) + b += v.to_bytes(32, spec.KZG_ENDIANNESS) return spec.Blob(b) diff --git a/tests/generators/kzg_4844/main.py b/tests/generators/kzg_4844/main.py index 2f3efeb21a..9297e524ac 100644 --- a/tests/generators/kzg_4844/main.py +++ b/tests/generators/kzg_4844/main.py @@ -27,11 +27,11 @@ def expect_exception(func, *args): def field_element_bytes(x): - return int.to_bytes(x % spec.BLS_MODULUS, 32, spec.ENDIANNESS) + return int.to_bytes(x % spec.BLS_MODULUS, 32, spec.KZG_ENDIANNESS) def field_element_bytes_unchecked(x): - return int.to_bytes(x, 32, spec.ENDIANNESS) + return int.to_bytes(x, 32, spec.KZG_ENDIANNESS) def encode_hex_list(a): @@ -54,7 +54,7 @@ def evaluate_blob_at(blob, z): ) -BLS_MODULUS_BYTES = spec.BLS_MODULUS.to_bytes(32, spec.ENDIANNESS) +BLS_MODULUS_BYTES = spec.BLS_MODULUS.to_bytes(32, spec.KZG_ENDIANNESS) G1 = bls.G1_to_bytes48(bls.G1()) G1_INVALID_TOO_FEW_BYTES = G1[:-1] From 340f3cc1a3c54bac1892050d7555b53d305ecd56 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Mon, 15 May 2023 17:25:48 +0800 Subject: [PATCH 075/117] Rebase the fc changes --- specs/capella/fork-choice.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/specs/capella/fork-choice.md b/specs/capella/fork-choice.md index c08be1c3f6..87fec02f8c 100644 --- a/specs/capella/fork-choice.md +++ b/specs/capella/fork-choice.md @@ -86,16 +86,22 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) assert block.slot > finalized_slot # Check block is a descendant of the finalized block at the checkpoint finalized slot - assert get_ancestor(store, block.parent_root, finalized_slot) == store.finalized_checkpoint.root + finalized_checkpoint_block = get_checkpoint_block( + store, + block.parent_root, + store.finalized_checkpoint.epoch, + ) + assert store.finalized_checkpoint.root == finalized_checkpoint_block # Check the block is valid and compute the post-state state = pre_state.copy() + block_root = hash_tree_root(block) state_transition(state, signed_block, True) # Add new block to the store - store.blocks[hash_tree_root(block)] = block + store.blocks[block_root] = block # Add new state for this block to the store - store.block_states[hash_tree_root(block)] = state + store.block_states[block_root] = state # Add proposer score boost if the block is timely time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT @@ -103,15 +109,9 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: if get_current_slot(store) == block.slot and is_before_attesting_interval: store.proposer_boost_root = hash_tree_root(block) - # Update justified checkpoint - if state.current_justified_checkpoint.epoch > store.justified_checkpoint.epoch: - if state.current_justified_checkpoint.epoch > store.best_justified_checkpoint.epoch: - store.best_justified_checkpoint = state.current_justified_checkpoint - if should_update_justified_checkpoint(store, state.current_justified_checkpoint): - store.justified_checkpoint = state.current_justified_checkpoint - - # Update finalized checkpoint - if state.finalized_checkpoint.epoch > store.finalized_checkpoint.epoch: - store.finalized_checkpoint = state.finalized_checkpoint - store.justified_checkpoint = state.current_justified_checkpoint + # Update checkpoints in store if necessary + update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) + + # Eagerly compute unrealized justification and finality. + compute_pulled_up_tip(store, block_root) ``` From 928fd42517b6ffc41ffd96e2e06066f47762ad4d Mon Sep 17 00:00:00 2001 From: g11tech Date: Mon, 15 May 2023 15:53:18 +0530 Subject: [PATCH 076/117] fix typo Co-authored-by: Hsiao-Wei Wang --- specs/deneb/polynomial-commitments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/polynomial-commitments.md b/specs/deneb/polynomial-commitments.md index fb2d702372..fe5ae845a7 100644 --- a/specs/deneb/polynomial-commitments.md +++ b/specs/deneb/polynomial-commitments.md @@ -77,7 +77,7 @@ Public functions MUST accept raw bytes as input and perform the required cryptog | `BYTES_PER_FIELD_ELEMENT` | `uint64(32)` | Bytes used to encode a BLS scalar field element | | `BYTES_PER_BLOB` | `uint64(BYTES_PER_FIELD_ELEMENT * FIELD_ELEMENTS_PER_BLOB)` | The number of bytes in a blob | | `G1_POINT_AT_INFINITY` | `Bytes48(b'\xc0' + b'\x00' * 47)` | Serialized form of the point at infinity on the G1 group | -| `KZG_ENDIANNESS` | `'big'` | The endianess of the field elements including blobs | +| `KZG_ENDIANNESS` | `'big'` | The endianness of the field elements including blobs | ## Preset From 058137327a4cb9163c4202e1ed3bd4d886d745cf Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 May 2023 00:27:00 +0800 Subject: [PATCH 077/117] Use new `engine_newPayloadV3` --- setup.py | 2 +- specs/_features/eip6110/beacon-chain.md | 10 +-- specs/bellatrix/beacon-chain.md | 25 +++++-- specs/capella/beacon-chain.md | 8 ++- specs/deneb/beacon-chain.md | 71 ++++++++----------- specs/deneb/validator.md | 4 +- .../test_process_execution_payload.py | 4 +- .../test/phase0/sanity/test_blocks.py | 2 +- 8 files changed, 67 insertions(+), 59 deletions(-) diff --git a/setup.py b/setup.py index b2316ed958..8ef66df232 100644 --- a/setup.py +++ b/setup.py @@ -578,7 +578,7 @@ def get_pow_chain_head() -> PowBlock: class NoopExecutionEngine(ExecutionEngine): - def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: return True def notify_forkchoice_updated(self: ExecutionEngine, diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index 708418e1cd..c9dccbdebb 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -178,12 +178,11 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) if is_execution_enabled(state, block.body): process_withdrawals(state, block.body.execution_payload) - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in EIP6110] + process_execution_payload(state, block.body, EXECUTION_ENGINE) # [Modified in EIP6110] process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) # [Modified in EIP6110] process_sync_aggregate(state, block.body.sync_aggregate) - process_blob_kzg_commitments(block.body) ``` #### Modified `process_operations` @@ -238,7 +237,9 @@ def process_deposit_receipt(state: BeaconState, deposit_receipt: DepositReceipt) *Note*: The function `process_execution_payload` is modified to use the new `ExecutionPayloadHeader` type. ```python -def process_execution_payload(state: BeaconState, payload: ExecutionPayload, execution_engine: ExecutionEngine) -> None: +def process_execution_payload(state: BeaconState, body: BeaconBlockBody, execution_engine: ExecutionEngine) -> None: + payload = body.execution_payload + # Verify consistency of the parent hash with respect to the previous execution payload header if is_merge_transition_complete(state): assert payload.parent_hash == state.latest_execution_payload_header.block_hash @@ -247,7 +248,8 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(payload) + versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] + assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload)) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index 1133cba06c..a4c7be16e9 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -33,6 +33,9 @@ - [Modified `slash_validator`](#modified-slash_validator) - [Beacon chain state transition function](#beacon-chain-state-transition-function) - [Execution engine](#execution-engine) + - [Request data](#request-data) + - [`NewPayloadRequest`](#newpayloadrequest) + - [Engine APIs](#engine-apis) - [`notify_new_payload`](#notify_new_payload) - [Block processing](#block-processing) - [Execution payload](#execution-payload) @@ -300,6 +303,18 @@ def slash_validator(state: BeaconState, ### Execution engine +#### Request data + +##### `NewPayloadRequest` + +```python +@dataclass +class NewPayloadRequest(object): + execution_payload: ExecutionPayload +``` + +#### Engine APIs + The implementation-dependent `ExecutionEngine` protocol encapsulates the execution sub-system logic via: * a state object `self.execution_state` of type `ExecutionState` @@ -313,7 +328,7 @@ The Engine API may be used to implement this and similarly defined functions via #### `notify_new_payload` ```python -def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: +def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: """ Return ``True`` if and only if ``execution_payload`` is valid with respect to ``self.execution_state``. """ @@ -328,7 +343,7 @@ def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayloa def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) if is_execution_enabled(state, block.body): - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [New in Bellatrix] + process_execution_payload(state, block.body, EXECUTION_ENGINE) # [New in Bellatrix] process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) @@ -340,7 +355,9 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: ##### `process_execution_payload` ```python -def process_execution_payload(state: BeaconState, payload: ExecutionPayload, execution_engine: ExecutionEngine) -> None: +def process_execution_payload(state: BeaconState, body: BeaconBlockBody, execution_engine: ExecutionEngine) -> None: + payload = body.execution_payload + # Verify consistency of the parent hash with respect to the previous execution payload header if is_merge_transition_complete(state): assert payload.parent_hash == state.latest_execution_payload_header.block_hash @@ -349,7 +366,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(payload) + assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload)) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index 1df617daaf..1799f4e6e3 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -333,7 +333,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) if is_execution_enabled(state, block.body): process_withdrawals(state, block.body.execution_payload) # [New in Capella] - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Capella] + process_execution_payload(state, block.body, EXECUTION_ENGINE) # [Modified in Capella] process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) # [Modified in Capella] @@ -407,7 +407,9 @@ def process_withdrawals(state: BeaconState, payload: ExecutionPayload) -> None: *Note*: The function `process_execution_payload` is modified to use the new `ExecutionPayloadHeader` type. ```python -def process_execution_payload(state: BeaconState, payload: ExecutionPayload, execution_engine: ExecutionEngine) -> None: +def process_execution_payload(state: BeaconState, body: BeaconBlockBody, execution_engine: ExecutionEngine) -> None: + payload = body.execution_payload + # Verify consistency of the parent hash with respect to the previous execution payload header if is_merge_transition_complete(state): assert payload.parent_hash == state.latest_execution_payload_header.block_hash @@ -416,7 +418,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(payload) + assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload)) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 359c7fc95b..52719dcf68 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -24,13 +24,15 @@ - [Helper functions](#helper-functions) - [Misc](#misc) - [`kzg_commitment_to_versioned_hash`](#kzg_commitment_to_versioned_hash) - - [`tx_peek_blob_versioned_hashes`](#tx_peek_blob_versioned_hashes) - - [`verify_kzg_commitments_against_transactions`](#verify_kzg_commitments_against_transactions) - [Beacon chain state transition function](#beacon-chain-state-transition-function) + - [Execution engine](#execution-engine) + - [Request data](#request-data) + - [Modified `NewPayloadRequest`](#modified-newpayloadrequest) + - [Engine APIs](#engine-apis) + - [Modified `notify_new_payload`](#modified-notify_new_payload) - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) - - [Blob KZG commitments](#blob-kzg-commitments) - [Testing](#testing) @@ -158,43 +160,33 @@ def kzg_commitment_to_versioned_hash(kzg_commitment: KZGCommitment) -> Versioned return VERSIONED_HASH_VERSION_KZG + hash(kzg_commitment)[1:] ``` -#### `tx_peek_blob_versioned_hashes` +## Beacon chain state transition function + +### Execution engine + +#### Request data -This function retrieves the hashes from the `SignedBlobTransaction` as defined in Deneb, using SSZ offsets. -Offsets are little-endian `uint32` values, as defined in the [SSZ specification](../../ssz/simple-serialize.md). -See [the full details of `blob_versioned_hashes` offset calculation](https://gist.github.com/protolambda/23bd106b66f6d4bb854ce46044aa3ca3). +##### Modified `NewPayloadRequest` ```python -def tx_peek_blob_versioned_hashes(opaque_tx: Transaction) -> Sequence[VersionedHash]: - assert opaque_tx[0] == BLOB_TX_TYPE - message_offset = 1 + uint32.decode_bytes(opaque_tx[1:5]) - # field offset: 32 + 8 + 32 + 32 + 8 + 4 + 32 + 4 + 4 + 32 = 188 - blob_versioned_hashes_offset = ( - message_offset - + uint32.decode_bytes(opaque_tx[(message_offset + 188):(message_offset + 192)]) - ) - # `VersionedHash` is a 32-byte object - assert (len(opaque_tx) - blob_versioned_hashes_offset) % 32 == 0 - return [ - VersionedHash(opaque_tx[x:(x + 32)]) - for x in range(blob_versioned_hashes_offset, len(opaque_tx), 32) - ] +@dataclass +class NewPayloadRequest(object): + execution_payload: ExecutionPayload + versioned_hashes: Sequence[VersionedHash] ``` -#### `verify_kzg_commitments_against_transactions` +#### Engine APIs + +#### Modified `notify_new_payload` ```python -def verify_kzg_commitments_against_transactions(transactions: Sequence[Transaction], - kzg_commitments: Sequence[KZGCommitment]) -> bool: - all_versioned_hashes: List[VersionedHash] = [] - for tx in transactions: - if tx[0] == BLOB_TX_TYPE: - all_versioned_hashes += tx_peek_blob_versioned_hashes(tx) - return all_versioned_hashes == [kzg_commitment_to_versioned_hash(commitment) for commitment in kzg_commitments] +def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: + """ + Return ``True`` if and only if ``execution_payload`` is valid with respect to ``self.execution_state``. + """ + ... ``` -## Beacon chain state transition function - ### Block processing ```python @@ -202,12 +194,11 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) if is_execution_enabled(state, block.body): process_withdrawals(state, block.body.execution_payload) - process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Deneb] + process_execution_payload(state, block.body, EXECUTION_ENGINE) # [Modified in Deneb] process_randao(state, block.body) process_eth1_data(state, block.body) process_operations(state, block.body) process_sync_aggregate(state, block.body.sync_aggregate) - process_blob_kzg_commitments(block.body) # [New in Deneb] ``` #### Execution payload @@ -215,7 +206,9 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: ##### `process_execution_payload` ```python -def process_execution_payload(state: BeaconState, payload: ExecutionPayload, execution_engine: ExecutionEngine) -> None: +def process_execution_payload(state: BeaconState, body: BeaconBlockBody, execution_engine: ExecutionEngine) -> None: + payload = body.execution_payload + # Verify consistency of the parent hash with respect to the previous execution payload header if is_merge_transition_complete(state): assert payload.parent_hash == state.latest_execution_payload_header.block_hash @@ -224,7 +217,8 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(payload) + versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] + assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes)) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( @@ -247,13 +241,6 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe ) ``` -#### Blob KZG commitments - -```python -def process_blob_kzg_commitments(body: BeaconBlockBody) -> None: - assert verify_kzg_commitments_against_transactions(body.execution_payload.transactions, body.blob_kzg_commitments) -``` - ## Testing *Note*: The function `initialize_beacon_state_from_eth1` is modified for pure Deneb testing only. diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index 6562c91ddd..e52ce9eaac 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -98,8 +98,8 @@ def validate_blobs_and_kzg_commitments(execution_payload: ExecutionPayload, blobs: Sequence[Blob], blob_kzg_commitments: Sequence[KZGCommitment], blob_kzg_proofs: Sequence[KZGProof]) -> None: - # Optionally sanity-check that the KZG commitments match the versioned hashes in the transactions - assert verify_kzg_commitments_against_transactions(execution_payload.transactions, blob_kzg_commitments) + # TODO: can we just remove it? + # assert verify_kzg_commitments_against_transactions(execution_payload.transactions, blob_kzg_commitments) # Optionally sanity-check that the KZG commitments match the blobs (as produced by the execution engine) assert len(blob_kzg_commitments) == len(blobs) == len(blob_kzg_proofs) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index 3ec58b31ed..05e16f1d1f 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -29,10 +29,10 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True, called_new_block = False class TestEngine(spec.NoopExecutionEngine): - def notify_new_payload(self, payload) -> bool: + def notify_new_payload(self, new_payload_request) -> bool: nonlocal called_new_block, execution_valid called_new_block = True - assert payload == execution_payload + assert new_payload_request.execution_payload == execution_payload return execution_valid if not valid: diff --git a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py index 2e1a2a369a..d31955e827 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/phase0/sanity/test_blocks.py @@ -143,7 +143,7 @@ def process_and_sign_block_without_header_validations(spec, state, block): ) if is_post_bellatrix(spec): if spec.is_execution_enabled(state, block.body): - spec.process_execution_payload(state, block.body.execution_payload, spec.EXECUTION_ENGINE) + spec.process_execution_payload(state, block.body, spec.EXECUTION_ENGINE) # Perform rest of process_block transitions spec.process_randao(state, block.body) From 5a6052f46caa540ccceaaf6fab6a2d4c91354dfd Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Mon, 15 May 2023 16:51:52 -0600 Subject: [PATCH 078/117] Update fork-choice.md Stylistic change to be in line with validations in other specifications --- specs/deneb/fork-choice.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index 9faa11077f..ce9973f130 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -30,9 +30,8 @@ This is the modification of the fork choice accompanying the Deneb upgrade. def validate_blobs(expected_kzg_commitments: Sequence[KZGCommitment], blobs: Sequence[Blob], proofs: Sequence[KZGProof]) -> None: - assert len(expected_kzg_commitments) == len(blobs) - assert len(blobs) == len(proofs) - + assert len(expected_kzg_commitments) == len(blobs) == len(proofs) + assert verify_blob_kzg_proof_batch(blobs, expected_kzg_commitments, proofs) ``` From bb38c56ddd0b6c29d472e6eb96b63a44d7c89abc Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 May 2023 20:07:21 +0800 Subject: [PATCH 079/117] Fix `dump_kzg_trusted_setup_files`. Use Fastest BLS lib (#3358) --- tests/core/pyspec/eth2spec/utils/kzg.py | 29 ++++++++++--------------- 1 file changed, 12 insertions(+), 17 deletions(-) diff --git a/tests/core/pyspec/eth2spec/utils/kzg.py b/tests/core/pyspec/eth2spec/utils/kzg.py index 4b5454f66f..cef1faa66d 100644 --- a/tests/core/pyspec/eth2spec/utils/kzg.py +++ b/tests/core/pyspec/eth2spec/utils/kzg.py @@ -10,20 +10,13 @@ from pathlib import Path from eth_utils import encode_hex -from py_ecc.optimized_bls12_381 import ( # noqa: F401 - G1, - G2, - Z1, - Z2, - curve_order as BLS_MODULUS, - add, - multiply, - neg, -) from py_ecc.typing import ( Optimized_Point3D, ) from eth2spec.utils import bls +from eth2spec.utils.bls import ( + BLS_MODULUS, +) PRIMITIVE_ROOT_OF_UNITY = 7 @@ -35,7 +28,7 @@ def generate_setup(generator: Optimized_Point3D, secret: int, length: int) -> Tu """ result = [generator] for _ in range(1, length): - result.append(multiply(result[-1], secret)) + result.append(bls.multiply(result[-1], secret)) return tuple(result) @@ -49,9 +42,9 @@ def fft(vals: Sequence[Optimized_Point3D], modulus: int, domain: int) -> Sequenc R = fft(vals[1::2], modulus, domain[::2]) o = [0] * len(vals) for i, (x, y) in enumerate(zip(L, R)): - y_times_root = multiply(y, domain[i]) - o[i] = add(x, y_times_root) - o[i + len(L)] = add(x, neg(y_times_root)) + y_times_root = bls.multiply(y, domain[i]) + o[i] = bls.add(x, y_times_root) + o[i + len(L)] = bls.add(x, bls.neg(y_times_root)) return o @@ -90,12 +83,14 @@ def get_lagrange(setup: Sequence[Optimized_Point3D]) -> Tuple[bytes]: # TODO: introduce an IFFT function for simplicity fft_output = fft(setup, BLS_MODULUS, domain) inv_length = pow(len(setup), BLS_MODULUS - 2, BLS_MODULUS) - return tuple(bls.G1_to_bytes48(multiply(fft_output[-i], inv_length)) for i in range(len(fft_output))) + return tuple(bls.G1_to_bytes48(bls.multiply(fft_output[-i], inv_length)) for i in range(len(fft_output))) def dump_kzg_trusted_setup_files(secret: int, g1_length: int, g2_length: int, output_dir: str) -> None: - setup_g1 = generate_setup(bls.G1, secret, g1_length) - setup_g2 = generate_setup(bls.G2, secret, g2_length) + bls.use_fastest() + + setup_g1 = generate_setup(bls.G1(), secret, g1_length) + setup_g2 = generate_setup(bls.G2(), secret, g2_length) setup_g1_lagrange = get_lagrange(setup_g1) roots_of_unity = compute_roots_of_unity(g1_length) From 0b2f604f86a037d5b92e82aa79ab8bfc3a55d812 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 May 2023 23:58:01 +0800 Subject: [PATCH 080/117] Fix tests --- specs/_features/eip6110/beacon-chain.md | 4 +- specs/deneb/beacon-chain.md | 5 ++- specs/deneb/validator.md | 6 +-- .../test_process_execution_payload.py | 12 ++--- .../test_process_withdrawals.py | 2 +- .../eth2spec/test/deneb/sanity/test_blocks.py | 45 ++++++++++++------- .../test/deneb/unittests/test_offset.py | 23 ---------- .../unittests/validator/test_validator.py | 6 +-- tests/formats/operations/README.md | 2 +- 9 files changed, 48 insertions(+), 57 deletions(-) delete mode 100644 tests/core/pyspec/eth2spec/test/deneb/unittests/test_offset.py diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index c9dccbdebb..591d8dc869 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -249,7 +249,9 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] - assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload)) + assert execution_engine.notify_new_payload( + NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes) + ) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 52719dcf68..2c4bb61335 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -217,8 +217,11 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid + # [Modified in Deneb] versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] - assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes)) + assert execution_engine.notify_new_payload( + NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes) + ) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index e52ce9eaac..03297c59e2 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -94,13 +94,9 @@ via `get_payload(payload_id).blobs_bundle`. 2. Validate `blobs` and `blob_kzg_commitments`: ```python -def validate_blobs_and_kzg_commitments(execution_payload: ExecutionPayload, - blobs: Sequence[Blob], +def validate_blobs_and_kzg_commitments(blobs: Sequence[Blob], blob_kzg_commitments: Sequence[KZGCommitment], blob_kzg_proofs: Sequence[KZGProof]) -> None: - # TODO: can we just remove it? - # assert verify_kzg_commitments_against_transactions(execution_payload.transactions, blob_kzg_commitments) - # Optionally sanity-check that the KZG commitments match the blobs (as produced by the execution engine) assert len(blob_kzg_commitments) == len(blobs) == len(blob_kzg_proofs) assert verify_blob_kzg_proof_batch(blobs, blob_kzg_commitments, blob_kzg_proofs) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index 05e16f1d1f..b835632354 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -21,10 +21,12 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True, - post-state ('post'). If ``valid == False``, run expecting ``AssertionError`` """ + # Before Deneb, only `body.execution_payload` matters. `BeaconBlockBody` is just a wrapper. + body = spec.BeaconBlockBody(execution_payload=execution_payload) yield 'pre', state yield 'execution', {'execution_valid': execution_valid} - yield 'execution_payload', execution_payload + yield 'body', body called_new_block = False @@ -32,22 +34,22 @@ class TestEngine(spec.NoopExecutionEngine): def notify_new_payload(self, new_payload_request) -> bool: nonlocal called_new_block, execution_valid called_new_block = True - assert new_payload_request.execution_payload == execution_payload + assert new_payload_request.execution_payload == body.execution_payload return execution_valid if not valid: - expect_assertion_error(lambda: spec.process_execution_payload(state, execution_payload, TestEngine())) + expect_assertion_error(lambda: spec.process_execution_payload(state, body, TestEngine())) yield 'post', None return - spec.process_execution_payload(state, execution_payload, TestEngine()) + spec.process_execution_payload(state, body, TestEngine()) # Make sure we called the engine assert called_new_block yield 'post', state - assert state.latest_execution_payload_header == get_execution_payload_header(spec, execution_payload) + assert state.latest_execution_payload_header == get_execution_payload_header(spec, body.execution_payload) def run_success_test(spec, state): diff --git a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py index d7813fb1f2..ad71e2c73c 100644 --- a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py +++ b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_withdrawals.py @@ -56,7 +56,7 @@ def verify_post_state(state, spec, expected_withdrawals, def run_withdrawals_processing(spec, state, execution_payload, num_expected_withdrawals=None, fully_withdrawable_indices=None, partial_withdrawals_indices=None, valid=True): """ - Run ``process_execution_payload``, yielding: + Run ``process_withdrawals``, yielding: - pre-state ('pre') - execution payload ('execution_payload') - post-state ('post'). diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 111565cce2..f27a774abe 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -53,7 +53,10 @@ def test_max_blobs(spec, state): @with_deneb_and_later @spec_state_test -def test_invalid_incorrect_blob_tx_type(spec, state): +def test_incorrect_blob_tx_type(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ yield 'pre', state block = build_empty_block_for_next_slot(spec, state) @@ -62,15 +65,18 @@ def test_invalid_incorrect_blob_tx_type(spec, state): opaque_tx = b'\x04' + opaque_tx[1:] # incorrect tx type block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + signed_block = state_transition_and_sign_block(spec, state, block) yield 'blocks', [signed_block] - yield 'post', None + yield 'post', state @with_deneb_and_later @spec_state_test -def test_invalid_incorrect_transaction_length_1_byte(spec, state): +def test_incorrect_transaction_length_1_byte(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ yield 'pre', state block = build_empty_block_for_next_slot(spec, state) @@ -79,15 +85,18 @@ def test_invalid_incorrect_transaction_length_1_byte(spec, state): opaque_tx = opaque_tx + b'\x12' # incorrect tx length block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + signed_block = state_transition_and_sign_block(spec, state, block) yield 'blocks', [signed_block] - yield 'post', None + yield 'post', state @with_deneb_and_later @spec_state_test -def test_invalid_incorrect_transaction_length_32_bytes(spec, state): +def test_incorrect_transaction_length_32_bytes(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ yield 'pre', state block = build_empty_block_for_next_slot(spec, state) @@ -96,15 +105,18 @@ def test_invalid_incorrect_transaction_length_32_bytes(spec, state): opaque_tx = opaque_tx + b'\x12' * 32 # incorrect tx length block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + signed_block = state_transition_and_sign_block(spec, state, block) yield 'blocks', [signed_block] - yield 'post', None + yield 'post', state @with_deneb_and_later @spec_state_test -def test_invalid_incorrect_commitment(spec, state): +def test_incorrect_commitment(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ yield 'pre', state block = build_empty_block_for_next_slot(spec, state) @@ -113,15 +125,18 @@ def test_invalid_incorrect_commitment(spec, state): block.body.blob_kzg_commitments = blob_kzg_commitments block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + signed_block = state_transition_and_sign_block(spec, state, block) yield 'blocks', [signed_block] - yield 'post', None + yield 'post', state @with_deneb_and_later @spec_state_test -def test_invalid_incorrect_commitments_order(spec, state): +def test_incorrect_commitments_order(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ yield 'pre', state block = build_empty_block_for_next_slot(spec, state) @@ -129,10 +144,10 @@ def test_invalid_incorrect_commitments_order(spec, state): block.body.blob_kzg_commitments = [blob_kzg_commitments[1], blob_kzg_commitments[0]] # incorrect order block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) + signed_block = state_transition_and_sign_block(spec, state, block) yield 'blocks', [signed_block] - yield 'post', None + yield 'post', state @with_deneb_and_later diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_offset.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_offset.py deleted file mode 100644 index 3c3b51ff1a..0000000000 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_offset.py +++ /dev/null @@ -1,23 +0,0 @@ - -from eth2spec.test.helpers.constants import ( - DENEB, - MINIMAL, -) -from eth2spec.test.helpers.sharding import ( - get_sample_opaque_tx, -) -from eth2spec.test.context import ( - with_phases, - spec_state_test, - with_presets, -) - - -@with_phases([DENEB]) -@spec_state_test -@with_presets([MINIMAL]) -def test_tx_peek_blob_versioned_hashes(spec, state): - otx, _, commitments, _ = get_sample_opaque_tx(spec) - data_hashes = spec.tx_peek_blob_versioned_hashes(otx) - expected = [spec.kzg_commitment_to_versioned_hash(blob_commitment) for blob_commitment in commitments] - assert expected == data_hashes diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py index 07039ccfeb..62b8e9dc1d 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py @@ -31,8 +31,7 @@ def test_validate_blobs_and_kzg_commitments(spec, state): block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - spec.validate_blobs_and_kzg_commitments(block.body.execution_payload, - blobs, + spec.validate_blobs_and_kzg_commitments(blobs, blob_kzg_commitments, proofs) @@ -52,7 +51,6 @@ def test_validate_blobs_and_kzg_commitments_missing_blob(spec, state): expect_assertion_error( lambda: spec.validate_blobs_and_kzg_commitments( - block.body.execution_payload, blobs[:-1], blob_kzg_commitments, proofs @@ -75,7 +73,6 @@ def test_validate_blobs_and_kzg_commitments_missing_proof(spec, state): expect_assertion_error( lambda: spec.validate_blobs_and_kzg_commitments( - block.body.execution_payload, blobs, blob_kzg_commitments, proofs[:-1] @@ -100,7 +97,6 @@ def test_validate_blobs_and_kzg_commitments_incorrect_blob(spec, state): expect_assertion_error( lambda: spec.validate_blobs_and_kzg_commitments( - block.body.execution_payload, blobs, blob_kzg_commitments, proofs diff --git a/tests/formats/operations/README.md b/tests/formats/operations/README.md index 245ce85653..ba764879bd 100644 --- a/tests/formats/operations/README.md +++ b/tests/formats/operations/README.md @@ -42,7 +42,7 @@ Operations: | `proposer_slashing` | `ProposerSlashing` | `proposer_slashing` | `process_proposer_slashing(state, proposer_slashing)` | | `voluntary_exit` | `SignedVoluntaryExit` | `voluntary_exit` | `process_voluntary_exit(state, voluntary_exit)` | | `sync_aggregate` | `SyncAggregate` | `sync_aggregate` | `process_sync_aggregate(state, sync_aggregate)` (new in Altair) | -| `execution_payload` | `ExecutionPayload` | `execution_payload` | `process_execution_payload(state, execution_payload)` (new in Bellatrix) | +| `execution_payload` | `BeaconBlockBody` | **`body`** | `process_execution_payload(state, body)` (new in Bellatrix) | | `withdrawals` | `ExecutionPayload` | `execution_payload` | `process_withdrawals(state, execution_payload)` (new in Capella) | | `bls_to_execution_change` | `SignedBLSToExecutionChange` | `address_change` | `process_bls_to_execution_change(state, address_change)` (new in Capella) | | `deposit_receipt` | `DepositReceipt` | `deposit_receipt` | `process_deposit_receipt(state, deposit_receipt)` (new in EIP6110) | From 32358e8fad3425eb23e406413d60744e0f274d40 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Wed, 17 May 2023 11:24:48 -0500 Subject: [PATCH 081/117] Add comment about zero elements in batch verification (#3367) --- specs/deneb/polynomial-commitments.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/polynomial-commitments.md b/specs/deneb/polynomial-commitments.md index e23c31fab8..39758ac888 100644 --- a/specs/deneb/polynomial-commitments.md +++ b/specs/deneb/polynomial-commitments.md @@ -566,7 +566,7 @@ def verify_blob_kzg_proof_batch(blobs: Sequence[Blob], proofs_bytes: Sequence[Bytes48]) -> bool: """ Given a list of blobs and blob KZG proofs, verify that they correspond to the provided commitments. - + Will return True if there are zero blobs/commitments/proofs. Public method. """ From db2e613aac14012234edf94d42cc34b8b5eaa49e Mon Sep 17 00:00:00 2001 From: terencechain Date: Wed, 17 May 2023 11:08:33 -0700 Subject: [PATCH 082/117] Nitpick: blob -> blob sidecar --- specs/deneb/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 3c6f3c88a5..507cd4c0cd 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -226,7 +226,7 @@ No more than `MAX_REQUEST_BLOB_SIDECARS` may be requested at a time. The response MUST consist of zero or more `response_chunk`. Each _successful_ `response_chunk` MUST contain a single `BlobSidecar` payload. -Clients MUST support requesting sidecars since `minimum_request_epoch`, where `minimum_request_epoch = max(finalized_epoch, current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, DENEB_FORK_EPOCH)`. If any root in the request content references a block earlier than `minimum_request_epoch`, peers MAY respond with error code `3: ResourceUnavailable` or not include the blob in the response. +Clients MUST support requesting sidecars since `minimum_request_epoch`, where `minimum_request_epoch = max(finalized_epoch, current_epoch - MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS, DENEB_FORK_EPOCH)`. If any root in the request content references a block earlier than `minimum_request_epoch`, peers MAY respond with error code `3: ResourceUnavailable` or not include the blob sidecar in the response. Clients MUST respond with at least one sidecar, if they have it. Clients MAY limit the number of blocks and sidecars in the response. From 44394ad1216b68dbcb5e00c9239e8b74998ee0de Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 18 May 2023 20:58:24 +0800 Subject: [PATCH 083/117] Fix CircleCI Python version --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5958a2fc69..fcdf483d50 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -157,7 +157,7 @@ jobs: path: tests/core/pyspec/test-reports test-eip6110: docker: - - image: circleci/python:3.8 + - image: circleci/python:3.9 working_directory: ~/specs-repo steps: - restore_cache: From 2f218f83368562abd83fa236e542033e5ebcbafc Mon Sep 17 00:00:00 2001 From: Suphanat Chunhapanya Date: Fri, 5 May 2023 15:09:43 +0700 Subject: [PATCH 084/117] Specify the number of sidecar subnets Previously the number of subnets is equal to MAX_BLOBS_PER_BLOCK which specifies the number of blobs per block. This commit now makes the number of subnets equal to BLOB_SIDECAR_SUBNET_COUNT instead. The advantage of doing this is that we can change MAX_BLOBS_PER_BLOCK without worrying about the p2p network structure and the number of subnets. --- specs/deneb/p2p-interface.md | 8 ++++---- specs/deneb/validator.md | 21 ++++++++++++++++++++- 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 507cd4c0cd..3136f190d2 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -22,7 +22,7 @@ The specification of these changes continues in the same format as the network s - [Topics and messages](#topics-and-messages) - [Global topics](#global-topics) - [`beacon_block`](#beacon_block) - - [`blob_sidecar_{index}`](#blob_sidecar_index) + - [`blob_sidecar_{subnet_id}`](#blob_sidecar_subnet_id) - [Transitioning the gossip](#transitioning-the-gossip) - [The Req/Resp domain](#the-reqresp-domain) - [Messages](#messages) @@ -107,7 +107,7 @@ The new topics along with the type of the `data` field of a gossipsub message ar | Name | Message Type | | - | - | -| `blob_sidecar_{index}` | `SignedBlobSidecar` (new) | +| `blob_sidecar_{subnet_id}` | `SignedBlobSidecar` (new) | ##### Global topics @@ -117,13 +117,13 @@ Deneb introduces new global topics for blob sidecars. The *type* of the payload of this topic changes to the (modified) `SignedBeaconBlock` found in deneb. -###### `blob_sidecar_{index}` +###### `blob_sidecar_{subnet_id}` This topic is used to propagate signed blob sidecars, one for each sidecar index. The number of indices is defined by `MAX_BLOBS_PER_BLOCK`. The following validations MUST pass before forwarding the `signed_blob_sidecar` on the network, assuming the alias `sidecar = signed_blob_sidecar.message`: -- _[REJECT]_ The sidecar is for the correct topic -- i.e. `sidecar.index` matches the topic `{index}`. +- _[REJECT]_ The sidecar is for the correct subnet -- i.e. `compute_subnet_for_blob_sidecar(sidecar.index) == subnet_id`. - _[IGNORE]_ The sidecar is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `sidecar.slot <= current_slot` (a client MAY queue future sidecars for processing at the appropriate slot). - _[IGNORE]_ The sidecar is from a slot greater than the latest finalized slot -- i.e. validate that `sidecar.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)` - _[IGNORE]_ The sidecar's block's parent (defined by `sidecar.block_parent_root`) has been seen (via both gossip and non-gossip sources) (a client MAY queue sidecars for processing once the parent block is retrieved). diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index 6562c91ddd..4bd9ab4a5e 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -10,6 +10,8 @@ - [Introduction](#introduction) - [Prerequisites](#prerequisites) +- [Constants](#constants) + - [Misc](#misc) - [Helpers](#helpers) - [`BlobsBundle`](#blobsbundle) - [Modified `GetPayloadResponse`](#modified-getpayloadresponse) @@ -38,6 +40,14 @@ All behaviors and definitions defined in this document, and documents it extends All terminology, constants, functions, and protocol mechanics defined in the updated [Beacon Chain doc of Deneb](./beacon-chain.md) are requisite for this document and used throughout. Please see related Beacon Chain doc before continuing and use them as a reference throughout. +## Constants + +### Misc + +| Name | Value | Unit | +| - | - | :-: | +| `BLOB_SIDECAR_SUBNET_COUNT` | `4` | The number of blob sidecar subnets used in the gossipsub protocol. | + ## Helpers ### `BlobsBundle` @@ -136,7 +146,7 @@ def get_blob_sidecars(block: BeaconBlock, ``` -Then for each sidecar, `signed_sidecar = SignedBlobSidecar(message=sidecar, signature=signature)` is constructed and published to the `blob_sidecar_{index}` topics according to its index. +Then for each sidecar, `signed_sidecar = SignedBlobSidecar(message=sidecar, signature=signature)` is constructed and published to the associated sidecar topic, the `blob_sidecar_{subnet_id}` pubsub topic. `signature` is obtained from: @@ -149,6 +159,15 @@ def get_blob_sidecar_signature(state: BeaconState, return bls.Sign(privkey, signing_root) ``` +The `subnet_id` for the `signed_sidecar` is calculated with: +- Let `blob_index = signed_sidecar.message.index`. +- Let `subnet_id = compute_subnet_for_blob_sidecar(blob_index)`. + +```python +def compute_subnet_for_blob_sidecar(blob_index: BlobIndex) -> uint64: + return uint64(blob_index % BLOB_SIDECAR_SUBNET_COUNT) +``` + After publishing the peers on the network may request the sidecar through sync-requests, or a local user may be interested. The validator MUST hold on to sidecars for `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` epochs and serve when capable, From 08a13261c209ca2ebb7d099b4cc6240a7b6ad8b7 Mon Sep 17 00:00:00 2001 From: Suphanat Chunhapanya Date: Tue, 9 May 2023 21:03:59 +0700 Subject: [PATCH 085/117] Use SubnetID instead of uint64 --- specs/deneb/validator.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index 4bd9ab4a5e..4303c90b5c 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -164,8 +164,8 @@ The `subnet_id` for the `signed_sidecar` is calculated with: - Let `subnet_id = compute_subnet_for_blob_sidecar(blob_index)`. ```python -def compute_subnet_for_blob_sidecar(blob_index: BlobIndex) -> uint64: - return uint64(blob_index % BLOB_SIDECAR_SUBNET_COUNT) +def compute_subnet_for_blob_sidecar(blob_index: BlobIndex) -> SubnetID: + return SubnetID(blob_index % BLOB_SIDECAR_SUBNET_COUNT) ``` After publishing the peers on the network may request the sidecar through sync-requests, or a local user may be interested. From 7097dcf27ae357a57c107f6939d1b96e202665f4 Mon Sep 17 00:00:00 2001 From: Alex Stokes Date: Thu, 18 May 2023 15:27:47 -0600 Subject: [PATCH 086/117] Clarify blob subnets --- specs/deneb/p2p-interface.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 3136f190d2..d67c144b21 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -119,7 +119,7 @@ The *type* of the payload of this topic changes to the (modified) `SignedBeaconB ###### `blob_sidecar_{subnet_id}` -This topic is used to propagate signed blob sidecars, one for each sidecar index. The number of indices is defined by `MAX_BLOBS_PER_BLOCK`. +This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. The following validations MUST pass before forwarding the `signed_blob_sidecar` on the network, assuming the alias `sidecar = signed_blob_sidecar.message`: From a7f8659725ad4372ae7f5be970c235b2e4f6dfbc Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 19 May 2023 17:17:24 +0300 Subject: [PATCH 087/117] KZG test vector format also uses big endian now --- tests/formats/kzg/compute_kzg_proof.md | 4 ++-- tests/formats/kzg/verify_kzg_proof.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/formats/kzg/compute_kzg_proof.md b/tests/formats/kzg/compute_kzg_proof.md index 0713d50d81..b10105129b 100644 --- a/tests/formats/kzg/compute_kzg_proof.md +++ b/tests/formats/kzg/compute_kzg_proof.md @@ -14,8 +14,8 @@ output: Tuple[KZGProof, Bytes32] -- The KZG proof and the value y = f(z) ``` - `blob` here is encoded as a string: hexadecimal encoding of `4096 * 32 = 131072` bytes, prefixed with `0x`. -- `z` here is encoded as a string: hexadecimal encoding of `32` bytes representing a little endian encoded field element, prefixed with `0x`. -- `y` here is encoded as a string: hexadecimal encoding of `32` bytes representing a little endian encoded field element, prefixed with `0x`. +- `z` here is encoded as a string: hexadecimal encoding of `32` bytes representing a big endian encoded field element, prefixed with `0x`. +- `y` here is encoded as a string: hexadecimal encoding of `32` bytes representing a big endian encoded field element, prefixed with `0x`. All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. diff --git a/tests/formats/kzg/verify_kzg_proof.md b/tests/formats/kzg/verify_kzg_proof.md index 143466b66f..18e02710c5 100644 --- a/tests/formats/kzg/verify_kzg_proof.md +++ b/tests/formats/kzg/verify_kzg_proof.md @@ -15,8 +15,8 @@ input: output: bool -- true (valid proof) or false (incorrect proof) ``` -- `z` here is encoded as a string: hexadecimal encoding of `32` bytes representing a little endian encoded field element, prefixed with `0x`. -- `y` here is encoded as a string: hexadecimal encoding of `32` bytes representing a little endian encoded field element, prefixed with `0x`. +- `z` here is encoded as a string: hexadecimal encoding of `32` bytes representing a big endian encoded field element, prefixed with `0x`. +- `y` here is encoded as a string: hexadecimal encoding of `32` bytes representing a big endian encoded field element, prefixed with `0x`. All byte(s) fields are encoded as strings, hexadecimal encoding, prefixed with `0x`. From f87e1436bfbc1f51a18aedc337c6013cd3b3305e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 16 May 2023 23:36:52 +0800 Subject: [PATCH 088/117] Add networking configs to config files --- configs/mainnet.yaml | 28 ++++++++ configs/minimal.yaml | 28 ++++++++ presets/mainnet/phase0.yaml | 5 ++ presets/minimal/phase0.yaml | 5 ++ setup.py | 1 + specs/phase0/p2p-interface.md | 72 +++++++++++++++++-- specs/phase0/validator.md | 45 ------------ .../unittests/test_config_invariants.py | 8 ++- .../validator/test_validator_unittest.py | 4 +- 9 files changed, 141 insertions(+), 55 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 5ad394c082..1a1537b26d 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -94,3 +94,31 @@ PROPOSER_SCORE_BOOST: 40 DEPOSIT_CHAIN_ID: 1 DEPOSIT_NETWORK_ID: 1 DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa + + +# Networking +# --------------------------------------------------------------- +# `2**20` (= 1048576, 1 MiB) +GOSSIP_MAX_SIZE: 1048576 +# `2**10` (= 1024) +MAX_REQUEST_BLOCKS: 1024 +# `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) +MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 +# `2**20` (=1048576, 1 MiB) +MAX_CHUNK_SIZE: 1048576 +# 5s +TTFB_TIMEOUT: 5 +# 10s +RESP_TIMEOUT: 10 +ATTESTATION_PROPAGATION_SLOT_RANGE: 32 +# 500ms +MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 +MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 +MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 +# 2 subnets per node +SUBNETS_PER_NODE: 2 +# 2**8 (= 64) +ATTESTATION_SUBNET_COUNT: 64 +ATTESTATION_SUBNET_EXTRA_BITS: 0 +# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS +ATTESTATION_SUBNET_PREFIX_BITS: 6 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 5895cfc707..32d5682a82 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -95,3 +95,31 @@ DEPOSIT_CHAIN_ID: 5 DEPOSIT_NETWORK_ID: 5 # Configured on a per testnet basis DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890 + + +# Networking +# --------------------------------------------------------------- +# `2**20` (= 1048576, 1 MiB) +GOSSIP_MAX_SIZE: 1048576 +# `2**10` (= 1024) +MAX_REQUEST_BLOCKS: 1024 +# [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272) +MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272 +# `2**20` (=1048576, 1 MiB) +MAX_CHUNK_SIZE: 1048576 +# 5s +TTFB_TIMEOUT: 5 +# 10s +RESP_TIMEOUT: 10 +ATTESTATION_PROPAGATION_SLOT_RANGE: 32 +# 500ms +MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 +MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 +MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 +# 2 subnets per node +SUBNETS_PER_NODE: 2 +# 2**8 (= 64) +ATTESTATION_SUBNET_COUNT: 64 +ATTESTATION_SUBNET_EXTRA_BITS: 0 +# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS +ATTESTATION_SUBNET_PREFIX_BITS: 6 diff --git a/presets/mainnet/phase0.yaml b/presets/mainnet/phase0.yaml index 02bc96c8cd..c84a935954 100644 --- a/presets/mainnet/phase0.yaml +++ b/presets/mainnet/phase0.yaml @@ -86,3 +86,8 @@ MAX_ATTESTATIONS: 128 MAX_DEPOSITS: 16 # 2**4 (= 16) MAX_VOLUNTARY_EXITS: 16 + +# Networking +# --------------------------------------------------------------- +# 2**8 (= 256) +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 diff --git a/presets/minimal/phase0.yaml b/presets/minimal/phase0.yaml index e7028f5a42..c55f7de7b7 100644 --- a/presets/minimal/phase0.yaml +++ b/presets/minimal/phase0.yaml @@ -86,3 +86,8 @@ MAX_ATTESTATIONS: 128 MAX_DEPOSITS: 16 # 2**4 (= 16) MAX_VOLUNTARY_EXITS: 16 + +# Networking +# --------------------------------------------------------------- +# 2**8 (= 256) +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 diff --git a/setup.py b/setup.py index f053412b5c..4f338f27fe 100644 --- a/setup.py +++ b/setup.py @@ -988,6 +988,7 @@ def finalize_options(self): specs/phase0/fork-choice.md specs/phase0/validator.md specs/phase0/weak-subjectivity.md + specs/phase0/p2p-interface.md """ if self.spec_fork in (ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110): self.md_doc_paths += """ diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 2503d906c5..3b8a91cc94 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -20,6 +20,9 @@ It consists of four main sections: - [Protocol Negotiation](#protocol-negotiation) - [Multiplexing](#multiplexing) - [Consensus-layer network interaction domains](#consensus-layer-network-interaction-domains) + - [Custom types](#custom-types) + - [Constants](#constants) + - [Preset](#preset) - [Configuration](#configuration) - [MetaData](#metadata) - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) @@ -53,6 +56,7 @@ It consists of four main sections: - [ENR structure](#enr-structure) - [Attestation subnet bitfield](#attestation-subnet-bitfield) - [`eth2` field](#eth2-field) + - [Attestation subnet subcription](#attestation-subnet-subcription) - [Design decision rationale](#design-decision-rationale) - [Transport](#transport-1) - [Why are we defining specific transports?](#why-are-we-defining-specific-transports) @@ -165,22 +169,46 @@ See the [Rationale](#design-decision-rationale) section below for tradeoffs. ## Consensus-layer network interaction domains +### Custom types + +We define the following Python custom types for type hinting and readability: + +| Name | SSZ equivalent | Description | +| - | - | - | +| `NodeID` | `uint256` | node identifier | +| `SubnetID` | `uint64` | subnet identifier | + +### Constants + +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `NODE_ID_BITS` | `256` | The bit length of uint256 is 256 | + +### Preset +| Name | Value | Unit | Duration | +| - | - | :-: | :-: | +| `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | + ### Configuration -This section outlines constants that are used in this spec. +This section outlines configurations that are used in this spec. | Name | Value | Description | |---|---|---| | `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum allowed size of uncompressed gossip messages. | | `MAX_REQUEST_BLOCKS` | `2**10` (= 1024) | Maximum number of blocks in a single request | | `MIN_EPOCHS_FOR_BLOCK_REQUESTS` | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) | The minimum epoch range over which a node must serve blocks | -| `MAX_CHUNK_SIZE` | `2**20` (1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | -| `TTFB_TIMEOUT` | `5s` | The maximum time to wait for first byte of request response (time-to-first-byte). | -| `RESP_TIMEOUT` | `10s` | The maximum time for complete response transfer. | +| `MAX_CHUNK_SIZE` | `2**20` (=1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | +| `TTFB_TIMEOUT` | `5` | The maximum duration in **seconds** to wait for first byte of request response (time-to-first-byte). | +| `RESP_TIMEOUT` | `10` | The maximum duration in **seconds** for complete response transfer. | | `ATTESTATION_PROPAGATION_SLOT_RANGE` | `32` | The maximum number of slots during which an attestation can be propagated. | -| `MAXIMUM_GOSSIP_CLOCK_DISPARITY` | `500ms` | The maximum milliseconds of clock disparity assumed between honest nodes. | -| `MESSAGE_DOMAIN_INVALID_SNAPPY` | `0x00000000` | 4-byte domain for gossip message-id isolation of *invalid* snappy messages | -| `MESSAGE_DOMAIN_VALID_SNAPPY` | `0x01000000` | 4-byte domain for gossip message-id isolation of *valid* snappy messages | +| `MAXIMUM_GOSSIP_CLOCK_DISPARITY` | `500` | The maximum **milliseconds** of clock disparity assumed between honest nodes. | +| `MESSAGE_DOMAIN_INVALID_SNAPPY` | `DomainType('0x00000000')` | 4-byte domain for gossip message-id isolation of *invalid* snappy messages | +| `MESSAGE_DOMAIN_VALID_SNAPPY` | `DomainType('0x01000000')` | 4-byte domain for gossip message-id isolation of *valid* snappy messages | +| `SUBNETS_PER_NODE` | `2` | The number of long-lived subnets a beacon node should be subscribed to. | +| `ATTESTATION_SUBNET_COUNT` | `2**6` (= 64) | The number of attestation subnets used in the gossipsub protocol. | +| `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | +| `ATTESTATION_SUBNET_PREFIX_BITS` | `int(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | | ### MetaData @@ -979,6 +1007,34 @@ Clients MAY connect to peers with the same `fork_digest` but a different `next_f Unless `ENRForkID` is manually updated to matching prior to the earlier `next_fork_epoch` of the two clients, these connecting clients will be unable to successfully interact starting at the earlier `next_fork_epoch`. +### Attestation subnet subcription + +Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each beacon node should: + +* Remain subscribed to `SUBNETS_PER_NODE` for `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs. +* Maintain advertisement of the selected subnets in their node's ENR `attnets` entry by setting the selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets. +* Select these subnets based on their node-id as specified by the following `compute_subscribed_subnets(node_id, epoch)` function. + +```python +def compute_subscribed_subnet(node_id: NodeID, epoch: Epoch, index: int) -> SubnetID: + node_id_prefix = node_id >> (NODE_ID_BITS - ATTESTATION_SUBNET_PREFIX_BITS) + node_offset = node_id % EPOCHS_PER_SUBNET_SUBSCRIPTION + permutation_seed = hash(uint_to_bytes(uint64((epoch + node_offset) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) + permutated_prefix = compute_shuffled_index( + node_id_prefix, + 1 << ATTESTATION_SUBNET_PREFIX_BITS, + permutation_seed, + ) + return SubnetID((permutated_prefix + index) % ATTESTATION_SUBNET_COUNT) +``` + +```python +def compute_subscribed_subnets(node_id: NodeID, epoch: Epoch) -> Sequence[SubnetID]: + return [compute_subscribed_subnet(node_id, epoch, index) for index in range(SUBNETS_PER_NODE)] +``` + +*Note*: When preparing for a hard fork, a node must select and subscribe to subnets of the future fork versioning at least `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. + ## Design decision rationale ### Transport @@ -1438,6 +1494,8 @@ the epoch range that a new node syncing from a checkpoint must backfill. [weak subjectivity guide](./weak-subjectivity.md). Specifically to find this max epoch range, we use the worst case event of a very large validator size (`>= MIN_PER_EPOCH_CHURN_LIMIT * CHURN_LIMIT_QUOTIENT`). +[0]: # (eth2spec: skip) + ```python MIN_EPOCHS_FOR_BLOCK_REQUESTS = ( MIN_VALIDATOR_WITHDRAWABILITY_DELAY diff --git a/specs/phase0/validator.md b/specs/phase0/validator.md index b0a9ac5077..602df09736 100644 --- a/specs/phase0/validator.md +++ b/specs/phase0/validator.md @@ -10,7 +10,6 @@ This is an accompanying document to [Phase 0 -- The Beacon Chain](./beacon-chain - [Introduction](#introduction) - [Prerequisites](#prerequisites) -- [Custom types](#custom-types) - [Constants](#constants) - [Misc](#misc) - [Containers](#containers) @@ -64,7 +63,6 @@ This is an accompanying document to [Phase 0 -- The Beacon Chain](./beacon-chain - [Aggregation bits](#aggregation-bits-1) - [Aggregate signature](#aggregate-signature-1) - [Broadcast aggregate](#broadcast-aggregate) -- [Phase 0 attestation subnet stability](#phase-0-attestation-subnet-stability) - [How to avoid slashing](#how-to-avoid-slashing) - [Proposer slashing](#proposer-slashing) - [Attester slashing](#attester-slashing) @@ -83,15 +81,6 @@ A validator is an entity that participates in the consensus of the Ethereum proo All terminology, constants, functions, and protocol mechanics defined in the [Phase 0 -- The Beacon Chain](./beacon-chain.md) and [Phase 0 -- Deposit Contract](./deposit-contract.md) doc are requisite for this document and used throughout. Please see the Phase 0 doc before continuing and use as a reference throughout. -## Custom types - -We define the following Python custom types for type hinting and readability: - -| Name | SSZ equivalent | Description | -| - | - | - | -| `NodeID` | `uint256` | node identifier | -| `SubnetID` | `uint64` | subnet identifier | - ## Constants ### Misc @@ -99,12 +88,6 @@ We define the following Python custom types for type hinting and readability: | Name | Value | Unit | Duration | | - | - | :-: | :-: | | `TARGET_AGGREGATORS_PER_COMMITTEE` | `2**4` (= 16) | validators | -| `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | -| `ATTESTATION_SUBNET_COUNT` | `64` | The number of attestation subnets used in the gossipsub protocol. | -| `ATTESTATION_SUBNET_EXTRA_BITS` | `0` | The number of extra bits of a NodeId to use when mapping to a subscribed subnet | -| `SUBNETS_PER_NODE` | `2` | The number of long-lived subnets a beacon node should be subscribed to. | -| `ATTESTATION_SUBNET_PREFIX_BITS` | `(ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS)` | | -| `NODE_ID_BITS` | `256` | The bit length of uint256 is 256 | ## Containers @@ -619,34 +602,6 @@ def get_aggregate_and_proof_signature(state: BeaconState, return bls.Sign(privkey, signing_root) ``` -## Phase 0 attestation subnet stability - -Because Phase 0 does not have shards and thus does not have Shard Committees, there is no stable backbone to the attestation subnets (`beacon_attestation_{subnet_id}`). To provide this stability, each beacon node should: - -* Remain subscribed to `SUBNETS_PER_NODE` for `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs. -* Maintain advertisement of the selected subnets in their node's ENR `attnets` entry by setting the selected `subnet_id` bits to `True` (e.g. `ENR["attnets"][subnet_id] = True`) for all persistent attestation subnets. -* Select these subnets based on their node-id as specified by the following `compute_subscribed_subnets(node_id, epoch)` function. - -```python -def compute_subscribed_subnet(node_id: NodeID, epoch: Epoch, index: int) -> SubnetID: - node_id_prefix = node_id >> (NODE_ID_BITS - int(ATTESTATION_SUBNET_PREFIX_BITS)) - node_offset = node_id % EPOCHS_PER_SUBNET_SUBSCRIPTION - permutation_seed = hash(uint_to_bytes(uint64((epoch + node_offset) // EPOCHS_PER_SUBNET_SUBSCRIPTION))) - permutated_prefix = compute_shuffled_index( - node_id_prefix, - 1 << int(ATTESTATION_SUBNET_PREFIX_BITS), - permutation_seed, - ) - return SubnetID((permutated_prefix + index) % ATTESTATION_SUBNET_COUNT) -``` - -```python -def compute_subscribed_subnets(node_id: NodeID, epoch: Epoch) -> Sequence[SubnetID]: - return [compute_subscribed_subnet(node_id, epoch, index) for index in range(SUBNETS_PER_NODE)] -``` - -*Note*: When preparing for a hard fork, a validator must select and subscribe to subnets of the future fork versioning at least `EPOCHS_PER_SUBNET_SUBSCRIPTION` epochs in advance of the fork. These new subnets for the fork are maintained in addition to those for the current fork until the fork occurs. After the fork occurs, let the subnets from the previous fork reach the end of life with no replacements. - ## How to avoid slashing "Slashing" is the burning of some amount of validator funds and immediate ejection from the active validator set. In Phase 0, there are two ways in which funds can be slashed: [proposer slashing](#proposer-slashing) and [attester slashing](#attester-slashing). Although being slashed has serious repercussions, it is simple enough to avoid being slashed all together by remaining _consistent_ with respect to the messages a validator has previously signed. diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py index b0fd06374d..6216ea2f00 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/test_config_invariants.py @@ -75,7 +75,13 @@ def test_time(spec, state): @with_all_phases @spec_state_test def test_networking(spec, state): - assert spec.SUBNETS_PER_NODE <= spec.ATTESTATION_SUBNET_COUNT + assert spec.config.MIN_EPOCHS_FOR_BLOCK_REQUESTS == ( + spec.config.MIN_VALIDATOR_WITHDRAWABILITY_DELAY + spec.config.CHURN_LIMIT_QUOTIENT // 2 + ) + assert spec.config.ATTESTATION_SUBNET_PREFIX_BITS == ( + spec.ceillog2(spec.config.ATTESTATION_SUBNET_COUNT) + spec.config.ATTESTATION_SUBNET_EXTRA_BITS + ) + assert spec.config.SUBNETS_PER_NODE <= spec.config.ATTESTATION_SUBNET_COUNT node_id_length = spec.NodeID(1).type_byte_length() # in bytes assert node_id_length * 8 == spec.NODE_ID_BITS # in bits diff --git a/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py b/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py index 177748eacd..918ab96e2e 100644 --- a/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py +++ b/tests/core/pyspec/eth2spec/test/phase0/unittests/validator/test_validator_unittest.py @@ -371,7 +371,7 @@ def test_compute_subnet_for_attestation(spec, state): slots_since_epoch_start = slot % spec.SLOTS_PER_EPOCH committees_since_epoch_start = committees_per_slot * slots_since_epoch_start - expected_subnet_id = (committees_since_epoch_start + committee_idx) % spec.ATTESTATION_SUBNET_COUNT + expected_subnet_id = (committees_since_epoch_start + committee_idx) % spec.config.ATTESTATION_SUBNET_COUNT assert actual_subnet_id == expected_subnet_id @@ -488,7 +488,7 @@ def run_compute_subscribed_subnets_arguments(spec, rng=random.Random(1111)): node_id = rng.randint(0, 2**40 - 1) # try VALIDATOR_REGISTRY_LIMIT epoch = rng.randint(0, 2**64 - 1) subnets = spec.compute_subscribed_subnets(node_id, epoch) - assert len(subnets) == spec.SUBNETS_PER_NODE + assert len(subnets) == spec.config.SUBNETS_PER_NODE @with_all_phases From 92324ca845dee97b3c817d075352b85fa7a1e9c7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Sat, 20 May 2023 01:14:10 +0800 Subject: [PATCH 089/117] Remove `is_merge_transition_complete` check from Capella --- specs/_features/eip4788/validator.md | 19 ++----------------- specs/_features/eip6110/beacon-chain.md | 3 +-- specs/_features/sharding/beacon-chain.md | 3 +-- specs/bellatrix/validator.md | 5 +++-- specs/capella/beacon-chain.md | 6 +++--- specs/capella/validator.md | 19 ++----------------- specs/deneb/beacon-chain.md | 3 +-- .../unittests/validator/test_validator.py | 11 +++++++---- 8 files changed, 20 insertions(+), 49 deletions(-) diff --git a/specs/_features/eip4788/validator.md b/specs/_features/eip4788/validator.md index 421e297ce9..f24c973539 100644 --- a/specs/_features/eip4788/validator.md +++ b/specs/_features/eip4788/validator.md @@ -64,27 +64,12 @@ parameter to the `PayloadAttributes`. ```python def prepare_execution_payload(state: BeaconState, - pow_chain: Dict[Hash32, PowBlock], safe_block_hash: Hash32, finalized_block_hash: Hash32, suggested_fee_recipient: ExecutionAddress, execution_engine: ExecutionEngine) -> Optional[PayloadId]: - if not is_merge_transition_complete(state): - is_terminal_block_hash_set = TERMINAL_BLOCK_HASH != Hash32() - is_activation_epoch_reached = get_current_epoch(state) >= TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH - if is_terminal_block_hash_set and not is_activation_epoch_reached: - # Terminal block hash is set but activation epoch is not yet reached, no prepare payload call is needed - return None - - terminal_pow_block = get_terminal_pow_block(pow_chain) - if terminal_pow_block is None: - # Pre-merge, no prepare payload call is needed - return None - # Signify merge via producing on top of the terminal PoW block - parent_hash = terminal_pow_block.block_hash - else: - # Post-merge, normal payload - parent_hash = state.latest_execution_payload_header.block_hash + # Verify consistency of the parent hash with respect to the previous execution payload header + parent_hash = state.latest_execution_payload_header.block_hash # Set the forkchoice head and initiate the payload build process payload_attributes = PayloadAttributes( diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index 4d69fb4e05..04ba5d3a76 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -238,8 +238,7 @@ def process_deposit_receipt(state: BeaconState, deposit_receipt: DepositReceipt) ```python def process_execution_payload(state: BeaconState, payload: ExecutionPayload, execution_engine: ExecutionEngine) -> None: # Verify consistency of the parent hash with respect to the previous execution payload header - if is_merge_transition_complete(state): - assert payload.parent_hash == state.latest_execution_payload_header.block_hash + assert payload.parent_hash == state.latest_execution_payload_header.block_hash # Verify prev_randao assert payload.prev_randao == get_randao_mix(state, get_current_epoch(state)) # Verify timestamp diff --git a/specs/_features/sharding/beacon-chain.md b/specs/_features/sharding/beacon-chain.md index f7de7af656..e67bc09ed6 100644 --- a/specs/_features/sharding/beacon-chain.md +++ b/specs/_features/sharding/beacon-chain.md @@ -370,8 +370,7 @@ def process_execution_payload(state: BeaconState, block: BeaconBlock, execution_ payload = block.body.payload_data.value.execution_payload # Verify consistency of the parent hash with respect to the previous execution payload header - if is_merge_transition_complete(state): - assert payload.parent_hash == state.latest_execution_payload_header.block_hash + assert payload.parent_hash == state.latest_execution_payload_header.block_hash # Verify random assert payload.random == get_randao_mix(state, get_current_epoch(state)) # Verify timestamp diff --git a/specs/bellatrix/validator.md b/specs/bellatrix/validator.md index a176d7534e..e6fd5443a7 100644 --- a/specs/bellatrix/validator.md +++ b/specs/bellatrix/validator.md @@ -118,12 +118,13 @@ To obtain an execution payload, a block proposer building a block on top of a `s ```python def prepare_execution_payload(state: BeaconState, - pow_chain: Dict[Hash32, PowBlock], safe_block_hash: Hash32, finalized_block_hash: Hash32, suggested_fee_recipient: ExecutionAddress, - execution_engine: ExecutionEngine) -> Optional[PayloadId]: + execution_engine: ExecutionEngine, + pow_chain: Optional[Dict[Hash32, PowBlock]]=None) -> Optional[PayloadId]: if not is_merge_transition_complete(state): + assert pow_chain is not None is_terminal_block_hash_set = TERMINAL_BLOCK_HASH != Hash32() is_activation_epoch_reached = get_current_epoch(state) >= TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH if is_terminal_block_hash_set and not is_activation_epoch_reached: diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index 72260108f9..3ea54ab124 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -331,7 +331,7 @@ def process_historical_summaries_update(state: BeaconState) -> None: ```python def process_block(state: BeaconState, block: BeaconBlock) -> None: process_block_header(state, block) - # Removed `is_execution_enabled` check + # [Modified in Capella] Removed `is_execution_enabled` check in Capella process_withdrawals(state, block.body.execution_payload) # [New in Capella] process_execution_payload(state, block.body.execution_payload, EXECUTION_ENGINE) # [Modified in Capella] process_randao(state, block.body) @@ -408,9 +408,9 @@ def process_withdrawals(state: BeaconState, payload: ExecutionPayload) -> None: ```python def process_execution_payload(state: BeaconState, payload: ExecutionPayload, execution_engine: ExecutionEngine) -> None: + # [Modified in Capella] Removed `is_merge_transition_complete` check in Capella # Verify consistency of the parent hash with respect to the previous execution payload header - if is_merge_transition_complete(state): - assert payload.parent_hash == state.latest_execution_payload_header.block_hash + assert payload.parent_hash == state.latest_execution_payload_header.block_hash # Verify prev_randao assert payload.prev_randao == get_randao_mix(state, get_current_epoch(state)) # Verify timestamp diff --git a/specs/capella/validator.md b/specs/capella/validator.md index 644ee476f9..12c93b6b0b 100644 --- a/specs/capella/validator.md +++ b/specs/capella/validator.md @@ -69,27 +69,12 @@ That is, `state` is the `previous_state` processed through any empty slots up to ```python def prepare_execution_payload(state: BeaconState, - pow_chain: Dict[Hash32, PowBlock], safe_block_hash: Hash32, finalized_block_hash: Hash32, suggested_fee_recipient: ExecutionAddress, execution_engine: ExecutionEngine) -> Optional[PayloadId]: - if not is_merge_transition_complete(state): - is_terminal_block_hash_set = TERMINAL_BLOCK_HASH != Hash32() - is_activation_epoch_reached = get_current_epoch(state) >= TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH - if is_terminal_block_hash_set and not is_activation_epoch_reached: - # Terminal block hash is set but activation epoch is not yet reached, no prepare payload call is needed - return None - - terminal_pow_block = get_terminal_pow_block(pow_chain) - if terminal_pow_block is None: - # Pre-merge, no prepare payload call is needed - return None - # Signify merge via producing on top of the terminal PoW block - parent_hash = terminal_pow_block.block_hash - else: - # Post-merge, normal payload - parent_hash = state.latest_execution_payload_header.block_hash + # [Modified in Capella] Removed `is_merge_transition_complete` check in Capella + parent_hash = state.latest_execution_payload_header.block_hash # Set the forkchoice head and initiate the payload build process payload_attributes = PayloadAttributes( diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 6aaa2567f6..5834d5a379 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -216,8 +216,7 @@ def process_block(state: BeaconState, block: BeaconBlock) -> None: ```python def process_execution_payload(state: BeaconState, payload: ExecutionPayload, execution_engine: ExecutionEngine) -> None: # Verify consistency of the parent hash with respect to the previous execution payload header - if is_merge_transition_complete(state): - assert payload.parent_hash == state.latest_execution_payload_header.block_hash + assert payload.parent_hash == state.latest_execution_payload_header.block_hash # Verify prev_randao assert payload.prev_randao == get_randao_mix(state, get_current_epoch(state)) # Verify timestamp diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/bellatrix/unittests/validator/test_validator.py index 6398aedd84..c55b6e3c3a 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/unittests/validator/test_validator.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/unittests/validator/test_validator.py @@ -4,9 +4,12 @@ from eth2spec.test.helpers.pow_block import ( prepare_random_pow_chain, ) +from eth2spec.test.helpers.constants import ( + BELLATRIX, +) from eth2spec.test.context import ( spec_state_test, - with_bellatrix_and_later, + with_phases, ) @@ -30,7 +33,7 @@ # it would return the first block (IS_HEAD_PARENT_BLOCK). -@with_bellatrix_and_later +@with_phases([BELLATRIX]) @spec_state_test def test_get_pow_block_at_terminal_total_difficulty(spec, state): for result in expected_results: @@ -90,7 +93,7 @@ def test_get_pow_block_at_terminal_total_difficulty(spec, state): ] -@with_bellatrix_and_later +@with_phases([BELLATRIX]) @spec_state_test def test_prepare_execution_payload(spec, state): for result in prepare_execution_payload_expected_results: @@ -157,11 +160,11 @@ def notify_forkchoice_updated(self, payload_id = spec.prepare_execution_payload( state=state, - pow_chain=pow_chain.to_dict(), safe_block_hash=safe_block_hash, finalized_block_hash=finalized_block_hash, suggested_fee_recipient=suggested_fee_recipient, execution_engine=TestEngine(), + pow_chain=pow_chain.to_dict(), ) assert payload_id == result_payload_id From 17dac8cab92d2982d7940966d0de40c5e3283828 Mon Sep 17 00:00:00 2001 From: gajinder Date: Tue, 25 Apr 2023 15:36:48 +0530 Subject: [PATCH 090/117] Update MAX_BLOBS_PER_BLOCK to a higher bound --- presets/mainnet/deneb.yaml | 4 ++-- specs/deneb/beacon-chain.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/presets/mainnet/deneb.yaml b/presets/mainnet/deneb.yaml index ebe33f2d10..7cf41bbf4a 100644 --- a/presets/mainnet/deneb.yaml +++ b/presets/mainnet/deneb.yaml @@ -4,5 +4,5 @@ # --------------------------------------------------------------- # `uint64(4096)` FIELD_ELEMENTS_PER_BLOB: 4096 -# `uint64(2**2)` (= 4) -MAX_BLOBS_PER_BLOCK: 4 +# `uint64(2**10)` (= 1024) +MAX_BLOBS_PER_BLOCK: 1024 diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 359c7fc95b..a123f921be 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -68,7 +68,7 @@ This upgrade adds blobs to the beacon chain as part of Deneb. This is an extensi | Name | Value | | - | - | -| `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | +| `MAX_BLOBS_PER_BLOCK` | `uint64(2**10)` (= 1024) | ## Configuration From 5e43f43df135a576cbdff51fa3aec6d85a08cf24 Mon Sep 17 00:00:00 2001 From: gajinder Date: Fri, 28 Apr 2023 10:55:35 +0530 Subject: [PATCH 091/117] update limit to 4844 friendly 16 blobs --- presets/mainnet/deneb.yaml | 4 ++-- specs/deneb/beacon-chain.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/presets/mainnet/deneb.yaml b/presets/mainnet/deneb.yaml index 7cf41bbf4a..d5dafe0d04 100644 --- a/presets/mainnet/deneb.yaml +++ b/presets/mainnet/deneb.yaml @@ -4,5 +4,5 @@ # --------------------------------------------------------------- # `uint64(4096)` FIELD_ELEMENTS_PER_BLOB: 4096 -# `uint64(2**10)` (= 1024) -MAX_BLOBS_PER_BLOCK: 1024 +# `uint64(2**4)` (= 16) +MAX_BLOBS_PER_BLOCK: 16 diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index a123f921be..fc59e3be81 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -68,7 +68,7 @@ This upgrade adds blobs to the beacon chain as part of Deneb. This is an extensi | Name | Value | | - | - | -| `MAX_BLOBS_PER_BLOCK` | `uint64(2**10)` (= 1024) | +| `MAX_BLOBS_PER_BLOCK` | `uint64(2**4)` (= 16) | ## Configuration From 1aad9b5fa0dce3a0043710bd6595646260459fb2 Mon Sep 17 00:00:00 2001 From: gajinder Date: Sat, 6 May 2023 17:19:28 +0530 Subject: [PATCH 092/117] adding a fixed theoretical limit for commitments in a block --- specs/deneb/beacon-chain.md | 4 ++-- specs/deneb/p2p-interface.md | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index fc59e3be81..934a840132 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -68,7 +68,7 @@ This upgrade adds blobs to the beacon chain as part of Deneb. This is an extensi | Name | Value | | - | - | -| `MAX_BLOBS_PER_BLOCK` | `uint64(2**4)` (= 16) | +| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**10)` (= 1024) | hardfork independent fixed theoretical limit | ## Configuration @@ -96,7 +96,7 @@ class BeaconBlockBody(Container): # Execution execution_payload: ExecutionPayload # [Modified in Deneb] bls_to_execution_changes: List[SignedBLSToExecutionChange, MAX_BLS_TO_EXECUTION_CHANGES] - blob_kzg_commitments: List[KZGCommitment, MAX_BLOBS_PER_BLOCK] # [New in Deneb] + blob_kzg_commitments: List[KZGCommitment, MAX_BLOB_COMMITMENTS_PER_BLOCK] # [New in Deneb] ``` #### `ExecutionPayload` diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index d67c144b21..514991bb7e 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -43,6 +43,7 @@ The specification of these changes continues in the same format as the network s | Name | Value | Description | |------------------------------------------|-----------------------------------|---------------------------------------------------------------------| | `MAX_REQUEST_BLOCKS_DENEB` | `2**7` (= 128) | Maximum number of blocks in a single request | +| `MAX_BLOBS_PER_BLOCK` | `2**4` (= 16) | Maximum number of blobs in a single request, to be always <= `MAX_BLOB_COMMITMENTS_PER_BLOCK` | | `MAX_REQUEST_BLOB_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK` | Maximum number of blob sidecars in a single request | | `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars | @@ -119,7 +120,7 @@ The *type* of the payload of this topic changes to the (modified) `SignedBeaconB ###### `blob_sidecar_{subnet_id}` -This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. +This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. However the actual blobs that will be generated by the EL will be limited by `MAX_DATA_GAS_PER_BLOCK // DATA_GAS_PER_BLOB` and will require an EL hardfork to upgrade the limit. The following validations MUST pass before forwarding the `signed_blob_sidecar` on the network, assuming the alias `sidecar = signed_blob_sidecar.message`: From 9f530a7741e220b2c2611452a98e8ee195e6f02c Mon Sep 17 00:00:00 2001 From: gajinder Date: Sun, 7 May 2023 19:52:59 +0530 Subject: [PATCH 093/117] update max commitments per block limit to blobs per tx limit from eip4844 --- specs/deneb/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 934a840132..971b67f0e0 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -68,7 +68,7 @@ This upgrade adds blobs to the beacon chain as part of Deneb. This is an extensi | Name | Value | | - | - | -| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**10)` (= 1024) | hardfork independent fixed theoretical limit | +| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | ## Configuration From a75292beebdea362b70d65e7ceb3b78ae03d5be4 Mon Sep 17 00:00:00 2001 From: gajinder Date: Thu, 11 May 2023 19:03:43 +0530 Subject: [PATCH 094/117] restore the 4844 max limit to 4 --- presets/mainnet/deneb.yaml | 4 ++-- specs/deneb/p2p-interface.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/presets/mainnet/deneb.yaml b/presets/mainnet/deneb.yaml index d5dafe0d04..ebe33f2d10 100644 --- a/presets/mainnet/deneb.yaml +++ b/presets/mainnet/deneb.yaml @@ -4,5 +4,5 @@ # --------------------------------------------------------------- # `uint64(4096)` FIELD_ELEMENTS_PER_BLOB: 4096 -# `uint64(2**4)` (= 16) -MAX_BLOBS_PER_BLOCK: 16 +# `uint64(2**2)` (= 4) +MAX_BLOBS_PER_BLOCK: 4 diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 514991bb7e..4de7f4a83e 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -43,7 +43,7 @@ The specification of these changes continues in the same format as the network s | Name | Value | Description | |------------------------------------------|-----------------------------------|---------------------------------------------------------------------| | `MAX_REQUEST_BLOCKS_DENEB` | `2**7` (= 128) | Maximum number of blocks in a single request | -| `MAX_BLOBS_PER_BLOCK` | `2**4` (= 16) | Maximum number of blobs in a single request, to be always <= `MAX_BLOB_COMMITMENTS_PER_BLOCK` | +| `MAX_BLOBS_PER_BLOCK` | `2**2` (= 4) | Maximum number of blobs in a single request, can never exceed `MAX_BLOB_COMMITMENTS_PER_BLOCK` limit | | `MAX_REQUEST_BLOB_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK` | Maximum number of blob sidecars in a single request | | `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars | From 8ccc2570d15774b474b2151337304be727750cf1 Mon Sep 17 00:00:00 2001 From: gajinder Date: Sat, 20 May 2023 19:35:22 +0530 Subject: [PATCH 095/117] apply feedback --- specs/deneb/beacon-chain.md | 3 +++ specs/deneb/p2p-interface.md | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 971b67f0e0..1926cc8ce6 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -40,6 +40,8 @@ This upgrade adds blobs to the beacon chain as part of Deneb. This is an extension of the Capella upgrade. +The blob transactions are packed into the execution payload by the EL/builder with their corresponding blobs being independently transmitted and are limited by `MAX_DATA_GAS_PER_BLOCK // DATA_GAS_PER_BLOB`. However the CL limit is independently defined by `MAX_BLOBS_PER_BLOCK`. + ## Custom types | Name | SSZ equivalent | Description | @@ -69,6 +71,7 @@ This upgrade adds blobs to the beacon chain as part of Deneb. This is an extensi | Name | Value | | - | - | | `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | +| `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | ## Configuration diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index 4de7f4a83e..d67c144b21 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -43,7 +43,6 @@ The specification of these changes continues in the same format as the network s | Name | Value | Description | |------------------------------------------|-----------------------------------|---------------------------------------------------------------------| | `MAX_REQUEST_BLOCKS_DENEB` | `2**7` (= 128) | Maximum number of blocks in a single request | -| `MAX_BLOBS_PER_BLOCK` | `2**2` (= 4) | Maximum number of blobs in a single request, can never exceed `MAX_BLOB_COMMITMENTS_PER_BLOCK` limit | | `MAX_REQUEST_BLOB_SIDECARS` | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK` | Maximum number of blob sidecars in a single request | | `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS` | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars | @@ -120,7 +119,7 @@ The *type* of the payload of this topic changes to the (modified) `SignedBeaconB ###### `blob_sidecar_{subnet_id}` -This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. However the actual blobs that will be generated by the EL will be limited by `MAX_DATA_GAS_PER_BLOCK // DATA_GAS_PER_BLOB` and will require an EL hardfork to upgrade the limit. +This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. The following validations MUST pass before forwarding the `signed_blob_sidecar` on the network, assuming the alias `sidecar = signed_blob_sidecar.message`: From 4458645f0cffc65133618690ff97ace401f83f27 Mon Sep 17 00:00:00 2001 From: gajinder Date: Mon, 22 May 2023 18:41:40 +0530 Subject: [PATCH 096/117] add check --- specs/deneb/beacon-chain.md | 1 + 1 file changed, 1 insertion(+) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 1926cc8ce6..b2f6b17634 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -254,6 +254,7 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe ```python def process_blob_kzg_commitments(body: BeaconBlockBody) -> None: + assert len(body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK assert verify_kzg_commitments_against_transactions(body.execution_payload.transactions, body.blob_kzg_commitments) ``` From feb1968e433daa1a4c540d9226a98c6c4f9b238b Mon Sep 17 00:00:00 2001 From: gajinder Date: Mon, 22 May 2023 18:44:12 +0530 Subject: [PATCH 097/117] add comment --- specs/deneb/beacon-chain.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index b2f6b17634..f492a948c0 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -254,7 +254,9 @@ def process_execution_payload(state: BeaconState, payload: ExecutionPayload, exe ```python def process_blob_kzg_commitments(body: BeaconBlockBody) -> None: + # Verify commitments are under limit assert len(body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK + # Verify commitments match with corresponding blob transactions assert verify_kzg_commitments_against_transactions(body.execution_payload.transactions, body.blob_kzg_commitments) ``` From 3247bcf8f721cef4b88deac6b3e22c18fbf815eb Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 May 2023 15:16:12 +0800 Subject: [PATCH 098/117] PR feedback from @ppopth --- specs/bellatrix/beacon-chain.md | 2 +- specs/deneb/beacon-chain.md | 17 +---------------- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index a4c7be16e9..9df07d69f4 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -330,7 +330,7 @@ The Engine API may be used to implement this and similarly defined functions via ```python def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: """ - Return ``True`` if and only if ``execution_payload`` is valid with respect to ``self.execution_state``. + Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ ... ``` diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 2c4bb61335..20455ab0de 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -30,7 +30,6 @@ - [Modified `NewPayloadRequest`](#modified-newpayloadrequest) - [Engine APIs](#engine-apis) - [Modified `notify_new_payload`](#modified-notify_new_payload) - - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) - [Testing](#testing) @@ -182,25 +181,11 @@ class NewPayloadRequest(object): ```python def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: """ - Return ``True`` if and only if ``execution_payload`` is valid with respect to ``self.execution_state``. + Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ ... ``` -### Block processing - -```python -def process_block(state: BeaconState, block: BeaconBlock) -> None: - process_block_header(state, block) - if is_execution_enabled(state, block.body): - process_withdrawals(state, block.body.execution_payload) - process_execution_payload(state, block.body, EXECUTION_ENGINE) # [Modified in Deneb] - process_randao(state, block.body) - process_eth1_data(state, block.body) - process_operations(state, block.body) - process_sync_aggregate(state, block.body.sync_aggregate) -``` - #### Execution payload ##### `process_execution_payload` From fc45220a7d1ee5bc75be4c5a0a834512f812f8f1 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 May 2023 15:38:54 +0800 Subject: [PATCH 099/117] Move old Deneb sanity tests to block_processing (operations) tests --- .../test/deneb/block_processing/__init__.py | 0 .../test_process_execution_payload.py | 175 ++++++++++++++++++ .../eth2spec/test/deneb/sanity/test_blocks.py | 138 -------------- tests/generators/operations/main.py | 5 +- 4 files changed, 179 insertions(+), 139 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/deneb/block_processing/__init__.py create mode 100644 tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/__init__.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py new file mode 100644 index 0000000000..68da478b76 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py @@ -0,0 +1,175 @@ +from random import Random + +from eth2spec.test.helpers.execution_payload import ( + build_empty_execution_payload, + compute_el_block_hash, + get_execution_payload_header, +) +from eth2spec.test.context import ( + spec_state_test, + expect_assertion_error, + with_deneb_and_later +) +from eth2spec.test.helpers.sharding import ( + get_sample_opaque_tx, +) + + +def run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments, + valid=True, execution_valid=True): + """ + Run ``process_execution_payload``, yielding: + - pre-state ('pre') + - execution payload ('execution_payload') + - execution details, to mock EVM execution ('execution.yml', a dict with 'execution_valid' key and boolean value) + - post-state ('post'). + If ``valid == False``, run expecting ``AssertionError`` + """ + # Before Deneb, only `body.execution_payload` matters. `BeaconBlockBody` is just a wrapper. + body = spec.BeaconBlockBody( + blob_kzg_commitments=blob_kzg_commitments, + execution_payload=execution_payload + ) + + yield 'pre', state + yield 'execution', {'execution_valid': execution_valid} + yield 'body', body + + called_new_block = False + + class TestEngine(spec.NoopExecutionEngine): + def notify_new_payload(self, new_payload_request) -> bool: + nonlocal called_new_block, execution_valid + called_new_block = True + assert new_payload_request.execution_payload == body.execution_payload + return execution_valid + + if not valid: + expect_assertion_error(lambda: spec.process_execution_payload(state, body, TestEngine())) + yield 'post', None + return + + spec.process_execution_payload(state, body, TestEngine()) + + # Make sure we called the engine + assert called_new_block + + yield 'post', state + + assert state.latest_execution_payload_header == get_execution_payload_header(spec, body.execution_payload) + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_blob_tx_type(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx = b'\x04' + opaque_tx[1:] # incorrect tx type + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_transaction_length_1_byte(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx = opaque_tx + b'\x12' # incorrect tx length + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_transaction_length_32_bytes(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + opaque_tx = opaque_tx + b'\x12' * 32 # incorrect tx length + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_commitment(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + blob_kzg_commitments[0] = b'\x12' * 48 # incorrect commitment + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_commitments_order(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=2, rng=Random(1111)) + blob_kzg_commitments = [blob_kzg_commitments[1], blob_kzg_commitments[0]] # incorrect order + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) + + +@with_deneb_and_later +@spec_state_test +def test_incorrect_block_hash(spec, state): + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = b'\x12' * 32 # incorrect block hash + + # CL itself doesn't verify EL block hash + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) + + +@with_deneb_and_later +@spec_state_test +def test_zeroed_commitment(spec, state): + """ + The blob is invalid, but the commitment is in correct form. + """ + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=1, is_valid_blob=False) + assert all(commitment == b'\x00' * 48 for commitment in blob_kzg_commitments) + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index f65514f23d..36caacd3fa 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -1,5 +1,3 @@ -import random - from eth2spec.test.helpers.state import ( state_transition_and_sign_block ) @@ -49,139 +47,3 @@ def test_one_blob(spec, state): @spec_state_test def test_max_blobs(spec, state): yield from run_block_with_blobs(spec, state, blob_count=spec.MAX_BLOBS_PER_BLOCK) - - -@with_deneb_and_later -@spec_state_test -def test_incorrect_blob_tx_type(spec, state): - """ - The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. - """ - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) - block.body.blob_kzg_commitments = blob_kzg_commitments - opaque_tx = b'\x04' + opaque_tx[1:] # incorrect tx type - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state - - -@with_deneb_and_later -@spec_state_test -def test_incorrect_transaction_length_1_byte(spec, state): - """ - The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. - """ - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) - block.body.blob_kzg_commitments = blob_kzg_commitments - opaque_tx = opaque_tx + b'\x12' # incorrect tx length - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state - - -@with_deneb_and_later -@spec_state_test -def test_incorrect_transaction_length_32_bytes(spec, state): - """ - The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. - """ - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) - block.body.blob_kzg_commitments = blob_kzg_commitments - opaque_tx = opaque_tx + b'\x12' * 32 # incorrect tx length - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state - - -@with_deneb_and_later -@spec_state_test -def test_incorrect_commitment(spec, state): - """ - The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. - """ - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) - blob_kzg_commitments[0] = b'\x12' * 48 # incorrect commitment - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state - - -@with_deneb_and_later -@spec_state_test -def test_incorrect_commitments_order(spec, state): - """ - The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. - """ - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=2, rng=random.Random(1111)) - block.body.blob_kzg_commitments = [blob_kzg_commitments[1], blob_kzg_commitments[0]] # incorrect order - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state - - -@with_deneb_and_later -@spec_state_test -def test_incorrect_block_hash(spec, state): - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = b'\x12' * 32 # incorrect block hash - # CL itself doesn't verify EL block hash - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state - - -@with_deneb_and_later -@spec_state_test -def test_zeroed_commitment(spec, state): - """ - The blob is invalid, but the commitment is in correct form. - """ - yield 'pre', state - - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=1, is_valid_blob=False) - assert all(commitment == b'\x00' * 48 for commitment in blob_kzg_commitments) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) - - yield 'blocks', [signed_block] - yield 'post', state diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index fc22179176..293bcdea7d 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -36,7 +36,10 @@ ]} capella_mods = combine_mods(_new_capella_mods, bellatrix_mods) - deneb_mods = capella_mods + _new_deneb_mods = {key: 'eth2spec.test.deneb.block_processing.test_process_' + key for key in [ + 'execution_payload', + ]} + deneb_mods = combine_mods(_new_deneb_mods, capella_mods) _new_eip6110_mods = {key: 'eth2spec.test.eip6110.block_processing.test_process_' + key for key in [ 'deposit_receipt', From bee8fa16e567cec88f68372a3768e4fec476a6b2 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 May 2023 18:34:00 +0800 Subject: [PATCH 100/117] Add test_invalid_correct_input__execution_invalid --- .../test_process_execution_payload.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py index 68da478b76..d3a04ddfec 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py @@ -173,3 +173,20 @@ def test_zeroed_commitment(spec, state): execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments) + + +@with_deneb_and_later +@spec_state_test +def test_invalid_correct_input__execution_invalid(spec, state): + """ + The versioned hashes are wrong, but the testing ExecutionEngine returns VALID by default. + """ + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec) + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments, + valid=False, execution_valid=False) From 73df1935b1dbc09036f793e2759e3c4f794fc04b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 May 2023 22:25:43 +0800 Subject: [PATCH 101/117] Use `verify_and_notify_new_payload` approach --- setup.py | 58 ++++++++++++++++++- specs/_features/eip6110/beacon-chain.md | 2 +- specs/bellatrix/beacon-chain.md | 6 +- specs/capella/beacon-chain.md | 2 +- specs/deneb/beacon-chain.md | 40 +++++++++++-- .../test_process_execution_payload.py | 8 ++- .../test_process_execution_payload.py | 2 +- 7 files changed, 104 insertions(+), 14 deletions(-) diff --git a/setup.py b/setup.py index e8bbeefbb9..609d2acf34 100644 --- a/setup.py +++ b/setup.py @@ -336,6 +336,10 @@ def sundry_functions(cls) -> str: """ raise NotImplementedError() + @classmethod + def execution_engine_cls(cls) -> str: + raise NotImplementedError() + @classmethod @abstractmethod def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: @@ -469,6 +473,12 @@ def wrapper(*args, **kw): # type: ignore ), _get_attesting_indices, lru_size=SLOTS_PER_EPOCH * MAX_COMMITTEES_PER_SLOT * 3)''' + + @classmethod + def execution_engine_cls(cls) -> str: + return "" + + @classmethod def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]: return {} @@ -573,12 +583,14 @@ def get_execution_state(_execution_state_root: Bytes32) -> ExecutionState: def get_pow_chain_head() -> PowBlock: - pass - + pass""" + @classmethod + def execution_engine_cls(cls) -> str: + return "\n\n" + """ class NoopExecutionEngine(ExecutionEngine): - def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: + def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: return True def notify_forkchoice_updated(self: ExecutionEngine, @@ -658,6 +670,39 @@ def retrieve_blobs_and_proofs(beacon_block_root: Root) -> PyUnion[Tuple[Blob, KZ # pylint: disable=unused-argument return ("TEST", "TEST")''' + @classmethod + def execution_engine_cls(cls) -> str: + return "\n\n" + """ +class NoopExecutionEngine(ExecutionEngine): + + def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + return True + + def notify_forkchoice_updated(self: ExecutionEngine, + head_block_hash: Hash32, + safe_block_hash: Hash32, + finalized_block_hash: Hash32, + payload_attributes: Optional[PayloadAttributes]) -> Optional[PayloadId]: + pass + + def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadResponse: + # pylint: disable=unused-argument + raise NotImplementedError("no default block production") + + def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + return True + + def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: + return True + + def verify_and_notify_new_payload(self: ExecutionEngine, + new_payload_request: NewPayloadRequest) -> bool: + return True + + +EXECUTION_ENGINE = NoopExecutionEngine()""" + + @classmethod def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: constants = { @@ -708,6 +753,12 @@ def objects_to_spec(preset_name: str, ) def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str: + if "verify_and_notify_new_payload" in protocol_def.functions: + # del protocol_def.functions['verify_and_notify_new_payload'] + protocol_def.functions['verify_and_notify_new_payload'] = """def verify_and_notify_new_payload(self: ExecutionEngine, + new_payload_request: NewPayloadRequest) -> bool: + ...""" + protocol = f"class {protocol_name}(Protocol):" for fn_source in protocol_def.functions.values(): fn_source = fn_source.replace("self: "+protocol_name, "self") @@ -783,6 +834,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str: + ('\n\n\n' + protocols_spec if protocols_spec != '' else '') + '\n\n\n' + functions_spec + '\n\n' + builder.sundry_functions() + + builder.execution_engine_cls() # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are # as same as the spec definition. + ('\n\n\n' + ssz_dep_constants_verification if ssz_dep_constants_verification != '' else '') diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index 591d8dc869..084f88f145 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -249,7 +249,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] - assert execution_engine.notify_new_payload( + assert execution_engine.verify_and_notify_new_payload( NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes) ) # Cache execution payload header diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index 9df07d69f4..9ae1e5a711 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -328,9 +328,9 @@ The Engine API may be used to implement this and similarly defined functions via #### `notify_new_payload` ```python -def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: +def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: """ - Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. + Return ``True`` if and only if ``execution_payload`` is valid with respect to ``self.execution_state``. """ ... ``` @@ -366,7 +366,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload)) + assert execution_engine.notify_new_payload(payload) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index 1799f4e6e3..c8e969d251 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -418,7 +418,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(NewPayloadRequest(execution_payload=payload)) + assert execution_engine.notify_new_payload(payload) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 20455ab0de..bed64db513 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -29,7 +29,9 @@ - [Request data](#request-data) - [Modified `NewPayloadRequest`](#modified-newpayloadrequest) - [Engine APIs](#engine-apis) - - [Modified `notify_new_payload`](#modified-notify_new_payload) + - [`is_valid_block_hash`](#is_valid_block_hash) + - [`is_valid_versioned_hashes`](#is_valid_versioned_hashes) + - [`verify_and_notify_new_payload`](#verify_and_notify_new_payload) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) - [Testing](#testing) @@ -176,14 +178,44 @@ class NewPayloadRequest(object): #### Engine APIs -#### Modified `notify_new_payload` +#### `is_valid_block_hash` + +[0]: # (eth2spec: skip) + +```python +def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + """ + Return ``True`` if and only if ``execution_payload.block_hash`` is computed correctly. + """ + ... +``` + +#### `is_valid_versioned_hashes` + +[0]: # (eth2spec: skip) + +```python +def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: + """ + Return ``True`` if and only if the version hashes computed by the blob transactions of + ``new_payload_request.execution_payload`` matches ``new_payload_request.version_hashes``. + """ + ... +``` + +#### `verify_and_notify_new_payload` ```python -def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: +def verify_and_notify_new_payload(self: ExecutionEngine, + new_payload_request: NewPayloadRequest) -> bool: """ Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ + assert self.is_valid_block_hash(new_payload_request.execution_payload) + assert self.is_valid_versioned_hashes(new_payload_request) + assert self.notify_new_payload(new_payload_request.execution_payload) ... + return True ``` #### Execution payload @@ -204,7 +236,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi # Verify the execution payload is valid # [Modified in Deneb] versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] - assert execution_engine.notify_new_payload( + assert execution_engine.verify_and_notify_new_payload( NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes) ) diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index b835632354..b54afafc34 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -31,7 +31,13 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True, called_new_block = False class TestEngine(spec.NoopExecutionEngine): - def notify_new_payload(self, new_payload_request) -> bool: + def notify_new_payload(self, payload) -> bool: + nonlocal called_new_block, execution_valid + called_new_block = True + assert payload == execution_payload + return execution_valid + + def verify_and_notify_new_payload(self, new_payload_request) -> bool: nonlocal called_new_block, execution_valid called_new_block = True assert new_payload_request.execution_payload == body.execution_payload diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py index d3a04ddfec..df59385ab4 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py @@ -38,7 +38,7 @@ def run_execution_payload_processing(spec, state, execution_payload, blob_kzg_co called_new_block = False class TestEngine(spec.NoopExecutionEngine): - def notify_new_payload(self, new_payload_request) -> bool: + def verify_and_notify_new_payload(self, new_payload_request) -> bool: nonlocal called_new_block, execution_valid called_new_block = True assert new_payload_request.execution_payload == body.execution_payload From 7ec5efb10666af3ccff9a3a84cbe46ba6578ac1a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 23 May 2023 23:55:35 +0800 Subject: [PATCH 102/117] Add `### Block processing` back --- specs/deneb/beacon-chain.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 20455ab0de..548d493ccc 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -30,6 +30,7 @@ - [Modified `NewPayloadRequest`](#modified-newpayloadrequest) - [Engine APIs](#engine-apis) - [Modified `notify_new_payload`](#modified-notify_new_payload) + - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) - [Testing](#testing) @@ -186,6 +187,8 @@ def notify_new_payload(self: ExecutionEngine, new_payload_request: NewPayloadReq ... ``` +### Block processing + #### Execution payload ##### `process_execution_payload` From 48618fe866383c07f4fc814b819af72bdff8b4b7 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 00:08:45 +0800 Subject: [PATCH 103/117] Fix tests --- .../test_process_execution_payload.py | 10 ++++++-- .../test_process_execution_payload.py | 24 +++++++++++++++++++ tests/generators/operations/main.py | 3 ++- 3 files changed, 34 insertions(+), 3 deletions(-) create mode 100644 tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_execution_payload.py diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index 3ec58b31ed..d960ceb0af 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -8,7 +8,13 @@ build_state_with_incomplete_transition, build_state_with_complete_transition, ) -from eth2spec.test.context import spec_state_test, expect_assertion_error, with_bellatrix_and_later +from eth2spec.test.context import ( + BELLATRIX, + expect_assertion_error, + spec_state_test, + with_bellatrix_and_later, + with_phases, +) from eth2spec.test.helpers.state import next_slot @@ -117,7 +123,7 @@ def test_invalid_bad_execution_regular_payload(spec, state): yield from run_bad_execution_test(spec, state) -@with_bellatrix_and_later +@with_phases([BELLATRIX]) @spec_state_test def test_bad_parent_hash_first_payload(spec, state): state = build_state_with_incomplete_transition(spec, state) diff --git a/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_execution_payload.py new file mode 100644 index 0000000000..1a2c1771a2 --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/capella/block_processing/test_process_execution_payload.py @@ -0,0 +1,24 @@ +from eth2spec.test.helpers.execution_payload import ( + build_empty_execution_payload, + compute_el_block_hash, + build_state_with_incomplete_transition, +) +from eth2spec.test.context import ( + spec_state_test, + with_capella_and_later, +) +from eth2spec.test.helpers.state import next_slot +from eth2spec.test.bellatrix.block_processing.test_process_execution_payload import run_execution_payload_processing + + +@with_capella_and_later +@spec_state_test +def test_invalid_bad_parent_hash_first_payload(spec, state): + state = build_state_with_incomplete_transition(spec, state) + next_slot(spec, state) + + execution_payload = build_empty_execution_payload(spec, state) + execution_payload.parent_hash = b'\x55' * 32 + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, valid=False) diff --git a/tests/generators/operations/main.py b/tests/generators/operations/main.py index fc22179176..3983931b35 100644 --- a/tests/generators/operations/main.py +++ b/tests/generators/operations/main.py @@ -30,8 +30,9 @@ bellatrix_mods = combine_mods(_new_bellatrix_mods, altair_mods) _new_capella_mods = {key: 'eth2spec.test.capella.block_processing.test_process_' + key for key in [ - 'deposit', 'bls_to_execution_change', + 'deposit', + 'execution_payload', 'withdrawals', ]} capella_mods = combine_mods(_new_capella_mods, bellatrix_mods) From dd5e6f813f82cd20636dfb36950ec281a428d48f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 01:58:49 +0800 Subject: [PATCH 104/117] Add `make_function_abstract` to make it more general --- setup.py | 14 +++++++++----- specs/deneb/beacon-chain.md | 4 ---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 609d2acf34..22a05ee320 100644 --- a/setup.py +++ b/setup.py @@ -736,6 +736,11 @@ def is_byte_vector(value: str) -> bool: return value.startswith(('ByteVector')) +def make_function_abstract(protocol_def: ProtocolDefinition, key: str): + function = protocol_def.functions[key].split('"""') + protocol_def.functions[key] = function[0] + "..." + + def objects_to_spec(preset_name: str, spec_object: SpecObject, builder: SpecBuilder, @@ -753,11 +758,10 @@ def objects_to_spec(preset_name: str, ) def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str: - if "verify_and_notify_new_payload" in protocol_def.functions: - # del protocol_def.functions['verify_and_notify_new_payload'] - protocol_def.functions['verify_and_notify_new_payload'] = """def verify_and_notify_new_payload(self: ExecutionEngine, - new_payload_request: NewPayloadRequest) -> bool: - ...""" + abstract_functions = ["verify_and_notify_new_payload"] + for key in protocol_def.functions.keys(): + if key in abstract_functions: + make_function_abstract(protocol_def, key) protocol = f"class {protocol_name}(Protocol):" for fn_source in protocol_def.functions.values(): diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index bed64db513..615af60d4d 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -180,8 +180,6 @@ class NewPayloadRequest(object): #### `is_valid_block_hash` -[0]: # (eth2spec: skip) - ```python def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: """ @@ -192,8 +190,6 @@ def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPaylo #### `is_valid_versioned_hashes` -[0]: # (eth2spec: skip) - ```python def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: """ From 68ce45b3058186ff9e8bdecb19d51327851a4259 Mon Sep 17 00:00:00 2001 From: djrtwo Date: Tue, 23 May 2023 13:52:36 -0600 Subject: [PATCH 105/117] move epoch subscription length to config --- configs/mainnet.yaml | 2 ++ configs/minimal.yaml | 2 ++ presets/mainnet/phase0.yaml | 5 ----- presets/minimal/phase0.yaml | 5 ----- specs/phase0/p2p-interface.md | 7 +------ 5 files changed, 5 insertions(+), 16 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 1a1537b26d..baf489739e 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -102,6 +102,8 @@ DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa GOSSIP_MAX_SIZE: 1048576 # `2**10` (= 1024) MAX_REQUEST_BLOCKS: 1024 +# `2**8` (= 256) +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 # `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 # `2**20` (=1048576, 1 MiB) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 32d5682a82..43f1fc83fa 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -103,6 +103,8 @@ DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890 GOSSIP_MAX_SIZE: 1048576 # `2**10` (= 1024) MAX_REQUEST_BLOCKS: 1024 +# `2**8` (= 256) +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 # [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272) MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272 # `2**20` (=1048576, 1 MiB) diff --git a/presets/mainnet/phase0.yaml b/presets/mainnet/phase0.yaml index c84a935954..02bc96c8cd 100644 --- a/presets/mainnet/phase0.yaml +++ b/presets/mainnet/phase0.yaml @@ -86,8 +86,3 @@ MAX_ATTESTATIONS: 128 MAX_DEPOSITS: 16 # 2**4 (= 16) MAX_VOLUNTARY_EXITS: 16 - -# Networking -# --------------------------------------------------------------- -# 2**8 (= 256) -EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 diff --git a/presets/minimal/phase0.yaml b/presets/minimal/phase0.yaml index c55f7de7b7..e7028f5a42 100644 --- a/presets/minimal/phase0.yaml +++ b/presets/minimal/phase0.yaml @@ -86,8 +86,3 @@ MAX_ATTESTATIONS: 128 MAX_DEPOSITS: 16 # 2**4 (= 16) MAX_VOLUNTARY_EXITS: 16 - -# Networking -# --------------------------------------------------------------- -# 2**8 (= 256) -EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 diff --git a/specs/phase0/p2p-interface.md b/specs/phase0/p2p-interface.md index 3b8a91cc94..de3e0e529c 100644 --- a/specs/phase0/p2p-interface.md +++ b/specs/phase0/p2p-interface.md @@ -22,7 +22,6 @@ It consists of four main sections: - [Consensus-layer network interaction domains](#consensus-layer-network-interaction-domains) - [Custom types](#custom-types) - [Constants](#constants) - - [Preset](#preset) - [Configuration](#configuration) - [MetaData](#metadata) - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub) @@ -184,11 +183,6 @@ We define the following Python custom types for type hinting and readability: | - | - | :-: | :-: | | `NODE_ID_BITS` | `256` | The bit length of uint256 is 256 | -### Preset -| Name | Value | Unit | Duration | -| - | - | :-: | :-: | -| `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | epochs | ~27 hours | - ### Configuration This section outlines configurations that are used in this spec. @@ -197,6 +191,7 @@ This section outlines configurations that are used in this spec. |---|---|---| | `GOSSIP_MAX_SIZE` | `2**20` (= 1048576, 1 MiB) | The maximum allowed size of uncompressed gossip messages. | | `MAX_REQUEST_BLOCKS` | `2**10` (= 1024) | Maximum number of blocks in a single request | +| `EPOCHS_PER_SUBNET_SUBSCRIPTION` | `2**8` (= 256) | Number of epochs on a subnet subscription (~27 hours) | | `MIN_EPOCHS_FOR_BLOCK_REQUESTS` | `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) | The minimum epoch range over which a node must serve blocks | | `MAX_CHUNK_SIZE` | `2**20` (=1048576, 1 MiB) | The maximum allowed size of uncompressed req/resp chunked responses. | | `TTFB_TIMEOUT` | `5` | The maximum duration in **seconds** to wait for first byte of request response (time-to-first-byte). | From 1f75acadd979734988b07c018ce6d454ccf8d0e9 Mon Sep 17 00:00:00 2001 From: Danny Ryan Date: Tue, 23 May 2023 14:10:35 -0600 Subject: [PATCH 106/117] Update VERSION.txt to 1.4.0-alpha.0 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index f0bb29e763..7792609910 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.3.0 +1.4.0-alpha.0 From 289d8147be38438fac7b196622e2a133b9f2adf9 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 11:12:03 +0800 Subject: [PATCH 107/117] Move `verify_and_notify_new_payload` to Bellatrix --- setup.py | 7 ++++ specs/bellatrix/beacon-chain.md | 32 ++++++++++++++++--- specs/capella/beacon-chain.md | 2 +- specs/deneb/beacon-chain.md | 14 ++------ sync/optimistic.md | 4 +-- .../test_process_execution_payload.py | 6 ---- .../eth2spec/test/helpers/optimistic_sync.py | 2 +- 7 files changed, 41 insertions(+), 26 deletions(-) diff --git a/setup.py b/setup.py index 22a05ee320..12a561514e 100644 --- a/setup.py +++ b/setup.py @@ -604,6 +604,13 @@ def get_payload(self: ExecutionEngine, payload_id: PayloadId) -> GetPayloadRespo # pylint: disable=unused-argument raise NotImplementedError("no default block production") + def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + return True + + def verify_and_notify_new_payload(self: ExecutionEngine, + new_payload_request: NewPayloadRequest) -> bool: + return True + EXECUTION_ENGINE = NoopExecutionEngine()""" diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index 9ae1e5a711..e8eef346d2 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -320,13 +320,13 @@ The implementation-dependent `ExecutionEngine` protocol encapsulates the executi * a state object `self.execution_state` of type `ExecutionState` * a notification function `self.notify_new_payload` which may apply changes to the `self.execution_state` -*Note*: `notify_new_payload` is a function accessed through the `EXECUTION_ENGINE` module which instantiates the `ExecutionEngine` protocol. - -The body of this function is implementation dependent. +The body of these functions are implementation dependent. The Engine API may be used to implement this and similarly defined functions via an external execution engine. #### `notify_new_payload` +`notify_new_payload` is a function accessed through the `EXECUTION_ENGINE` module which instantiates the `ExecutionEngine` protocol. + ```python def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: """ @@ -335,6 +335,30 @@ def notify_new_payload(self: ExecutionEngine, execution_payload: ExecutionPayloa ... ``` +#### `is_valid_block_hash` + +```python +def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: + """ + Return ``True`` if and only if ``execution_payload.block_hash`` is computed correctly. + """ + ... +``` + +#### `verify_and_notify_new_payload` + +```python +def verify_and_notify_new_payload(self: ExecutionEngine, + new_payload_request: NewPayloadRequest) -> bool: + """ + Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. + """ + assert self.is_valid_block_hash(new_payload_request.execution_payload) + assert self.notify_new_payload(new_payload_request.execution_payload) + ... + return True +``` + ### Block processing *Note*: The call to the `process_execution_payload` must happen before the call to the `process_randao` as the former depends on the `randao_mix` computed with the reveal of the previous block. @@ -366,7 +390,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(payload) + assert execution_engine.verify_and_notify_new_payload(NewPayloadRequest(execution_payload=payload)) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/capella/beacon-chain.md b/specs/capella/beacon-chain.md index c8e969d251..af40830176 100644 --- a/specs/capella/beacon-chain.md +++ b/specs/capella/beacon-chain.md @@ -418,7 +418,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) # Verify the execution payload is valid - assert execution_engine.notify_new_payload(payload) + assert execution_engine.verify_and_notify_new_payload(NewPayloadRequest(execution_payload=payload)) # Cache execution payload header state.latest_execution_payload_header = ExecutionPayloadHeader( parent_hash=payload.parent_hash, diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 517913a366..bffa6a4567 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -179,16 +179,6 @@ class NewPayloadRequest(object): #### Engine APIs -#### `is_valid_block_hash` - -```python -def is_valid_block_hash(self: ExecutionEngine, execution_payload: ExecutionPayload) -> bool: - """ - Return ``True`` if and only if ``execution_payload.block_hash`` is computed correctly. - """ - ... -``` - #### `is_valid_versioned_hashes` ```python @@ -200,7 +190,7 @@ def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPay ... ``` -#### `verify_and_notify_new_payload` +#### Modified `verify_and_notify_new_payload` ```python def verify_and_notify_new_payload(self: ExecutionEngine, @@ -209,7 +199,7 @@ def verify_and_notify_new_payload(self: ExecutionEngine, Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ assert self.is_valid_block_hash(new_payload_request.execution_payload) - assert self.is_valid_versioned_hashes(new_payload_request) + assert self.is_valid_versioned_hashes(new_payload_request) # [Modified in Deneb] assert self.notify_new_payload(new_payload_request.execution_payload) ... return True diff --git a/sync/optimistic.md b/sync/optimistic.md index 14eb99fb11..1fd1ba46e6 100644 --- a/sync/optimistic.md +++ b/sync/optimistic.md @@ -159,7 +159,7 @@ these conditions.* To optimistically import a block: -- The [`notify_new_payload`](../specs/bellatrix/beacon-chain.md#notify_new_payload) function MUST return `True` if the execution +- The [`verify_and_notify_new_payload`](../specs/bellatrix/beacon-chain.md#verify_and_notify_new_payload) function MUST return `True` if the execution engine returns `NOT_VALIDATED` or `VALID`. An `INVALIDATED` response MUST return `False`. - The [`validate_merge_block`](../specs/bellatrix/fork-choice.md#validate_merge_block) function MUST NOT raise an assertion if both the @@ -172,7 +172,7 @@ In addition to this change in validation, the consensus engine MUST track which blocks returned `NOT_VALIDATED` and which returned `VALID` for subsequent processing. Optimistically imported blocks MUST pass all verifications included in -`process_block` (withstanding the modifications to `notify_new_payload`). +`process_block` (withstanding the modifications to `verify_and_notify_new_payload`). A consensus engine MUST be able to retrospectively (i.e., after import) modify the status of `NOT_VALIDATED` blocks to be either `VALID` or `INVALIDATED` based upon responses diff --git a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py index b54afafc34..1aa4a2c37e 100644 --- a/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/bellatrix/block_processing/test_process_execution_payload.py @@ -31,12 +31,6 @@ def run_execution_payload_processing(spec, state, execution_payload, valid=True, called_new_block = False class TestEngine(spec.NoopExecutionEngine): - def notify_new_payload(self, payload) -> bool: - nonlocal called_new_block, execution_valid - called_new_block = True - assert payload == execution_payload - return execution_valid - def verify_and_notify_new_payload(self, new_payload_request) -> bool: nonlocal called_new_block, execution_valid called_new_block = True diff --git a/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py b/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py index 816c7a10b7..ad23dbebc9 100644 --- a/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py +++ b/tests/core/pyspec/eth2spec/test/helpers/optimistic_sync.py @@ -91,7 +91,7 @@ def add_optimistic_block(spec, mega_store, signed_block, test_steps, Add a block with optimistic sync logic ``valid`` indicates if the given ``signed_block.message.body.execution_payload`` is valid/invalid - from ``notify_new_payload`` method response. + from ``verify_and_notify_new_payload`` method response. """ block = signed_block.message block_root = block.hash_tree_root() From 212a314287568a1414f389569a0bda0a8343354f Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 11:27:26 +0800 Subject: [PATCH 108/117] Fix ToC and remove `validate_blobs_and_kzg_commitments` --- specs/bellatrix/beacon-chain.md | 2 + specs/deneb/beacon-chain.md | 3 +- specs/deneb/validator.md | 13 +-- .../unittests/validator/test_validator.py | 87 ------------------- 4 files changed, 4 insertions(+), 101 deletions(-) diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index e8eef346d2..6cc69d747f 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -37,6 +37,8 @@ - [`NewPayloadRequest`](#newpayloadrequest) - [Engine APIs](#engine-apis) - [`notify_new_payload`](#notify_new_payload) + - [`is_valid_block_hash`](#is_valid_block_hash) + - [`verify_and_notify_new_payload`](#verify_and_notify_new_payload) - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 5c31f1fc0f..85c1fbdceb 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -29,9 +29,8 @@ - [Request data](#request-data) - [Modified `NewPayloadRequest`](#modified-newpayloadrequest) - [Engine APIs](#engine-apis) - - [`is_valid_block_hash`](#is_valid_block_hash) - [`is_valid_versioned_hashes`](#is_valid_versioned_hashes) - - [`verify_and_notify_new_payload`](#verify_and_notify_new_payload) + - [Modified `verify_and_notify_new_payload`](#modified-verify_and_notify_new_payload) - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) diff --git a/specs/deneb/validator.md b/specs/deneb/validator.md index fa6f1651e4..ae5f266737 100644 --- a/specs/deneb/validator.md +++ b/specs/deneb/validator.md @@ -101,18 +101,7 @@ All validator responsibilities remain unchanged other than those noted below. 1. After retrieving the execution payload from the execution engine as specified in Capella, use the `payload_id` to retrieve `blobs`, `blob_kzg_commitments`, and `blob_kzg_proofs` via `get_payload(payload_id).blobs_bundle`. -2. Validate `blobs` and `blob_kzg_commitments`: - -```python -def validate_blobs_and_kzg_commitments(blobs: Sequence[Blob], - blob_kzg_commitments: Sequence[KZGCommitment], - blob_kzg_proofs: Sequence[KZGProof]) -> None: - # Optionally sanity-check that the KZG commitments match the blobs (as produced by the execution engine) - assert len(blob_kzg_commitments) == len(blobs) == len(blob_kzg_proofs) - assert verify_blob_kzg_proof_batch(blobs, blob_kzg_commitments, blob_kzg_proofs) -``` - -3. If valid, set `block.body.blob_kzg_commitments = blob_kzg_commitments`. +2. Set `block.body.blob_kzg_commitments = blob_kzg_commitments`. #### Constructing the `SignedBlobSidecar`s diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py index 62b8e9dc1d..876824107a 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py @@ -2,7 +2,6 @@ always_bls, spec_state_test, with_deneb_and_later, - expect_assertion_error ) from eth2spec.test.helpers.execution_payload import ( compute_el_block_hash, @@ -18,92 +17,6 @@ ) -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_and_kzg_commitments(spec, state): - """ - Test `validate_blobs_and_kzg_commitments` - """ - blob_count = 4 - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_opaque_tx(spec, blob_count=blob_count) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - - spec.validate_blobs_and_kzg_commitments(blobs, - blob_kzg_commitments, - proofs) - - -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_and_kzg_commitments_missing_blob(spec, state): - """ - Test `validate_blobs_and_kzg_commitments` - """ - blob_count = 4 - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_opaque_tx(spec, blob_count=blob_count) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - - expect_assertion_error( - lambda: spec.validate_blobs_and_kzg_commitments( - blobs[:-1], - blob_kzg_commitments, - proofs - ) - ) - - -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_and_kzg_commitments_missing_proof(spec, state): - """ - Test `validate_blobs_and_kzg_commitments` - """ - blob_count = 4 - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_opaque_tx(spec, blob_count=blob_count) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - - expect_assertion_error( - lambda: spec.validate_blobs_and_kzg_commitments( - blobs, - blob_kzg_commitments, - proofs[:-1] - ) - ) - - -@with_deneb_and_later -@spec_state_test -def test_validate_blobs_and_kzg_commitments_incorrect_blob(spec, state): - """ - Test `validate_blobs_and_kzg_commitments` - """ - blob_count = 4 - block = build_empty_block_for_next_slot(spec, state) - opaque_tx, blobs, blob_kzg_commitments, proofs = get_sample_opaque_tx(spec, blob_count=blob_count) - block.body.blob_kzg_commitments = blob_kzg_commitments - block.body.execution_payload.transactions = [opaque_tx] - block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - - blobs[1] = spec.Blob(blobs[1][:13] + bytes([(blobs[1][13] + 1) % 256]) + blobs[1][14:]) - - expect_assertion_error( - lambda: spec.validate_blobs_and_kzg_commitments( - blobs, - blob_kzg_commitments, - proofs - ) - ) - - @with_deneb_and_later @spec_state_test def test_blob_sidecar_signature(spec, state): From 53a9221d1a67799549f55438a2b1ce5e533d3b5b Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 11:55:22 +0800 Subject: [PATCH 109/117] Fix ToC --- specs/deneb/beacon-chain.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 85c1fbdceb..1ab1df6116 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -29,8 +29,8 @@ - [Request data](#request-data) - [Modified `NewPayloadRequest`](#modified-newpayloadrequest) - [Engine APIs](#engine-apis) - - [`is_valid_versioned_hashes`](#is_valid_versioned_hashes) - - [Modified `verify_and_notify_new_payload`](#modified-verify_and_notify_new_payload) + - [`is_valid_versioned_hashes`](#is_valid_versioned_hashes) + - [Modified `verify_and_notify_new_payload`](#modified-verify_and_notify_new_payload) - [Block processing](#block-processing) - [Execution payload](#execution-payload) - [`process_execution_payload`](#process_execution_payload) @@ -178,7 +178,7 @@ class NewPayloadRequest(object): #### Engine APIs -#### `is_valid_versioned_hashes` +##### `is_valid_versioned_hashes` ```python def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPayloadRequest) -> bool: @@ -189,7 +189,7 @@ def is_valid_versioned_hashes(self: ExecutionEngine, new_payload_request: NewPay ... ``` -#### Modified `verify_and_notify_new_payload` +##### Modified `verify_and_notify_new_payload` ```python def verify_and_notify_new_payload(self: ExecutionEngine, From ec1ee74edb97112fdd0361a6d7c46d8a509684a6 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 12:17:07 +0800 Subject: [PATCH 110/117] Fix typo Co-authored-by: Danny Ryan --- specs/deneb/beacon-chain.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 1ab1df6116..735dcf628c 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -198,7 +198,7 @@ def verify_and_notify_new_payload(self: ExecutionEngine, Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ assert self.is_valid_block_hash(new_payload_request.execution_payload) - assert self.is_valid_versioned_hashes(new_payload_request) # [Modified in Deneb] + assert self.is_valid_versioned_hashes(new_payload_request) # [New in Deneb] assert self.notify_new_payload(new_payload_request.execution_payload) ... return True From 7a827638e66c982a341e2b10b303f772ea9d1f70 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 12:55:49 +0800 Subject: [PATCH 111/117] Ensure `verify_and_notify_new_payload` returns bool --- specs/bellatrix/beacon-chain.md | 7 ++++--- specs/deneb/beacon-chain.md | 14 ++++++++++---- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/specs/bellatrix/beacon-chain.md b/specs/bellatrix/beacon-chain.md index 6cc69d747f..6f67be96a7 100644 --- a/specs/bellatrix/beacon-chain.md +++ b/specs/bellatrix/beacon-chain.md @@ -355,9 +355,10 @@ def verify_and_notify_new_payload(self: ExecutionEngine, """ Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ - assert self.is_valid_block_hash(new_payload_request.execution_payload) - assert self.notify_new_payload(new_payload_request.execution_payload) - ... + if not self.is_valid_block_hash(new_payload_request.execution_payload): + return False + if not self.notify_new_payload(new_payload_request.execution_payload): + return False return True ``` diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index 735dcf628c..bcbf4d1d00 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -197,10 +197,16 @@ def verify_and_notify_new_payload(self: ExecutionEngine, """ Return ``True`` if and only if ``new_payload_request`` is valid with respect to ``self.execution_state``. """ - assert self.is_valid_block_hash(new_payload_request.execution_payload) - assert self.is_valid_versioned_hashes(new_payload_request) # [New in Deneb] - assert self.notify_new_payload(new_payload_request.execution_payload) - ... + if not self.is_valid_block_hash(new_payload_request.execution_payload): + return False + + # [New in Deneb] + if not self.is_valid_versioned_hashes(new_payload_request): + return False + + if not self.notify_new_payload(new_payload_request.execution_payload): + return False + return True ``` From f0a4281afdeeea8cc41c85405fb35203cf60463a Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 16:11:32 +0800 Subject: [PATCH 112/117] Add tests. Add validation in the p2p beacon block gossiping --- presets/mainnet/deneb.yaml | 2 ++ presets/minimal/deneb.yaml | 2 ++ specs/_features/eip6110/beacon-chain.md | 2 ++ specs/deneb/p2p-interface.md | 5 +++++ .../test_process_execution_payload.py | 13 ++++++++++++ .../eth2spec/test/deneb/sanity/test_blocks.py | 20 +++++++++++++++---- 6 files changed, 40 insertions(+), 4 deletions(-) diff --git a/presets/mainnet/deneb.yaml b/presets/mainnet/deneb.yaml index ebe33f2d10..10c5025ed8 100644 --- a/presets/mainnet/deneb.yaml +++ b/presets/mainnet/deneb.yaml @@ -4,5 +4,7 @@ # --------------------------------------------------------------- # `uint64(4096)` FIELD_ELEMENTS_PER_BLOB: 4096 +# `uint64(2**12)` (= 4096) +MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096 # `uint64(2**2)` (= 4) MAX_BLOBS_PER_BLOCK: 4 diff --git a/presets/minimal/deneb.yaml b/presets/minimal/deneb.yaml index e51b5587d0..91120f9dae 100644 --- a/presets/minimal/deneb.yaml +++ b/presets/minimal/deneb.yaml @@ -4,5 +4,7 @@ # --------------------------------------------------------------- # [customized] FIELD_ELEMENTS_PER_BLOB: 4 +# [customized] +MAX_BLOB_COMMITMENTS_PER_BLOCK: 16 # `uint64(2**2)` (= 4) MAX_BLOBS_PER_BLOCK: 4 diff --git a/specs/_features/eip6110/beacon-chain.md b/specs/_features/eip6110/beacon-chain.md index 5f6fd99cfa..e1384fdae0 100644 --- a/specs/_features/eip6110/beacon-chain.md +++ b/specs/_features/eip6110/beacon-chain.md @@ -244,6 +244,8 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi assert payload.prev_randao == get_randao_mix(state, get_current_epoch(state)) # Verify timestamp assert payload.timestamp == compute_timestamp_at_slot(state, state.slot) + # Verify commitments are under limit + assert len(body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK # Verify the execution payload is valid versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] assert execution_engine.verify_and_notify_new_payload( diff --git a/specs/deneb/p2p-interface.md b/specs/deneb/p2p-interface.md index d67c144b21..2d6454737f 100644 --- a/specs/deneb/p2p-interface.md +++ b/specs/deneb/p2p-interface.md @@ -117,6 +117,11 @@ Deneb introduces new global topics for blob sidecars. The *type* of the payload of this topic changes to the (modified) `SignedBeaconBlock` found in deneb. +New validation: + +- _[REJECT]_ The length of KZG commitments is less than or equal to the limitation defined in Consensus Layer -- + i.e. validate that `len(body.signed_beacon_block.message.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK` + ###### `blob_sidecar_{subnet_id}` This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`. diff --git a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py index df59385ab4..988070278d 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py +++ b/tests/core/pyspec/eth2spec/test/deneb/block_processing/test_process_execution_payload.py @@ -190,3 +190,16 @@ def test_invalid_correct_input__execution_invalid(spec, state): yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments, valid=False, execution_valid=False) + + +@with_deneb_and_later +@spec_state_test +def test_invalid_exceed_max_blobs_per_block(spec, state): + execution_payload = build_empty_execution_payload(spec, state) + + opaque_tx, _, blob_kzg_commitments, _ = get_sample_opaque_tx(spec, blob_count=spec.MAX_BLOBS_PER_BLOCK + 1) + + execution_payload.transactions = [opaque_tx] + execution_payload.block_hash = compute_el_block_hash(spec, execution_payload) + + yield from run_execution_payload_processing(spec, state, execution_payload, blob_kzg_commitments, valid=False) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 36caacd3fa..13a042b9f7 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -16,7 +16,7 @@ ) -def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1): +def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1, valid=True): yield 'pre', state block = build_empty_block_for_next_slot(spec, state) @@ -25,10 +25,16 @@ def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1): block.body.execution_payload.transactions = [opaque_tx] block.body.execution_payload.excess_data_gas = excess_data_gas block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - signed_block = state_transition_and_sign_block(spec, state, block) + + print(len(block.body.blob_kzg_commitments)) + + if valid: + signed_block = state_transition_and_sign_block(spec, state, block) + else: + signed_block = state_transition_and_sign_block(spec, state, block, expect_fail=True) yield 'blocks', [signed_block] - yield 'post', state + yield 'post', state if valid else None @with_deneb_and_later @@ -45,5 +51,11 @@ def test_one_blob(spec, state): @with_deneb_and_later @spec_state_test -def test_max_blobs(spec, state): +def test_max_blobs_per_block(spec, state): yield from run_block_with_blobs(spec, state, blob_count=spec.MAX_BLOBS_PER_BLOCK) + + +@with_deneb_and_later +@spec_state_test +def test_invalid_exceed_max_blobs_per_block(spec, state): + yield from run_block_with_blobs(spec, state, blob_count=spec.MAX_BLOBS_PER_BLOCK + 1, valid=False) From e9cc8dcc051ccff90916547027e6088b55083da5 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 16:32:39 +0800 Subject: [PATCH 113/117] PR feedback of Danny + verify `MAX_BLOBS_PER_BLOCK` size in unittest --- specs/deneb/beacon-chain.md | 2 +- .../test/deneb/unittests/test_config_invariants.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) create mode 100644 tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index c8b79f9193..cd02d24941 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -73,7 +73,7 @@ The blob transactions are packed into the execution payload by the EL/builder wi | Name | Value | | - | - | -| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | +| `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | | `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | ## Configuration diff --git a/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py new file mode 100644 index 0000000000..13a54225ed --- /dev/null +++ b/tests/core/pyspec/eth2spec/test/deneb/unittests/test_config_invariants.py @@ -0,0 +1,12 @@ +from eth2spec.test.context import ( + single_phase, + spec_test, + with_deneb_and_later, +) + + +@with_deneb_and_later +@spec_test +@single_phase +def test_length(spec): + assert spec.MAX_BLOBS_PER_BLOCK < spec.MAX_BLOB_COMMITMENTS_PER_BLOCK From 9bc27bad306c5b5be7c6e3381c8a01e10232bd91 Mon Sep 17 00:00:00 2001 From: djrtwo Date: Wed, 24 May 2023 05:41:42 -0600 Subject: [PATCH 114/117] a couple of minor cleanups --- specs/deneb/beacon-chain.md | 2 +- tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index cd02d24941..e1690c0622 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -234,7 +234,7 @@ def process_execution_payload(state: BeaconState, body: BeaconBlockBody, executi assert len(body.blob_kzg_commitments) <= MAX_BLOBS_PER_BLOCK # Verify the execution payload is valid - # [Modified in Deneb] Pass `versioned_hashes` to Engine API + # [Modified in Deneb] Pass `versioned_hashes` to Execution Engine versioned_hashes = [kzg_commitment_to_versioned_hash(commitment) for commitment in body.blob_kzg_commitments] assert execution_engine.verify_and_notify_new_payload( NewPayloadRequest(execution_payload=payload, versioned_hashes=versioned_hashes) diff --git a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py index 13a042b9f7..ba0797a048 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py +++ b/tests/core/pyspec/eth2spec/test/deneb/sanity/test_blocks.py @@ -26,8 +26,6 @@ def run_block_with_blobs(spec, state, blob_count, excess_data_gas=1, valid=True) block.body.execution_payload.excess_data_gas = excess_data_gas block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload) - print(len(block.body.blob_kzg_commitments)) - if valid: signed_block = state_transition_and_sign_block(spec, state, block) else: From 8d32e79b8d40993eeb6fa2cb67183682518bd3aa Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 20:54:22 +0800 Subject: [PATCH 115/117] Fix multiprocessing --- tests/generators/shuffling/main.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/tests/generators/shuffling/main.py b/tests/generators/shuffling/main.py index b85fd42a20..88c586b6c2 100644 --- a/tests/generators/shuffling/main.py +++ b/tests/generators/shuffling/main.py @@ -1,5 +1,5 @@ -from eth_utils import to_tuple from typing import Iterable +import random from eth2spec.gen_helpers.gen_base import gen_runner, gen_typing from eth2spec.test.helpers.typing import PresetBaseName @@ -8,6 +8,16 @@ from eth2spec.test.helpers.constants import PHASE0, MINIMAL, MAINNET +def generate_random_bytes(rng=random.Random(5566)): + random_bytes = bytes(rng.randint(0, 255) for _ in range(32)) + return random_bytes + + +# NOTE: somehow the random.Random generated seeds do not have pickle issue. +rng = random.Random(1234) +seeds = [generate_random_bytes(rng) for i in range(30)] + + def shuffling_case_fn(spec, seed, count): yield 'mapping', 'data', { 'seed': '0x' + seed.hex(), @@ -20,9 +30,8 @@ def shuffling_case(spec, seed, count): return f'shuffle_0x{seed.hex()}_{count}', lambda: shuffling_case_fn(spec, seed, count) -@to_tuple def shuffling_test_cases(spec): - for seed in [spec.hash(seed_init_value.to_bytes(length=4, byteorder='little')) for seed_init_value in range(30)]: + for seed in seeds: for count in [0, 1, 2, 3, 5, 10, 33, 100, 1000, 9999]: yield shuffling_case(spec, seed, count) @@ -47,11 +56,13 @@ def cases_fn() -> Iterable[gen_typing.TestCase]: handler_name='core', suite_name='shuffle', case_name=case_name, - case_fn=case_fn + case_fn=case_fn, ) return gen_typing.TestProvider(prepare=prepare_fn, make_cases=cases_fn) if __name__ == "__main__": - gen_runner.run_generator("shuffling", [create_provider(MINIMAL), create_provider(MAINNET)]) + gen_runner.run_generator("shuffling", [ + create_provider(MINIMAL), create_provider(MAINNET)] + ) From a10affa24580602670b319ffc5e349664b53b92e Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Wed, 24 May 2023 21:15:01 +0800 Subject: [PATCH 116/117] Fix preset table --- specs/deneb/beacon-chain.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/specs/deneb/beacon-chain.md b/specs/deneb/beacon-chain.md index e1690c0622..175a80edf5 100644 --- a/specs/deneb/beacon-chain.md +++ b/specs/deneb/beacon-chain.md @@ -71,8 +71,8 @@ The blob transactions are packed into the execution payload by the EL/builder wi ### Execution -| Name | Value | -| - | - | +| Name | Value | Description | +| - | - | - | | `MAX_BLOB_COMMITMENTS_PER_BLOCK` | `uint64(2**12)` (= 4096) | hardfork independent fixed theoretical limit same as `LIMIT_BLOBS_PER_TX` (see EIP 4844) | | `MAX_BLOBS_PER_BLOCK` | `uint64(2**2)` (= 4) | Maximum number of blobs in a single block limited by `MAX_BLOB_COMMITMENTS_PER_BLOCK` | From b2771c67c31f5635b4d87eefe7f3a03aeeddeb7c Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Thu, 25 May 2023 00:16:19 +0800 Subject: [PATCH 117/117] Fallback to MODE_SINGLE_PROCESS --- tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py index b8c3e591fb..2f462dddb1 100644 --- a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py +++ b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/settings.py @@ -5,7 +5,7 @@ MODE_SINGLE_PROCESS = 'MODE_SINGLE_PROCESS' MODE_MULTIPROCESSING = 'MODE_MULTIPROCESSING' # Test generator mode -GENERATOR_MODE = MODE_MULTIPROCESSING +GENERATOR_MODE = MODE_SINGLE_PROCESS # Number of subprocesses when using MODE_MULTIPROCESSING NUM_PROCESS = multiprocessing.cpu_count() // 2 - 1