Merge pull request #3889 from ethereum/dev

release `v1.5.0-alpha.5`

configs/mainnet.yaml

@@ -161,12 +161,10 @@ WHISK_PROPOSER_SELECTION_GAP: 2
 
 # EIP7594
 NUMBER_OF_COLUMNS: 128
-MAX_CELLS_IN_EXTENDED_MATRIX: 768
-DATA_COLUMN_SIDECAR_SUBNET_COUNT: 32
+DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128
 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384
 SAMPLES_PER_SLOT: 8
-CUSTODY_REQUIREMENT: 1
-TARGET_NUMBER_OF_PEERS: 70
+CUSTODY_REQUIREMENT: 4
 
 # [New in Electra:EIP7251]
 MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 128000000000 # 2**7 * 10**9 (= 128,000,000,000)

configs/minimal.yaml

@@ -160,12 +160,10 @@ WHISK_PROPOSER_SELECTION_GAP: 1
 
 # EIP7594
 NUMBER_OF_COLUMNS: 128
-MAX_CELLS_IN_EXTENDED_MATRIX: 768
-DATA_COLUMN_SIDECAR_SUBNET_COUNT: 32
+DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128
 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384
 SAMPLES_PER_SLOT: 8
-CUSTODY_REQUIREMENT: 1
-TARGET_NUMBER_OF_PEERS: 70
+CUSTODY_REQUIREMENT: 4
 
 # [New in Electra:EIP7251]
 MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 64000000000 # 2**6 * 10**9 (= 64,000,000,000)

pysetup/spec_builders/eip7594.py

@@ -12,14 +12,21 @@ def imports(cls, preset_name: str):
         return f'''
 from eth2spec.deneb import {preset_name} as deneb
 '''
+
+
+    @classmethod
+    def sundry_functions(cls) -> str:
+        return """
+def retrieve_column_sidecars(beacon_block_root: Root) -> Sequence[DataColumnSidecar]:
+    return []
+"""
 
     @classmethod
     def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
         return {
             'FIELD_ELEMENTS_PER_CELL': spec_object.preset_vars['FIELD_ELEMENTS_PER_CELL'].value,
             'FIELD_ELEMENTS_PER_EXT_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_EXT_BLOB'].value,
             'NUMBER_OF_COLUMNS': spec_object.config_vars['NUMBER_OF_COLUMNS'].value,
-            'MAX_CELLS_IN_EXTENDED_MATRIX': spec_object.config_vars['MAX_CELLS_IN_EXTENDED_MATRIX'].value,
         }
 
     @classmethod

specs/_features/eip7594/fork-choice.md

@@ -0,0 +1,97 @@
+# EIP-7594 -- Fork Choice
+
+## Table of contents
+<!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+- [Introduction](#introduction)
+- [Helpers](#helpers)
+  - [Modified `is_data_available`](#modified-is_data_available)
+- [Updated fork-choice handlers](#updated-fork-choice-handlers)
+  - [Modified `on_block`](#modified-on_block)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+<!-- /TOC -->
+
+## Introduction
+
+This is the modification of the fork choice accompanying EIP-7594.
+
+## Helpers
+
+### Modified `is_data_available`
+
+```python
+def is_data_available(beacon_block_root: Root) -> bool:
+    # `retrieve_column_sidecars` is implementation and context dependent, replacing
+    # `retrieve_blobs_and_proofs`. For the given block root, it returns all column 
+    # sidecars to sample, or raises an exception if they are not available. 
+    # The p2p network does not guarantee sidecar retrieval outside of 
+    # `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS` epochs.  
+    column_sidecars = retrieve_column_sidecars(beacon_block_root)
+    return all(
+        verify_data_column_sidecar_kzg_proofs(column_sidecar)
+        for column_sidecar in column_sidecars
+    )
+```
+
+## Updated fork-choice handlers
+
+### Modified `on_block`
+
+*Note*: The only modification is that `is_data_available` does not take `blob_kzg_commitments` as input.
+
+```python
+def on_block(store: Store, signed_block: SignedBeaconBlock) -> None:
+    """
+    Run ``on_block`` upon receiving a new block.
+    """
+    block = signed_block.message
+    # Parent block must be known
+    assert block.parent_root in store.block_states
+    # Make a copy of the state to avoid mutability issues
+    state = copy(store.block_states[block.parent_root])
+    # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past.
+    assert get_current_slot(store) >= block.slot
+
+    # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor)
+    finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch)
+    assert block.slot > finalized_slot
+    # Check block is a descendant of the finalized block at the checkpoint finalized slot
+    finalized_checkpoint_block = get_checkpoint_block(
+        store,
+        block.parent_root,
+        store.finalized_checkpoint.epoch,
+    )
+    assert store.finalized_checkpoint.root == finalized_checkpoint_block
+
+    # [Modified in EIP7594]
+    assert is_data_available(hash_tree_root(block))
+
+    # Check the block is valid and compute the post-state
+    block_root = hash_tree_root(block)
+    state_transition(state, signed_block, True)
+
+    # Add new block to the store
+    store.blocks[block_root] = block
+    # Add new state for this block to the store
+    store.block_states[block_root] = state
+
+    # Add block timeliness to the store
+    time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT
+    is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT
+    is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval
+    store.block_timeliness[hash_tree_root(block)] = is_timely
+
+    # Add proposer score boost if the block is timely and not conflicting with an existing block
+    is_first_block = store.proposer_boost_root == Root()
+    if is_timely and is_first_block:
+        store.proposer_boost_root = hash_tree_root(block)
+
+    # Update checkpoints in store if necessary
+    update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint)
+
+    # Eagerly compute unrealized justification and finality.
+    compute_pulled_up_tip(store, block_root)
+```

specs/_features/eip7594/peer-sampling.md

@@ -0,0 +1,118 @@
+# EIP-7594 -- Peer Sampling 
+
+**Notice**: This document is a work-in-progress for researchers and implementers.
+
+## Table of contents
+
+<!-- TOC -->
+<!-- START doctoc generated TOC please keep comment here to allow auto update -->
+<!-- DON'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->
+
+- [Introduction](#introduction)
+- [Helper functions](#helper-functions)
+  - [`get_extended_sample_count`](#get_extended_sample_count)
+- [Peer discovery](#peer-discovery)
+- [Peer sampling](#peer-sampling)
+  - [Sample selection](#sample-selection)
+  - [Sample queries](#sample-queries)
+- [Peer scoring](#peer-scoring)
+- [DAS providers](#das-providers)
+
+<!-- END doctoc generated TOC please keep comment here to allow auto update -->
+<!-- /TOC -->
+
+## Introduction
+
+The purpose of this document is to complement [EIP-7594 -- Data Availability Sampling Core](das-core.md) by specifying the peer sampling functionality of the full PeerDAS protocol. Initially, this functionality may not be implemented by all clients. In such cases, it is replaced by [subnet sampling](das-core.md#subnet-sampling), which is an extension of the custody component of the protocol.
+
+## Helper functions
+
+### `get_extended_sample_count`
+
+```python
+def get_extended_sample_count(allowed_failures: uint64) -> uint64:
+    assert 0 <= allowed_failures <= NUMBER_OF_COLUMNS // 2
+    """
+    Return the sample count if allowing failures.
+
+    This helper demonstrates how to calculate the number of columns to query per slot when
+    allowing given number of failures, assuming uniform random selection without replacement.
+    Nested functions are direct replacements of Python library functions math.comb and
+    scipy.stats.hypergeom.cdf, with the same signatures.
+    """
+
+    def math_comb(n: int, k: int) -> int:
+        if not 0 <= k <= n:
+            return 0
+        r = 1
+        for i in range(min(k, n - k)):
+            r = r * (n - i) // (i + 1)
+        return r
+
+    def hypergeom_cdf(k: uint64, M: uint64, n: uint64, N: uint64) -> float:
+        # NOTE: It contains float-point computations.
+        # Convert uint64 to Python integers before computations.
+        k = int(k)
+        M = int(M)
+        n = int(n)
+        N = int(N)
+        return sum([math_comb(n, i) * math_comb(M - n, N - i) / math_comb(M, N)
+                    for i in range(k + 1)])
+
+    worst_case_missing = NUMBER_OF_COLUMNS // 2 + 1
+    false_positive_threshold = hypergeom_cdf(0, NUMBER_OF_COLUMNS,
+                                             worst_case_missing, SAMPLES_PER_SLOT)
+    for sample_count in range(SAMPLES_PER_SLOT, NUMBER_OF_COLUMNS + 1):
+        if hypergeom_cdf(allowed_failures, NUMBER_OF_COLUMNS,
+                         worst_case_missing, sample_count) <= false_positive_threshold:
+            break
+    return sample_count
+```
+
+## Peer discovery
+
+At each slot, a node needs to be able to readily sample from *any* set of columns. To this end, a node SHOULD find and maintain a set of diverse and reliable peers that can regularly satisfy their sampling demands.
+
+A node runs a background peer discovery process, maintaining peers of various custody distributions (both `custody_size` and column assignments). The combination of advertised `custody_size` size and public node-id make this readily and publicly accessible. The peer set should cover the whole column space, with some redundancy. The number of peers, or at least the redundancy implied by the custody distributions over the the peer set, should be tuned upward in the event of failed sampling.
+
+*Note*: while high-capacity and super-full nodes are high value with respect to satisfying sampling requirements, a node SHOULD maintain a distribution across node capacities as to not centralize the p2p graph too much (in the extreme becomes hub/spoke) and to distribute sampling load better across all nodes.
+
+*Note*: A DHT-based peer discovery mechanism is expected to be utilized in the above. The beacon-chain network currently utilizes discv5 in a similar method as described for finding peers of particular distributions of attestation subnets. Additional peer discovery methods are valuable to integrate (e.g., latent peer discovery via libp2p gossipsub) to add a defense in breadth against one of the discovery methods being attacked.
+
+## Peer sampling
+
+### Sample selection
+
+At each slot, a node SHOULD select at least `SAMPLES_PER_SLOT` column IDs for sampling. It is recommended to use uniform random selection without replacement based on local randomness. Sampling is considered successful if the node manages to retrieve all selected columns.
+
+Alternatively, a node MAY use a method that selects more than `SAMPLES_PER_SLOT` columns while allowing some missing, respecting the same target false positive threshold (the probability of successful sampling of an unavailable block) as dictated by the `SAMPLES_PER_SLOT` parameter. If using uniform random selection without replacement, a node can use the `get_extended_sample_count(allowed_failures) -> sample_count` helper function to determine the sample count (number of unique column IDs) for any selected number of allowed failures. Sampling is then considered successful if any `sample_count - allowed_failures` columns are retrieved successfully.
+
+For reference, the table below shows the number of samples and the number of allowed missing columns assuming `NUMBER_OF_COLUMNS = 128` and `SAMPLES_PER_SLOT = 16`.
+
+| Allowed missing | 0| 1| 2| 3| 4| 5| 6| 7| 8|
+|-----------------|--|--|--|--|--|--|--|--|--|
+| Sample count    |16|20|24|27|29|32|35|37|40|
+
+### Sample queries
+
+A node SHOULD maintain a diverse set of peers for each column and each slot by verifying responsiveness to sample queries.
+
+A node SHOULD query for samples from selected peers via `DataColumnSidecarsByRoot` request. A node utilizes `get_custody_columns` helper to determine which peer(s) it could request from, identifying a list of candidate peers for each selected column.
+
+If more than one candidate peer is found for a given column, a node SHOULD randomize its peer selection to distribute sample query load in the network. Nodes MAY use peer scoring to tune this selection (for example, by using weighted selection or by using a cut-off threshold). If possible, it is also recommended to avoid requesting many columns from the same peer in order to avoid relying on and exposing the sample selection to a single peer.
+
+If a node already has a column because of custody, it is not required to send out queries for that column.
+
+If a node has enough good/honest peers across all columns, and the data is being made available, the above procedure has a high chance of success.
+
+## Peer scoring
+
+Due to the deterministic custody functions, a node knows exactly what a peer should be able to respond to. In the event that a peer does not respond to samples of their custodied rows/columns, a node may downscore or disconnect from a peer.
+
+## DAS providers
+
+A DAS provider is a consistently-available-for-DAS-queries, super-full (or high capacity) node. To the p2p, these look just like other nodes but with high advertised capacity, and they should generally be able to be latently found via normal discovery.
+
+DAS providers can also be found out-of-band and configured into a node to connect to directly and prioritize. Nodes can add some set of these to their local configuration for persistent connection to bolster their DAS quality of service.
+
+Such direct peering utilizes a feature supported out of the box today on all nodes and can complement (and reduce attackability and increase quality-of-service) alternative peer discovery mechanisms.

specs/capella/fork-choice.md

@@ -75,8 +75,6 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None:
     block = signed_block.message
     # Parent block must be known
     assert block.parent_root in store.block_states
-    # Make a copy of the state to avoid mutability issues
-    pre_state = copy(store.block_states[block.parent_root])
     # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past.
     assert get_current_slot(store) >= block.slot
 
@@ -92,7 +90,8 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None:
     assert store.finalized_checkpoint.root == finalized_checkpoint_block
 
     # Check the block is valid and compute the post-state
-    state = pre_state.copy()
+    # Make a copy of the state to avoid mutability issues
+    state = copy(store.block_states[block.parent_root])
     block_root = hash_tree_root(block)
     state_transition(state, signed_block, True)
 

specs/deneb/fork-choice.md

@@ -74,8 +74,6 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None:
     block = signed_block.message
     # Parent block must be known
     assert block.parent_root in store.block_states
-    # Make a copy of the state to avoid mutability issues
-    pre_state = copy(store.block_states[block.parent_root])
     # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past.
     assert get_current_slot(store) >= block.slot
 
@@ -98,7 +96,8 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None:
     assert is_data_available(hash_tree_root(block), block.body.blob_kzg_commitments)
 
     # Check the block is valid and compute the post-state
-    state = pre_state.copy()
+    # Make a copy of the state to avoid mutability issues
+    state = copy(store.block_states[block.parent_root])
     block_root = hash_tree_root(block)
     state_transition(state, signed_block, True)
 

tests/core/pyspec/eth2spec/VERSION.txt

@@ -1 +1 @@
-1.5.0-alpha.4
+1.5.0-alpha.5

tests/core/pyspec/eth2spec/test/context.py

@@ -429,6 +429,13 @@ def decorator(fn):
     return decorator
 
 
+def with_all_phases_from_except(earliest_phase, except_phases=None):
+    """
+    A decorator factory for running a tests with every phase except the ones listed
+    """
+    return with_all_phases_from(earliest_phase, [phase for phase in ALL_PHASES if phase not in except_phases])
+
+
 def with_all_phases_except(exclusion_phases):
     """
     A decorator factory for running a tests with every phase except the ones listed

tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py

@@ -2,7 +2,12 @@
 
 from eth2spec.test.context import (
     spec_state_test,
-    with_deneb_and_later,
+    with_all_phases_from_except,
+)
+
+from eth2spec.test.helpers.constants import (
+    DENEB,
+    EIP7594,
 )
 
 from eth2spec.test.helpers.block import (
@@ -34,7 +39,7 @@ def get_block_with_blob(spec, state, rng=None):
     return block, blobs, blob_kzg_proofs
 
 
-@with_deneb_and_later
+@with_all_phases_from_except(DENEB, [EIP7594])
 @spec_state_test
 def test_simple_blob_data(spec, state):
     rng = Random(1234)
@@ -69,7 +74,7 @@ def test_simple_blob_data(spec, state):
     yield 'steps', test_steps
 
 
-@with_deneb_and_later
+@with_all_phases_from_except(DENEB, [EIP7594])
 @spec_state_test
 def test_invalid_incorrect_proof(spec, state):
     rng = Random(1234)
@@ -97,7 +102,7 @@ def test_invalid_incorrect_proof(spec, state):
     yield 'steps', test_steps
 
 
-@with_deneb_and_later
+@with_all_phases_from_except(DENEB, [EIP7594])
 @spec_state_test
 def test_invalid_data_unavailable(spec, state):
     rng = Random(1234)
@@ -125,7 +130,7 @@ def test_invalid_data_unavailable(spec, state):
     yield 'steps', test_steps
 
 
-@with_deneb_and_later
+@with_all_phases_from_except(DENEB, [EIP7594])
 @spec_state_test
 def test_invalid_wrong_proofs_length(spec, state):
     rng = Random(1234)
@@ -153,7 +158,7 @@ def test_invalid_wrong_proofs_length(spec, state):
     yield 'steps', test_steps
 
 
-@with_deneb_and_later
+@with_all_phases_from_except(DENEB, [EIP7594])
 @spec_state_test
 def test_invalid_wrong_blobs_length(spec, state):
     rng = Random(1234)