From 4bd2f924de52aea1a232aedb41a5394675e895ff Mon Sep 17 00:00:00 2001 From: fradamt Date: Wed, 7 Aug 2024 11:28:16 +0200 Subject: [PATCH 01/14] custody-based fork-choice --- pysetup/spec_builders/eip7594.py | 8 ++ specs/_features/eip7594/fork-choice.md | 122 +++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 specs/_features/eip7594/fork-choice.md diff --git a/pysetup/spec_builders/eip7594.py b/pysetup/spec_builders/eip7594.py index 3329378320..dfbc427a70 100644 --- a/pysetup/spec_builders/eip7594.py +++ b/pysetup/spec_builders/eip7594.py @@ -12,6 +12,14 @@ def imports(cls, preset_name: str): return f''' from eth2spec.deneb import {preset_name} as deneb ''' + + + @classmethod + def sundry_functions(cls) -> str: + return """ +def retrieve_column_sidecars(beacon_block_root: Root) -> Sequence[DataColumnSidecar]: + return [] +""" @classmethod def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md new file mode 100644 index 0000000000..cc4caa36c5 --- /dev/null +++ b/specs/_features/eip7594/fork-choice.md @@ -0,0 +1,122 @@ +# EIP-7594 -- Fork Choice + +## Table of contents + + + + +- [Introduction](#introduction) + - [Helpers](#helpers) + - [Modified `is_data_available`](#modified-is_data_available) + - [New `is_chain_available`](#new-is_chain_available) + - [Modified `get_head`](#modified-get_head) + - [New `is_peer_sampling_required`](#new-is_peer_sampling_required) +- [Updated fork-choice handlers](#updated-fork-choice-handlers) + - [Modified `on_block`](#modified-on_block) + - [Pull-up tip helpers](#pull-up-tip-helpers) + - [Modified `compute_pulled_up_tip`](#modified-compute_pulled_up_tip) + + + + +## Introduction + +This is the modification of the fork choice accompanying EIP-7594. + +### Helpers + +#### Modified `is_data_available` + +```python +def is_data_available(beacon_block_root: Root) -> bool: + # `retrieve_column_sidecars` is implementation and context dependent, replacing `retrieve_blobs_and_proofs`. + # For the given block root, it returns all column sidecars to custody, or raises an exception if they are not available. # The p2p network does not guarantee sidecar retrieval outside of `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS` epochs. + column_sidecars = retrieve_column_sidecars(beacon_block_root) + return all( + verify_data_column_sidecar_kzg_proofs(column_sidecar) + for column_sidecar in column_sidecars + ) +``` + +#### Modified `get_head` + +*Note*: children of the current `head` are required to be available in order to be considered by the fork-choice. + +```python +def get_head(store: Store) -> Root: + # Get filtered block tree that only includes viable branches + blocks = get_filtered_block_tree(store) + # Execute the LMD-GHOST fork choice + head = store.justified_checkpoint.root + while True: + # Get available children for the current slot + children = [ + root for (root, block) in blocks.items() + if ( + block.parent_root == head + and is_data_available(root) + ) + ] + if len(children) == 0: + return head + # Sort by latest attesting balance with ties broken lexicographically + # Ties broken by favoring block with lexicographically higher root + head = max(children, key=lambda root: (get_weight(store, root), root)) +``` + +## Updated fork-choice handlers + +### Modified `on_block` + +*Note*: The blob data availability check is removed. + +```python +def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: + """ + Run ``on_block`` upon receiving a new block. + """ + block = signed_block.message + # Parent block must be known + assert block.parent_root in store.block_states + # Make a copy of the state to avoid mutability issues + state = copy(store.block_states[block.parent_root]) + # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. + assert get_current_slot(store) >= block.slot + + # Check that block is later than the finalized epoch slot (optimization to reduce calls to get_ancestor) + finalized_slot = compute_start_slot_at_epoch(store.finalized_checkpoint.epoch) + assert block.slot > finalized_slot + # Check block is a descendant of the finalized block at the checkpoint finalized slot + finalized_checkpoint_block = get_checkpoint_block( + store, + block.parent_root, + store.finalized_checkpoint.epoch, + ) + assert store.finalized_checkpoint.root == finalized_checkpoint_block + + # Check the block is valid and compute the post-state + block_root = hash_tree_root(block) + state_transition(state, signed_block, True) + + # Add new block to the store + store.blocks[block_root] = block + # Add new state for this block to the store + store.block_states[block_root] = state + + # Add block timeliness to the store + time_into_slot = (store.time - store.genesis_time) % SECONDS_PER_SLOT + is_before_attesting_interval = time_into_slot < SECONDS_PER_SLOT // INTERVALS_PER_SLOT + is_timely = get_current_slot(store) == block.slot and is_before_attesting_interval + store.block_timeliness[hash_tree_root(block)] = is_timely + + # Add proposer score boost if the block is timely and not conflicting with an existing block + is_first_block = store.proposer_boost_root == Root() + if is_timely and is_first_block: + store.proposer_boost_root = hash_tree_root(block) + + # Update checkpoints in store if necessary + update_checkpoints(store, state.current_justified_checkpoint, state.finalized_checkpoint) + + # Eagerly compute unrealized justification and finality. + compute_pulled_up_tip(store, block_root) +``` From 376ab85406d20d60f3cde7a4a7d84b47ea3128a0 Mon Sep 17 00:00:00 2001 From: fradamt Date: Wed, 7 Aug 2024 14:52:36 +0200 Subject: [PATCH 02/14] Split peer sampling spec from das-core.md --- specs/_features/eip7594/das-core.md | 118 +---------------------- specs/_features/eip7594/fork-choice.md | 14 +-- specs/_features/eip7594/peer-sampling.md | 115 ++++++++++++++++++++++ 3 files changed, 127 insertions(+), 120 deletions(-) create mode 100644 specs/_features/eip7594/peer-sampling.md diff --git a/specs/_features/eip7594/das-core.md b/specs/_features/eip7594/das-core.md index 97bce0d0eb..18a12e3980 100644 --- a/specs/_features/eip7594/das-core.md +++ b/specs/_features/eip7594/das-core.md @@ -23,21 +23,14 @@ - [`compute_extended_matrix`](#compute_extended_matrix) - [`recover_matrix`](#recover_matrix) - [`get_data_column_sidecars`](#get_data_column_sidecars) - - [`get_extended_sample_count`](#get_extended_sample_count) - [Custody](#custody) - [Custody requirement](#custody-requirement) - [Public, deterministic selection](#public-deterministic-selection) -- [Peer discovery](#peer-discovery) +- [Subnet sampling](#subnet-sampling) - [Extended data](#extended-data) - [Column gossip](#column-gossip) - [Parameters](#parameters) -- [Peer sampling](#peer-sampling) - - [Sample selection](#sample-selection) - - [Sample queries](#sample-queries) -- [Peer scoring](#peer-scoring) - [Reconstruction and cross-seeding](#reconstruction-and-cross-seeding) -- [DAS providers](#das-providers) -- [A note on fork choice](#a-note-on-fork-choice) - [FAQs](#faqs) - [Row (blob) custody](#row-blob-custody) - [Subnet stability](#subnet-stability) @@ -75,15 +68,14 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | Description | | - | - | - | -| `DATA_COLUMN_SIDECAR_SUBNET_COUNT` | `32` | The number of data column sidecar subnets used in the gossipsub protocol | +| `DATA_COLUMN_SIDECAR_SUBNET_COUNT` | `128` | The number of data column sidecar subnets used in the gossipsub protocol | ### Custody setting | Name | Value | Description | | - | - | - | | `SAMPLES_PER_SLOT` | `8` | Number of `DataColumnSidecar` random samples a node queries per slot | -| `CUSTODY_REQUIREMENT` | `1` | Minimum number of subnets an honest node custodies and serves samples from | -| `TARGET_NUMBER_OF_PEERS` | `70` | Suggested minimum peer count | +| `CUSTODY_REQUIREMENT` | `4` | Minimum number of subnets an honest node custodies and serves samples from | ### Containers @@ -227,48 +219,6 @@ def get_data_column_sidecars(signed_block: SignedBeaconBlock, return sidecars ``` -#### `get_extended_sample_count` - -```python -def get_extended_sample_count(allowed_failures: uint64) -> uint64: - assert 0 <= allowed_failures <= NUMBER_OF_COLUMNS // 2 - """ - Return the sample count if allowing failures. - - This helper demonstrates how to calculate the number of columns to query per slot when - allowing given number of failures, assuming uniform random selection without replacement. - Nested functions are direct replacements of Python library functions math.comb and - scipy.stats.hypergeom.cdf, with the same signatures. - """ - - def math_comb(n: int, k: int) -> int: - if not 0 <= k <= n: - return 0 - r = 1 - for i in range(min(k, n - k)): - r = r * (n - i) // (i + 1) - return r - - def hypergeom_cdf(k: uint64, M: uint64, n: uint64, N: uint64) -> float: - # NOTE: It contains float-point computations. - # Convert uint64 to Python integers before computations. - k = int(k) - M = int(M) - n = int(n) - N = int(N) - return sum([math_comb(n, i) * math_comb(M - n, N - i) / math_comb(M, N) - for i in range(k + 1)]) - - worst_case_missing = NUMBER_OF_COLUMNS // 2 + 1 - false_positive_threshold = hypergeom_cdf(0, NUMBER_OF_COLUMNS, - worst_case_missing, SAMPLES_PER_SLOT) - for sample_count in range(SAMPLES_PER_SLOT, NUMBER_OF_COLUMNS + 1): - if hypergeom_cdf(allowed_failures, NUMBER_OF_COLUMNS, - worst_case_missing, sample_count) <= false_positive_threshold: - break - return sample_count -``` - ## Custody ### Custody requirement @@ -285,17 +235,9 @@ The particular columns that a node custodies are selected pseudo-randomly as a f *Note*: increasing the `custody_size` parameter for a given `node_id` extends the returned list (rather than being an entirely new shuffle) such that if `custody_size` is unknown, the default `CUSTODY_REQUIREMENT` will be correct for a subset of the node's custody. -## Peer discovery - -At each slot, a node needs to be able to readily sample from *any* set of columns. To this end, a node SHOULD find and maintain a set of diverse and reliable peers that can regularly satisfy their sampling demands. +## Subnet sampling -A node runs a background peer discovery process, maintaining at least `TARGET_NUMBER_OF_PEERS` of various custody distributions (both `custody_size` and column assignments). The combination of advertised `custody_size` size and public node-id make this readily and publicly accessible. - -`TARGET_NUMBER_OF_PEERS` should be tuned upward in the event of failed sampling. - -*Note*: while high-capacity and super-full nodes are high value with respect to satisfying sampling requirements, a node SHOULD maintain a distribution across node capacities as to not centralize the p2p graph too much (in the extreme becomes hub/spoke) and to distribute sampling load better across all nodes. - -*Note*: A DHT-based peer discovery mechanism is expected to be utilized in the above. The beacon-chain network currently utilizes discv5 in a similar method as described for finding peers of particular distributions of attestation subnets. Additional peer discovery methods are valuable to integrate (e.g., latent peer discovery via libp2p gossipsub) to add a defense in breadth against one of the discovery methods being attacked. +At each slot, a node advertising `custody_subnet_count` downloads a minimum of `subnet_sampling_size = max(SAMPLES_PER_SLOT, custody_subnet_count)` total subnets. The corresponding set of columns is selected by `get_custody_columns(node_id, subnet_sampling_size)`, so that in particular the subset of columns to custody is consistent with the output of `get_custody_columns(node_id, custody_subnet_count)`. Sampling is considered successful if the node manages to retrieve all selected columns. ## Extended data @@ -309,36 +251,6 @@ For each column -- use `data_column_sidecar_{subnet_id}` subnets, where `subnet_ Verifiable samples from their respective column are distributed on the assigned subnet. To custody a particular column, a node joins the respective gossipsub subnet. If a node fails to get a column on the column subnet, a node can also utilize the Req/Resp protocol to query the missing column from other peers. -## Peer sampling - -### Sample selection - -At each slot, a node SHOULD select at least `SAMPLES_PER_SLOT` column IDs for sampling. It is recommended to use uniform random selection without replacement based on local randomness. Sampling is considered successful if the node manages to retrieve all selected columns. - -Alternatively, a node MAY use a method that selects more than `SAMPLES_PER_SLOT` columns while allowing some missing, respecting the same target false positive threshold (the probability of successful sampling of an unavailable block) as dictated by the `SAMPLES_PER_SLOT` parameter. If using uniform random selection without replacement, a node can use the `get_extended_sample_count(allowed_failures) -> sample_count` helper function to determine the sample count (number of unique column IDs) for any selected number of allowed failures. Sampling is then considered successful if any `sample_count - allowed_failures` columns are retrieved successfully. - -For reference, the table below shows the number of samples and the number of allowed missing columns assuming `NUMBER_OF_COLUMNS = 128` and `SAMPLES_PER_SLOT = 16`. - -| Allowed missing | 0| 1| 2| 3| 4| 5| 6| 7| 8| -|-----------------|--|--|--|--|--|--|--|--|--| -| Sample count |16|20|24|27|29|32|35|37|40| - -### Sample queries - -A node SHOULD maintain a diverse set of peers for each column and each slot by verifying responsiveness to sample queries. - -A node SHOULD query for samples from selected peers via `DataColumnSidecarsByRoot` request. A node utilizes `get_custody_columns` helper to determine which peer(s) it could request from, identifying a list of candidate peers for each selected column. - -If more than one candidate peer is found for a given column, a node SHOULD randomize its peer selection to distribute sample query load in the network. Nodes MAY use peer scoring to tune this selection (for example, by using weighted selection or by using a cut-off threshold). If possible, it is also recommended to avoid requesting many columns from the same peer in order to avoid relying on and exposing the sample selection to a single peer. - -If a node already has a column because of custody, it is not required to send out queries for that column. - -If a node has enough good/honest peers across all columns, and the data is being made available, the above procedure has a high chance of success. - -## Peer scoring - -Due to the deterministic custody functions, a node knows exactly what a peer should be able to respond to. In the event that a peer does not respond to samples of their custodied rows/columns, a node may downscore or disconnect from a peer. - ## Reconstruction and cross-seeding If the node obtains 50%+ of all the columns, it SHOULD reconstruct the full data matrix via `recover_matrix` helper. Nodes MAY delay this reconstruction allowing time for other columns to arrive over the network. If delaying reconstruction, nodes may use a random delay in order to desynchronize reconstruction among nodes, thus reducing overall CPU load. @@ -351,26 +263,6 @@ Once the node obtains a column through reconstruction, the node MUST expose the *Note*: There may be anti-DoS and quality-of-service considerations around how to send samples and consider samples -- is each individual sample a message or are they sent in aggregate forms. -## DAS providers - -A DAS provider is a consistently-available-for-DAS-queries, super-full (or high capacity) node. To the p2p, these look just like other nodes but with high advertised capacity, and they should generally be able to be latently found via normal discovery. - -DAS providers can also be found out-of-band and configured into a node to connect to directly and prioritize. Nodes can add some set of these to their local configuration for persistent connection to bolster their DAS quality of service. - -Such direct peering utilizes a feature supported out of the box today on all nodes and can complement (and reduce attackability and increase quality-of-service) alternative peer discovery mechanisms. - -## A note on fork choice - -*Fork choice spec TBD, but it will just be a replacement of `is_data_available()` call in Deneb with column sampling instead of full download. Note the `is_data_available(slot_N)` will likely do a `-1` follow distance so that you just need to check the availability of slot `N-1` for slot `N` (starting with the block proposer of `N`).* - -The fork choice rule (essentially a DA filter) is *orthogonal to a given DAS design*, other than the efficiency of a particular design impacting it. - -In any DAS design, there are probably a few degrees of freedom around timing, acceptability of short-term re-orgs, etc. - -For example, the fork choice rule might require validators to do successful DAS on slot `N` to be able to include block of slot `N` in its fork choice. That's the tightest DA filter. But trailing filters are also probably acceptable, knowing that there might be some failures/short re-orgs but that they don't hurt the aggregate security. For example, the rule could be — DAS must be completed for slot N-1 for a child block in N to be included in the fork choice. - -Such trailing techniques and their analysis will be valuable for any DAS construction. The question is — can you relax how quickly you need to do DA and in the worst case not confirm unavailable data via attestations/finality, and what impact does it have on short-term re-orgs and fast confirmation rules. - ## FAQs ### Row (blob) custody diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index cc4caa36c5..82d52b20cb 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -8,13 +8,9 @@ - [Introduction](#introduction) - [Helpers](#helpers) - [Modified `is_data_available`](#modified-is_data_available) - - [New `is_chain_available`](#new-is_chain_available) - [Modified `get_head`](#modified-get_head) - - [New `is_peer_sampling_required`](#new-is_peer_sampling_required) - [Updated fork-choice handlers](#updated-fork-choice-handlers) - [Modified `on_block`](#modified-on_block) - - [Pull-up tip helpers](#pull-up-tip-helpers) - - [Modified `compute_pulled_up_tip`](#modified-compute_pulled_up_tip) @@ -29,8 +25,11 @@ This is the modification of the fork choice accompanying EIP-7594. ```python def is_data_available(beacon_block_root: Root) -> bool: - # `retrieve_column_sidecars` is implementation and context dependent, replacing `retrieve_blobs_and_proofs`. - # For the given block root, it returns all column sidecars to custody, or raises an exception if they are not available. # The p2p network does not guarantee sidecar retrieval outside of `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS` epochs. + # `retrieve_column_sidecars` is implementation and context dependent, replacing + # `retrieve_blobs_and_proofs`. For the given block root, it returns all column + # sidecars to custody, or raises an exception if they are not available. + # The p2p network does not guarantee sidecar retrieval outside of + # `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS` epochs. column_sidecars = retrieve_column_sidecars(beacon_block_root) return all( verify_data_column_sidecar_kzg_proofs(column_sidecar) @@ -68,7 +67,8 @@ def get_head(store: Store) -> Root: ### Modified `on_block` -*Note*: The blob data availability check is removed. +*Note*: The blob data availability check is removed. We import blocks regardless +of their availability status, and move all filtering to the fork-choice (`get_head`). ```python def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: diff --git a/specs/_features/eip7594/peer-sampling.md b/specs/_features/eip7594/peer-sampling.md new file mode 100644 index 0000000000..26aa7ab99d --- /dev/null +++ b/specs/_features/eip7594/peer-sampling.md @@ -0,0 +1,115 @@ +# EIP-7594 -- Peer Sampling + +**Notice**: This document is a work-in-progress for researchers and implementers. + +## Table of contents + + + + + + - [Helper functions](#helper-functions) + - [`get_extended_sample_count`](#get_extended_sample_count) +- [Peer discovery](#peer-discovery) +- [Peer sampling](#peer-sampling) + - [Sample selection](#sample-selection) + - [Sample queries](#sample-queries) +- [Peer scoring](#peer-scoring) +- [DAS providers](#das-providers) + + + + +The purpose of this document is to complement [EIP-7594 -- Data Availability Sampling Core](das-core.md), by specifying the peer sampling functionality of the full PeerDAS protocol. This functionality may initially not be implemented, or only implemented by some clients, in which case it is replaced by [subnet sampling](das-core.md#subnet-sampling), an extension of the custody component of the protocol. + +### Helper functions + +#### `get_extended_sample_count` + +```python +def get_extended_sample_count(allowed_failures: uint64) -> uint64: + assert 0 <= allowed_failures <= NUMBER_OF_COLUMNS // 2 + """ + Return the sample count if allowing failures. + + This helper demonstrates how to calculate the number of columns to query per slot when + allowing given number of failures, assuming uniform random selection without replacement. + Nested functions are direct replacements of Python library functions math.comb and + scipy.stats.hypergeom.cdf, with the same signatures. + """ + + def math_comb(n: int, k: int) -> int: + if not 0 <= k <= n: + return 0 + r = 1 + for i in range(min(k, n - k)): + r = r * (n - i) // (i + 1) + return r + + def hypergeom_cdf(k: uint64, M: uint64, n: uint64, N: uint64) -> float: + # NOTE: It contains float-point computations. + # Convert uint64 to Python integers before computations. + k = int(k) + M = int(M) + n = int(n) + N = int(N) + return sum([math_comb(n, i) * math_comb(M - n, N - i) / math_comb(M, N) + for i in range(k + 1)]) + + worst_case_missing = NUMBER_OF_COLUMNS // 2 + 1 + false_positive_threshold = hypergeom_cdf(0, NUMBER_OF_COLUMNS, + worst_case_missing, SAMPLES_PER_SLOT) + for sample_count in range(SAMPLES_PER_SLOT, NUMBER_OF_COLUMNS + 1): + if hypergeom_cdf(allowed_failures, NUMBER_OF_COLUMNS, + worst_case_missing, sample_count) <= false_positive_threshold: + break + return sample_count +``` + +## Peer discovery + +At each slot, a node needs to be able to readily sample from *any* set of columns. To this end, a node SHOULD find and maintain a set of diverse and reliable peers that can regularly satisfy their sampling demands. + +A node runs a background peer discovery process, maintaining peers of various custody distributions (both `custody_size` and column assignments). The combination of advertised `custody_size` size and public node-id make this readily and publicly accessible. The peer set should cover the whole column space, with some redundancy. The number of peers, or at least the redundancy implied by the custody distributions over the the peer set, should be tuned upward in the event of failed sampling. + +*Note*: while high-capacity and super-full nodes are high value with respect to satisfying sampling requirements, a node SHOULD maintain a distribution across node capacities as to not centralize the p2p graph too much (in the extreme becomes hub/spoke) and to distribute sampling load better across all nodes. + +*Note*: A DHT-based peer discovery mechanism is expected to be utilized in the above. The beacon-chain network currently utilizes discv5 in a similar method as described for finding peers of particular distributions of attestation subnets. Additional peer discovery methods are valuable to integrate (e.g., latent peer discovery via libp2p gossipsub) to add a defense in breadth against one of the discovery methods being attacked. + +## Peer sampling + +### Sample selection + +At each slot, a node SHOULD select at least `SAMPLES_PER_SLOT` column IDs for sampling. It is recommended to use uniform random selection without replacement based on local randomness. Sampling is considered successful if the node manages to retrieve all selected columns. + +Alternatively, a node MAY use a method that selects more than `SAMPLES_PER_SLOT` columns while allowing some missing, respecting the same target false positive threshold (the probability of successful sampling of an unavailable block) as dictated by the `SAMPLES_PER_SLOT` parameter. If using uniform random selection without replacement, a node can use the `get_extended_sample_count(allowed_failures) -> sample_count` helper function to determine the sample count (number of unique column IDs) for any selected number of allowed failures. Sampling is then considered successful if any `sample_count - allowed_failures` columns are retrieved successfully. + +For reference, the table below shows the number of samples and the number of allowed missing columns assuming `NUMBER_OF_COLUMNS = 128` and `SAMPLES_PER_SLOT = 16`. + +| Allowed missing | 0| 1| 2| 3| 4| 5| 6| 7| 8| +|-----------------|--|--|--|--|--|--|--|--|--| +| Sample count |16|20|24|27|29|32|35|37|40| + +### Sample queries + +A node SHOULD maintain a diverse set of peers for each column and each slot by verifying responsiveness to sample queries. + +A node SHOULD query for samples from selected peers via `DataColumnSidecarsByRoot` request. A node utilizes `get_custody_columns` helper to determine which peer(s) it could request from, identifying a list of candidate peers for each selected column. + +If more than one candidate peer is found for a given column, a node SHOULD randomize its peer selection to distribute sample query load in the network. Nodes MAY use peer scoring to tune this selection (for example, by using weighted selection or by using a cut-off threshold). If possible, it is also recommended to avoid requesting many columns from the same peer in order to avoid relying on and exposing the sample selection to a single peer. + +If a node already has a column because of custody, it is not required to send out queries for that column. + +If a node has enough good/honest peers across all columns, and the data is being made available, the above procedure has a high chance of success. + +## Peer scoring + +Due to the deterministic custody functions, a node knows exactly what a peer should be able to respond to. In the event that a peer does not respond to samples of their custodied rows/columns, a node may downscore or disconnect from a peer. + +## DAS providers + +A DAS provider is a consistently-available-for-DAS-queries, super-full (or high capacity) node. To the p2p, these look just like other nodes but with high advertised capacity, and they should generally be able to be latently found via normal discovery. + +DAS providers can also be found out-of-band and configured into a node to connect to directly and prioritize. Nodes can add some set of these to their local configuration for persistent connection to bolster their DAS quality of service. + +Such direct peering utilizes a feature supported out of the box today on all nodes and can complement (and reduce attackability and increase quality-of-service) alternative peer discovery mechanisms. \ No newline at end of file From 111123723af8693611519846016fa841ccaf7096 Mon Sep 17 00:00:00 2001 From: fradamt Date: Wed, 7 Aug 2024 15:07:37 +0200 Subject: [PATCH 03/14] fix function comment --- specs/_features/eip7594/fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index 82d52b20cb..1c2fe89c12 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -27,7 +27,7 @@ This is the modification of the fork choice accompanying EIP-7594. def is_data_available(beacon_block_root: Root) -> bool: # `retrieve_column_sidecars` is implementation and context dependent, replacing # `retrieve_blobs_and_proofs`. For the given block root, it returns all column - # sidecars to custody, or raises an exception if they are not available. + # sidecars to sample, or raises an exception if they are not available. # The p2p network does not guarantee sidecar retrieval outside of # `MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS` epochs. column_sidecars = retrieve_column_sidecars(beacon_block_root) From 37c08a3081aecc9f380619bc654c7755921f01a6 Mon Sep 17 00:00:00 2001 From: fradamt Date: Wed, 7 Aug 2024 16:34:03 +0200 Subject: [PATCH 04/14] don't run deneb on_block tests for peerdas (changed behavior) --- .../test/deneb/fork_choice/test_on_block.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py index f8ffa62346..83f7346e31 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py @@ -2,9 +2,11 @@ from eth2spec.test.context import ( spec_state_test, - with_deneb_and_later, + with_phases, ) +from eth2spec.test.helpers.constants import DENEB + from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, ) @@ -34,7 +36,7 @@ def get_block_with_blob(spec, state, rng=None): return block, blobs, blob_kzg_proofs -@with_deneb_and_later +@with_phases([DENEB]) @spec_state_test def test_simple_blob_data(spec, state): rng = Random(1234) @@ -69,7 +71,7 @@ def test_simple_blob_data(spec, state): yield 'steps', test_steps -@with_deneb_and_later +@with_phases([DENEB]) @spec_state_test def test_invalid_incorrect_proof(spec, state): rng = Random(1234) @@ -97,7 +99,7 @@ def test_invalid_incorrect_proof(spec, state): yield 'steps', test_steps -@with_deneb_and_later +@with_phases([DENEB]) @spec_state_test def test_invalid_data_unavailable(spec, state): rng = Random(1234) @@ -125,7 +127,7 @@ def test_invalid_data_unavailable(spec, state): yield 'steps', test_steps -@with_deneb_and_later +@with_phases([DENEB]) @spec_state_test def test_invalid_wrong_proofs_length(spec, state): rng = Random(1234) @@ -153,7 +155,7 @@ def test_invalid_wrong_proofs_length(spec, state): yield 'steps', test_steps -@with_deneb_and_later +@with_phases([DENEB]) @spec_state_test def test_invalid_wrong_blobs_length(spec, state): rng = Random(1234) @@ -178,4 +180,4 @@ def test_invalid_wrong_blobs_length(spec, state): assert spec.get_head(store) != signed_block.message.hash_tree_root() - yield 'steps', test_steps + yield 'steps', test_steps \ No newline at end of file From 7a6770a732c35a843aa7d1320f0741e34a09c779 Mon Sep 17 00:00:00 2001 From: fradamt Date: Wed, 7 Aug 2024 16:52:06 +0200 Subject: [PATCH 05/14] fix headers Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- specs/_features/eip7594/das-core.md | 20 +++++++++---------- specs/_features/eip7594/fork-choice.md | 12 +++++------ specs/_features/eip7594/peer-sampling.md | 11 ++++++---- .../test/deneb/fork_choice/test_on_block.py | 2 +- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/specs/_features/eip7594/das-core.md b/specs/_features/eip7594/das-core.md index 18a12e3980..3d43db4830 100644 --- a/specs/_features/eip7594/das-core.md +++ b/specs/_features/eip7594/das-core.md @@ -18,11 +18,11 @@ - [Containers](#containers) - [`DataColumnSidecar`](#datacolumnsidecar) - [`MatrixEntry`](#matrixentry) - - [Helper functions](#helper-functions) - - [`get_custody_columns`](#get_custody_columns) - - [`compute_extended_matrix`](#compute_extended_matrix) - - [`recover_matrix`](#recover_matrix) - - [`get_data_column_sidecars`](#get_data_column_sidecars) +- [Helper functions](#helper-functions) + - [`get_custody_columns`](#get_custody_columns) + - [`compute_extended_matrix`](#compute_extended_matrix) + - [`recover_matrix`](#recover_matrix) + - [`get_data_column_sidecars`](#get_data_column_sidecars) - [Custody](#custody) - [Custody requirement](#custody-requirement) - [Public, deterministic selection](#public-deterministic-selection) @@ -101,9 +101,9 @@ class MatrixEntry(Container): row_index: RowIndex ``` -### Helper functions +## Helper functions -#### `get_custody_columns` +### `get_custody_columns` ```python def get_custody_columns(node_id: NodeID, custody_subnet_count: uint64) -> Sequence[ColumnIndex]: @@ -133,7 +133,7 @@ def get_custody_columns(node_id: NodeID, custody_subnet_count: uint64) -> Sequen ]) ``` -#### `compute_extended_matrix` +### `compute_extended_matrix` ```python def compute_extended_matrix(blobs: Sequence[Blob]) -> List[MatrixEntry, MAX_CELLS_IN_EXTENDED_MATRIX]: @@ -156,7 +156,7 @@ def compute_extended_matrix(blobs: Sequence[Blob]) -> List[MatrixEntry, MAX_CELL return extended_matrix ``` -#### `recover_matrix` +### `recover_matrix` ```python def recover_matrix(partial_matrix: Sequence[MatrixEntry], @@ -183,7 +183,7 @@ def recover_matrix(partial_matrix: Sequence[MatrixEntry], return extended_matrix ``` -#### `get_data_column_sidecars` +### `get_data_column_sidecars` ```python def get_data_column_sidecars(signed_block: SignedBeaconBlock, diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index 1c2fe89c12..a0fc8b9525 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -6,9 +6,9 @@ - [Introduction](#introduction) - - [Helpers](#helpers) - - [Modified `is_data_available`](#modified-is_data_available) - - [Modified `get_head`](#modified-get_head) +- [Helpers](#helpers) + - [Modified `is_data_available`](#modified-is_data_available) + - [Modified `get_head`](#modified-get_head) - [Updated fork-choice handlers](#updated-fork-choice-handlers) - [Modified `on_block`](#modified-on_block) @@ -19,9 +19,9 @@ This is the modification of the fork choice accompanying EIP-7594. -### Helpers +## Helpers -#### Modified `is_data_available` +### Modified `is_data_available` ```python def is_data_available(beacon_block_root: Root) -> bool: @@ -37,7 +37,7 @@ def is_data_available(beacon_block_root: Root) -> bool: ) ``` -#### Modified `get_head` +### Modified `get_head` *Note*: children of the current `head` are required to be available in order to be considered by the fork-choice. diff --git a/specs/_features/eip7594/peer-sampling.md b/specs/_features/eip7594/peer-sampling.md index 26aa7ab99d..a57575ef6d 100644 --- a/specs/_features/eip7594/peer-sampling.md +++ b/specs/_features/eip7594/peer-sampling.md @@ -8,8 +8,9 @@ - - [Helper functions](#helper-functions) - - [`get_extended_sample_count`](#get_extended_sample_count) +- [Introduction](#introduction) +- [Helper functions](#helper-functions) + - [`get_extended_sample_count`](#get_extended_sample_count) - [Peer discovery](#peer-discovery) - [Peer sampling](#peer-sampling) - [Sample selection](#sample-selection) @@ -20,11 +21,13 @@ +## Introduction + The purpose of this document is to complement [EIP-7594 -- Data Availability Sampling Core](das-core.md), by specifying the peer sampling functionality of the full PeerDAS protocol. This functionality may initially not be implemented, or only implemented by some clients, in which case it is replaced by [subnet sampling](das-core.md#subnet-sampling), an extension of the custody component of the protocol. -### Helper functions +## Helper functions -#### `get_extended_sample_count` +### `get_extended_sample_count` ```python def get_extended_sample_count(allowed_failures: uint64) -> uint64: diff --git a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py index 83f7346e31..2334fcdc8a 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py @@ -180,4 +180,4 @@ def test_invalid_wrong_blobs_length(spec, state): assert spec.get_head(store) != signed_block.message.hash_tree_root() - yield 'steps', test_steps \ No newline at end of file + yield 'steps', test_steps From 04bda8b174c23ef52babcbf5da55d51257ea5022 Mon Sep 17 00:00:00 2001 From: fradamt Date: Wed, 7 Aug 2024 17:41:26 +0200 Subject: [PATCH 06/14] adjust configs --- configs/mainnet.yaml | 5 ++--- configs/minimal.yaml | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 39904616b2..6f85bd543e 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -159,11 +159,10 @@ WHISK_PROPOSER_SELECTION_GAP: 2 # EIP7594 NUMBER_OF_COLUMNS: 128 MAX_CELLS_IN_EXTENDED_MATRIX: 768 -DATA_COLUMN_SIDECAR_SUBNET_COUNT: 32 +DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 SAMPLES_PER_SLOT: 8 -CUSTODY_REQUIREMENT: 1 -TARGET_NUMBER_OF_PEERS: 70 +CUSTODY_REQUIREMENT: 4 # [New in Electra:EIP7251] MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 128000000000 # 2**7 * 10**9 (= 128,000,000,000) diff --git a/configs/minimal.yaml b/configs/minimal.yaml index 8ba65cec09..fb35686571 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -158,11 +158,10 @@ WHISK_PROPOSER_SELECTION_GAP: 1 # EIP7594 NUMBER_OF_COLUMNS: 128 MAX_CELLS_IN_EXTENDED_MATRIX: 768 -DATA_COLUMN_SIDECAR_SUBNET_COUNT: 32 +DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 SAMPLES_PER_SLOT: 8 -CUSTODY_REQUIREMENT: 1 -TARGET_NUMBER_OF_PEERS: 70 +CUSTODY_REQUIREMENT: 4 # [New in Electra:EIP7251] MIN_PER_EPOCH_CHURN_LIMIT_ELECTRA: 64000000000 # 2**6 * 10**9 (= 64,000,000,000) From 06ead3aa0d8bb03b0ec847bbd2a916519c0ca2df Mon Sep 17 00:00:00 2001 From: fradamt <104826920+fradamt@users.noreply.github.com> Date: Wed, 7 Aug 2024 22:13:41 +0200 Subject: [PATCH 07/14] Apply suggestions from code review Co-authored-by: Justin Traglia <95511699+jtraglia@users.noreply.github.com> --- specs/_features/eip7594/fork-choice.md | 5 +---- specs/_features/eip7594/peer-sampling.md | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index a0fc8b9525..89a3ec0601 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -51,10 +51,7 @@ def get_head(store: Store) -> Root: # Get available children for the current slot children = [ root for (root, block) in blocks.items() - if ( - block.parent_root == head - and is_data_available(root) - ) + if block.parent_root == head and is_data_available(root): ] if len(children) == 0: return head diff --git a/specs/_features/eip7594/peer-sampling.md b/specs/_features/eip7594/peer-sampling.md index a57575ef6d..5da03cb71a 100644 --- a/specs/_features/eip7594/peer-sampling.md +++ b/specs/_features/eip7594/peer-sampling.md @@ -23,7 +23,7 @@ ## Introduction -The purpose of this document is to complement [EIP-7594 -- Data Availability Sampling Core](das-core.md), by specifying the peer sampling functionality of the full PeerDAS protocol. This functionality may initially not be implemented, or only implemented by some clients, in which case it is replaced by [subnet sampling](das-core.md#subnet-sampling), an extension of the custody component of the protocol. +The purpose of this document is to complement [EIP-7594 -- Data Availability Sampling Core](das-core.md) by specifying the peer sampling functionality of the full PeerDAS protocol. Initially, this functionality may not be implemented by all clients. In such cases, it is replaced by [subnet sampling](das-core.md#subnet-sampling), which is an extension of the custody component of the protocol. ## Helper functions From 80b94ba058545dce32654c1f97d41e2504c69486 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Wed, 7 Aug 2024 15:24:48 -0500 Subject: [PATCH 08/14] Fix bad suggestion --- specs/_features/eip7594/fork-choice.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index 89a3ec0601..66a32c6a7d 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -51,7 +51,7 @@ def get_head(store: Store) -> Root: # Get available children for the current slot children = [ root for (root, block) in blocks.items() - if block.parent_root == head and is_data_available(root): + if block.parent_root == head and is_data_available(root) ] if len(children) == 0: return head From 04ee34c5049a2e3a582737e37b68f9f8321af6fc Mon Sep 17 00:00:00 2001 From: fradamt Date: Thu, 8 Aug 2024 10:01:20 +0200 Subject: [PATCH 09/14] revert to deneb-style fork-choice (is_data_available in on_block) --- specs/_features/eip7594/fork-choice.md | 30 ++++---------------------- 1 file changed, 4 insertions(+), 26 deletions(-) diff --git a/specs/_features/eip7594/fork-choice.md b/specs/_features/eip7594/fork-choice.md index 66a32c6a7d..a6ddf05087 100644 --- a/specs/_features/eip7594/fork-choice.md +++ b/specs/_features/eip7594/fork-choice.md @@ -8,7 +8,6 @@ - [Introduction](#introduction) - [Helpers](#helpers) - [Modified `is_data_available`](#modified-is_data_available) - - [Modified `get_head`](#modified-get_head) - [Updated fork-choice handlers](#updated-fork-choice-handlers) - [Modified `on_block`](#modified-on_block) @@ -37,35 +36,11 @@ def is_data_available(beacon_block_root: Root) -> bool: ) ``` -### Modified `get_head` - -*Note*: children of the current `head` are required to be available in order to be considered by the fork-choice. - -```python -def get_head(store: Store) -> Root: - # Get filtered block tree that only includes viable branches - blocks = get_filtered_block_tree(store) - # Execute the LMD-GHOST fork choice - head = store.justified_checkpoint.root - while True: - # Get available children for the current slot - children = [ - root for (root, block) in blocks.items() - if block.parent_root == head and is_data_available(root) - ] - if len(children) == 0: - return head - # Sort by latest attesting balance with ties broken lexicographically - # Ties broken by favoring block with lexicographically higher root - head = max(children, key=lambda root: (get_weight(store, root), root)) -``` - ## Updated fork-choice handlers ### Modified `on_block` -*Note*: The blob data availability check is removed. We import blocks regardless -of their availability status, and move all filtering to the fork-choice (`get_head`). +*Note*: The only modification is that `is_data_available` does not take `blob_kzg_commitments` as input. ```python def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: @@ -91,6 +66,9 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: ) assert store.finalized_checkpoint.root == finalized_checkpoint_block + # [Modified in EIP7594] + assert is_data_available(hash_tree_root(block)) + # Check the block is valid and compute the post-state block_root = hash_tree_root(block) state_transition(state, signed_block, True) From 75e601a16761249d514e61ce1294998e92245912 Mon Sep 17 00:00:00 2001 From: fradamt Date: Fri, 9 Aug 2024 09:13:13 +0200 Subject: [PATCH 10/14] remove redundant copy() --- specs/capella/fork-choice.md | 5 ++--- specs/deneb/fork-choice.md | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/specs/capella/fork-choice.md b/specs/capella/fork-choice.md index 85b9d24756..a66410cf0b 100644 --- a/specs/capella/fork-choice.md +++ b/specs/capella/fork-choice.md @@ -75,8 +75,6 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: block = signed_block.message # Parent block must be known assert block.parent_root in store.block_states - # Make a copy of the state to avoid mutability issues - pre_state = copy(store.block_states[block.parent_root]) # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. assert get_current_slot(store) >= block.slot @@ -92,7 +90,8 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: assert store.finalized_checkpoint.root == finalized_checkpoint_block # Check the block is valid and compute the post-state - state = pre_state.copy() + # Make a copy of the state to avoid mutability issues + state = copy(store.block_states[block.parent_root]) block_root = hash_tree_root(block) state_transition(state, signed_block, True) diff --git a/specs/deneb/fork-choice.md b/specs/deneb/fork-choice.md index 101e07a0fc..836fe61743 100644 --- a/specs/deneb/fork-choice.md +++ b/specs/deneb/fork-choice.md @@ -74,8 +74,6 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: block = signed_block.message # Parent block must be known assert block.parent_root in store.block_states - # Make a copy of the state to avoid mutability issues - pre_state = copy(store.block_states[block.parent_root]) # Blocks cannot be in the future. If they are, their consideration must be delayed until they are in the past. assert get_current_slot(store) >= block.slot @@ -98,7 +96,8 @@ def on_block(store: Store, signed_block: SignedBeaconBlock) -> None: assert is_data_available(hash_tree_root(block), block.body.blob_kzg_commitments) # Check the block is valid and compute the post-state - state = pre_state.copy() + # Make a copy of the state to avoid mutability issues + state = copy(store.block_states[block.parent_root]) block_root = hash_tree_root(block) state_transition(state, signed_block, True) From d10cc002d55fe506bd949a170194becc61fe9fba Mon Sep 17 00:00:00 2001 From: Potuz Date: Wed, 14 Aug 2024 08:46:57 -0300 Subject: [PATCH 11/14] Run Deneb on_block tests in Electra but not 7594 --- tests/core/pyspec/eth2spec/test/context.py | 7 +++++++ .../test/deneb/fork_choice/test_on_block.py | 17 ++++++++++------- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/tests/core/pyspec/eth2spec/test/context.py b/tests/core/pyspec/eth2spec/test/context.py index e805e1c120..8b2e8de6d3 100644 --- a/tests/core/pyspec/eth2spec/test/context.py +++ b/tests/core/pyspec/eth2spec/test/context.py @@ -429,6 +429,13 @@ def decorator(fn): return decorator +def with_all_phases_from_except(earliest_phase, except_phases=None): + """ + A decorator factory for running a tests with every phase except the ones listed + """ + return with_all_phases_from(earliest_phase, [phase for phase in ALL_PHASES if phase not in except_phases]) + + def with_all_phases_except(exclusion_phases): """ A decorator factory for running a tests with every phase except the ones listed diff --git a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py index 2334fcdc8a..a7e7f784e4 100644 --- a/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py +++ b/tests/core/pyspec/eth2spec/test/deneb/fork_choice/test_on_block.py @@ -2,10 +2,13 @@ from eth2spec.test.context import ( spec_state_test, - with_phases, + with_all_phases_from_except, ) -from eth2spec.test.helpers.constants import DENEB +from eth2spec.test.helpers.constants import ( + DENEB, + EIP7594, +) from eth2spec.test.helpers.block import ( build_empty_block_for_next_slot, @@ -36,7 +39,7 @@ def get_block_with_blob(spec, state, rng=None): return block, blobs, blob_kzg_proofs -@with_phases([DENEB]) +@with_all_phases_from_except(DENEB, [EIP7594]) @spec_state_test def test_simple_blob_data(spec, state): rng = Random(1234) @@ -71,7 +74,7 @@ def test_simple_blob_data(spec, state): yield 'steps', test_steps -@with_phases([DENEB]) +@with_all_phases_from_except(DENEB, [EIP7594]) @spec_state_test def test_invalid_incorrect_proof(spec, state): rng = Random(1234) @@ -99,7 +102,7 @@ def test_invalid_incorrect_proof(spec, state): yield 'steps', test_steps -@with_phases([DENEB]) +@with_all_phases_from_except(DENEB, [EIP7594]) @spec_state_test def test_invalid_data_unavailable(spec, state): rng = Random(1234) @@ -127,7 +130,7 @@ def test_invalid_data_unavailable(spec, state): yield 'steps', test_steps -@with_phases([DENEB]) +@with_all_phases_from_except(DENEB, [EIP7594]) @spec_state_test def test_invalid_wrong_proofs_length(spec, state): rng = Random(1234) @@ -155,7 +158,7 @@ def test_invalid_wrong_proofs_length(spec, state): yield 'steps', test_steps -@with_phases([DENEB]) +@with_all_phases_from_except(DENEB, [EIP7594]) @spec_state_test def test_invalid_wrong_blobs_length(spec, state): rng = Random(1234) From 4eee3b0c3b91844f25cf38a1de01fcbbd200e6a9 Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 14 Aug 2024 20:19:50 -0500 Subject: [PATCH 12/14] Remove matrix specific configuration value This also removes references to the "extended matrix" in favor of just "matrix" which I think is better. It's not an extended matrix, it's a matrix of extended blobs. Technically it's just a matrix of cells/proofs. --- configs/mainnet.yaml | 1 - configs/minimal.yaml | 1 - pysetup/spec_builders/eip7594.py | 1 - specs/_features/eip7594/das-core.md | 35 +++++++++---------- .../test/eip7594/unittests/das/test_das.py | 14 ++++---- .../unittests/test_config_invariants.py | 2 -- 6 files changed, 23 insertions(+), 31 deletions(-) diff --git a/configs/mainnet.yaml b/configs/mainnet.yaml index 1c2911f8f5..831d20afdc 100644 --- a/configs/mainnet.yaml +++ b/configs/mainnet.yaml @@ -161,7 +161,6 @@ WHISK_PROPOSER_SELECTION_GAP: 2 # EIP7594 NUMBER_OF_COLUMNS: 128 -MAX_CELLS_IN_EXTENDED_MATRIX: 768 DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 SAMPLES_PER_SLOT: 8 diff --git a/configs/minimal.yaml b/configs/minimal.yaml index eb4f77aa26..e1f23b68ec 100644 --- a/configs/minimal.yaml +++ b/configs/minimal.yaml @@ -160,7 +160,6 @@ WHISK_PROPOSER_SELECTION_GAP: 1 # EIP7594 NUMBER_OF_COLUMNS: 128 -MAX_CELLS_IN_EXTENDED_MATRIX: 768 DATA_COLUMN_SIDECAR_SUBNET_COUNT: 128 MAX_REQUEST_DATA_COLUMN_SIDECARS: 16384 SAMPLES_PER_SLOT: 8 diff --git a/pysetup/spec_builders/eip7594.py b/pysetup/spec_builders/eip7594.py index dfbc427a70..109ad8736f 100644 --- a/pysetup/spec_builders/eip7594.py +++ b/pysetup/spec_builders/eip7594.py @@ -27,7 +27,6 @@ def hardcoded_custom_type_dep_constants(cls, spec_object) -> str: 'FIELD_ELEMENTS_PER_CELL': spec_object.preset_vars['FIELD_ELEMENTS_PER_CELL'].value, 'FIELD_ELEMENTS_PER_EXT_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_EXT_BLOB'].value, 'NUMBER_OF_COLUMNS': spec_object.config_vars['NUMBER_OF_COLUMNS'].value, - 'MAX_CELLS_IN_EXTENDED_MATRIX': spec_object.config_vars['MAX_CELLS_IN_EXTENDED_MATRIX'].value, } @classmethod diff --git a/specs/_features/eip7594/das-core.md b/specs/_features/eip7594/das-core.md index 3d43db4830..2096fb9a31 100644 --- a/specs/_features/eip7594/das-core.md +++ b/specs/_features/eip7594/das-core.md @@ -20,7 +20,7 @@ - [`MatrixEntry`](#matrixentry) - [Helper functions](#helper-functions) - [`get_custody_columns`](#get_custody_columns) - - [`compute_extended_matrix`](#compute_extended_matrix) + - [`compute_matrix`](#compute_matrix) - [`recover_matrix`](#recover_matrix) - [`get_data_column_sidecars`](#get_data_column_sidecars) - [Custody](#custody) @@ -62,7 +62,6 @@ The following values are (non-configurable) constants used throughout the specif | Name | Value | Description | | - | - | - | | `NUMBER_OF_COLUMNS` | `uint64(CELLS_PER_EXT_BLOB)` (= 128) | Number of columns in the extended data matrix | -| `MAX_CELLS_IN_EXTENDED_MATRIX` | `uint64(MAX_BLOBS_PER_BLOCK * NUMBER_OF_COLUMNS)` (= 768) | The data size of `ExtendedMatrix` | ### Networking @@ -133,54 +132,52 @@ def get_custody_columns(node_id: NodeID, custody_subnet_count: uint64) -> Sequen ]) ``` -### `compute_extended_matrix` +### `compute_matrix` ```python -def compute_extended_matrix(blobs: Sequence[Blob]) -> List[MatrixEntry, MAX_CELLS_IN_EXTENDED_MATRIX]: +def compute_matrix(blobs: Sequence[Blob]) -> Sequence[MatrixEntry]: """ - Return the full ``ExtendedMatrix``. + Return the full, flattened sequence of matrix entries. - This helper demonstrates the relationship between blobs and ``ExtendedMatrix``. - The data structure for storing cells is implementation-dependent. + This helper demonstrates the relationship between blobs and the matrix of cells/proofs. + The data structure for storing cells/proofs is implementation-dependent. """ - extended_matrix = [] + matrix = [] for blob_index, blob in enumerate(blobs): cells, proofs = compute_cells_and_kzg_proofs(blob) for cell_index, (cell, proof) in enumerate(zip(cells, proofs)): - extended_matrix.append(MatrixEntry( + matrix.append(MatrixEntry( cell=cell, kzg_proof=proof, row_index=blob_index, column_index=cell_index, )) - return extended_matrix + return matrix ``` ### `recover_matrix` ```python -def recover_matrix(partial_matrix: Sequence[MatrixEntry], - blob_count: uint64) -> List[MatrixEntry, MAX_CELLS_IN_EXTENDED_MATRIX]: +def recover_matrix(partial_matrix: Sequence[MatrixEntry], blob_count: uint64) -> Sequence[MatrixEntry]: """ - Return the recovered extended matrix. + Recover the full, flattened sequence of matrix entries. This helper demonstrates how to apply ``recover_cells_and_kzg_proofs``. - The data structure for storing cells is implementation-dependent. + The data structure for storing cells/proofs is implementation-dependent. """ - extended_matrix = [] + matrix = [] for blob_index in range(blob_count): cell_indices = [e.column_index for e in partial_matrix if e.row_index == blob_index] cells = [e.cell for e in partial_matrix if e.row_index == blob_index] - recovered_cells, recovered_proofs = recover_cells_and_kzg_proofs(cell_indices, cells) for cell_index, (cell, proof) in enumerate(zip(recovered_cells, recovered_proofs)): - extended_matrix.append(MatrixEntry( + matrix.append(MatrixEntry( cell=cell, kzg_proof=proof, row_index=blob_index, column_index=cell_index, )) - return extended_matrix + return matrix ``` ### `get_data_column_sidecars` @@ -241,7 +238,7 @@ At each slot, a node advertising `custody_subnet_count` downloads a minimum of ` ## Extended data -In this construction, we extend the blobs using a one-dimensional erasure coding extension. The matrix comprises maximum `MAX_BLOBS_PER_BLOCK` rows and fixed `NUMBER_OF_COLUMNS` columns, with each row containing a `Blob` and its corresponding extension. `compute_extended_matrix` demonstrates the relationship between blobs and custom type `ExtendedMatrix`. +In this construction, we extend the blobs using a one-dimensional erasure coding extension. The matrix comprises maximum `MAX_BLOBS_PER_BLOCK` rows and fixed `NUMBER_OF_COLUMNS` columns, with each row containing a `Blob` and its corresponding extension. `compute_matrix` demonstrates the relationship between blobs and the matrix, a potential method of storing cells/proofs. ## Column gossip diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py index 7110f2373e..625136b73e 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/das/test_das.py @@ -19,15 +19,15 @@ def chunks(lst, n): @with_eip7594_and_later @spec_test @single_phase -def test_compute_extended_matrix(spec): +def test_compute_matrix(spec): rng = random.Random(5566) blob_count = 2 input_blobs = [get_sample_blob(spec, rng=rng) for _ in range(blob_count)] - extended_matrix = spec.compute_extended_matrix(input_blobs) - assert len(extended_matrix) == spec.CELLS_PER_EXT_BLOB * blob_count + matrix = spec.compute_matrix(input_blobs) + assert len(matrix) == spec.CELLS_PER_EXT_BLOB * blob_count - rows = chunks(extended_matrix, spec.CELLS_PER_EXT_BLOB) + rows = chunks(matrix, spec.CELLS_PER_EXT_BLOB) assert len(rows) == blob_count for row in rows: assert len(row) == spec.CELLS_PER_EXT_BLOB @@ -53,11 +53,11 @@ def test_recover_matrix(spec): # Compute an extended matrix with two blobs blob_count = 2 blobs = [get_sample_blob(spec, rng=rng) for _ in range(blob_count)] - extended_matrix = spec.compute_extended_matrix(blobs) + matrix = spec.compute_matrix(blobs) # Construct a matrix with some entries missing partial_matrix = [] - for blob_entries in chunks(extended_matrix, spec.CELLS_PER_EXT_BLOB): + for blob_entries in chunks(matrix, spec.CELLS_PER_EXT_BLOB): rng.shuffle(blob_entries) partial_matrix.extend(blob_entries[:N_SAMPLES]) @@ -65,7 +65,7 @@ def test_recover_matrix(spec): recovered_matrix = spec.recover_matrix(partial_matrix, blob_count) # Ensure that the recovered matrix matches the original matrix - assert recovered_matrix == extended_matrix + assert recovered_matrix == matrix @with_eip7594_and_later diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py index 776ea883aa..6d20882cda 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py @@ -17,8 +17,6 @@ def test_invariants(spec): assert spec.config.NUMBER_OF_COLUMNS % spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT == 0 assert spec.config.MAX_REQUEST_DATA_COLUMN_SIDECARS == ( spec.config.MAX_REQUEST_BLOCKS_DENEB * spec.config.NUMBER_OF_COLUMNS - ) - assert spec.config.MAX_CELLS_IN_EXTENDED_MATRIX == spec.config.MAX_BLOBS_PER_BLOCK * spec.config.NUMBER_OF_COLUMNS @with_eip7594_and_later From 121a3b7b240f64046848c068e17715198ffa880b Mon Sep 17 00:00:00 2001 From: Justin Traglia Date: Wed, 14 Aug 2024 20:32:11 -0500 Subject: [PATCH 13/14] Fix mistake --- .../eth2spec/test/eip7594/unittests/test_config_invariants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py index 6d20882cda..fc54cc3088 100644 --- a/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py +++ b/tests/core/pyspec/eth2spec/test/eip7594/unittests/test_config_invariants.py @@ -17,6 +17,7 @@ def test_invariants(spec): assert spec.config.NUMBER_OF_COLUMNS % spec.config.DATA_COLUMN_SIDECAR_SUBNET_COUNT == 0 assert spec.config.MAX_REQUEST_DATA_COLUMN_SIDECARS == ( spec.config.MAX_REQUEST_BLOCKS_DENEB * spec.config.NUMBER_OF_COLUMNS + ) @with_eip7594_and_later From 7391de10a8a08b027748b70105b10ae2da7cd481 Mon Sep 17 00:00:00 2001 From: Hsiao-Wei Wang Date: Tue, 20 Aug 2024 23:55:52 +0800 Subject: [PATCH 14/14] bump version to 1.5.0-alpha.5 --- tests/core/pyspec/eth2spec/VERSION.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/core/pyspec/eth2spec/VERSION.txt b/tests/core/pyspec/eth2spec/VERSION.txt index 2ac49a507e..8c993fb5e5 100644 --- a/tests/core/pyspec/eth2spec/VERSION.txt +++ b/tests/core/pyspec/eth2spec/VERSION.txt @@ -1 +1 @@ -1.5.0-alpha.4 +1.5.0-alpha.5