Merge pull request #3853 from ethereum/dev

Release v1.5.0-alpha.4

.gitignore

@@ -25,6 +25,7 @@ tests/core/pyspec/eth2spec/electra/
 tests/core/pyspec/eth2spec/whisk/
 tests/core/pyspec/eth2spec/eip7594/
 tests/core/pyspec/eth2spec/eip6800/
+tests/core/pyspec/eth2spec/eip7732/
 
 # coverage reports
 .htmlcov

Makefile

@@ -14,7 +14,6 @@ SOLIDITY_FILE_NAME = deposit_contract.json
 DEPOSIT_CONTRACT_TESTER_DIR = ${SOLIDITY_DEPOSIT_CONTRACT_DIR}/web3_tester
 CONFIGS_DIR = ./configs
 TEST_PRESET_TYPE ?= minimal
-NUMBER_OF_CORES=16
 # Collect a list of generator names
 GENERATORS = $(sort $(dir $(wildcard $(GENERATOR_DIR)/*/.)))
 # Map this list of generator paths to "gen_{generator name}" entries
@@ -35,7 +34,7 @@ MARKDOWN_FILES = $(wildcard $(SPEC_DIR)/*/*.md) \
                  $(wildcard $(SPEC_DIR)/_features/*/*/*.md) \
                  $(wildcard $(SSZ_DIR)/*.md)
 
-ALL_EXECUTABLE_SPEC_NAMES = phase0 altair bellatrix capella deneb electra whisk eip6800
+ALL_EXECUTABLE_SPEC_NAMES = phase0 altair bellatrix capella deneb electra whisk eip6800 eip7732
 # The parameters for commands. Use `foreach` to avoid listing specs again.
 COVERAGE_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), --cov=eth2spec.$S.$(TEST_PRESET_TYPE))
 PYLINT_SCOPE := $(foreach S,$(ALL_EXECUTABLE_SPEC_NAMES), ./eth2spec/$S)
@@ -118,7 +117,7 @@ install_test: preinstallation
 # Testing against `minimal` or `mainnet` config by default
 test: pyspec
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n 4 --disable-bls $(COVERAGE_SCOPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
+	python3 -m pytest -n auto --disable-bls $(COVERAGE_SCOPE) --cov-report="html:$(COV_HTML_OUT)" --cov-branch eth2spec
 
 # Testing against `minimal` or `mainnet` config by default
 find_test: pyspec
@@ -129,10 +128,10 @@ citest: pyspec
 	mkdir -p $(TEST_REPORT_DIR);
 ifdef fork
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n $(NUMBER_OF_CORES) --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec
+	python3 -m pytest -n auto --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --fork=$(fork) --junitxml=test-reports/test_results.xml eth2spec
 else
 	. venv/bin/activate; cd $(PY_SPEC_DIR); \
-	python3 -m pytest -n $(NUMBER_OF_CORES) --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec
+	python3 -m pytest -n auto --bls-type=fastest --preset=$(TEST_PRESET_TYPE) --junitxml=test-reports/test_results.xml eth2spec
 endif
 
 

README.md

@@ -26,7 +26,7 @@ Features are researched and developed in parallel, and then consolidated into se
 ### In-development Specifications
 | Code Name or Topic | Specs | Notes |
 | - | - | - |
-| Electra | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/electra/beacon-chain.md)</li><li>[EIP-6110 fork](specs/electra/fork.md)</li></ul><li>Additions</li><ul><li>[Honest validator guide changes](specs/electra/validator.md)</li></ul></ul> |
+| Electra | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/electra/beacon-chain.md)</li><li>[EIP-6110 fork](specs/electra/fork.md)</li></ul><li>Additions</li><ul><li>[Light client sync protocol changes](specs/electra/light-client/sync-protocol.md) ([fork](specs/electra/light-client/fork.md), [full node](specs/electra/light-client/full-node.md), [networking](specs/electra/light-client/p2p-interface.md))</li></ul><ul><li>[Honest validator guide changes](specs/electra/validator.md)</li></ul></ul> |
 | Sharding (outdated) | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/_features/sharding/beacon-chain.md)</li></ul><li>Additions</li><ul><li>[P2P networking](specs/_features/sharding/p2p-interface.md)</li></ul></ul> |
 | Custody Game (outdated) | <ul><li>Core</li><ul><li>[Beacon Chain changes](specs/_features/custody_game/beacon-chain.md)</li></ul><li>Additions</li><ul><li>[Honest validator guide changes](specs/_features/custody_game/validator.md)</li></ul></ul> | Dependent on sharding |
 | Data Availability Sampling (outdated) | <ul><li>Core</li><ul><li>[Core types and functions](specs/_features/das/das-core.md)</li><li>[Fork choice changes](specs/_features/das/fork-choice.md)</li></ul><li>Additions</li><ul><li>[P2P Networking](specs/_features/das/p2p-interface.md)</li><li>[Sampling process](specs/_features/das/sampling.md)</li></ul></ul> | <ul><li> Dependent on sharding</li><li>[Technical explainer](https://hackmd.io/@HWeNw8hNRimMm2m2GH56Cw/B1YJPGkpD)</li></ul> |

configs/mainnet.yaml

@@ -59,6 +59,9 @@ EIP7594_FORK_EPOCH: 18446744073709551615
 # WHISK
 WHISK_FORK_VERSION: 0x08000000  # temporary stub
 WHISK_FORK_EPOCH: 18446744073709551615
+# EIP7732
+EIP7732_FORK_VERSION: 0x09000000  # temporary stub
+EIP7732_FORK_EPOCH: 18446744073709551615
 
 # Time parameters
 # ---------------------------------------------------------------
@@ -147,6 +150,8 @@ MAX_REQUEST_BLOB_SIDECARS: 768
 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096
 # `6`
 BLOB_SIDECAR_SUBNET_COUNT: 6
+## `uint64(6)`
+MAX_BLOBS_PER_BLOCK: 6
 
 # Whisk
 # `Epoch(2**8)`

configs/minimal.yaml

@@ -58,6 +58,9 @@ EIP7594_FORK_EPOCH: 18446744073709551615
 # WHISK
 WHISK_FORK_VERSION: 0x08000001
 WHISK_FORK_EPOCH: 18446744073709551615
+# EIP7732
+EIP7732_FORK_VERSION: 0x09000001
+EIP7732_FORK_EPOCH: 18446744073709551615
 
 # Time parameters
 # ---------------------------------------------------------------
@@ -148,6 +151,8 @@ MAX_REQUEST_BLOB_SIDECARS: 768
 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096
 # `6`
 BLOB_SIDECAR_SUBNET_COUNT: 6
+## `uint64(6)`
+MAX_BLOBS_PER_BLOCK: 6
 
 # Whisk
 WHISK_EPOCHS_PER_SHUFFLING_PHASE: 4

docker/README.md

@@ -6,15 +6,15 @@ This dockerfile sets up the dependencies required to run consensus-spec tests. T
 
 Handy commands:
 - `docker run -it $IMAGE_NAME /bin/sh` will give you a shell inside the docker container to manually run any tests
-- `docker run  $IMAGE_NAME make citest` will run the make citest command inside the docker container
+- `docker run $IMAGE_NAME make citest` will run the make citest command inside the docker container
 
 Ideally manual running of docker containers is for advanced users, we recommend the script based approach described below for most users.
 
-The `scripts/build_run_docker_tests.sh` script will cover most usecases. The script allows the user to configure the fork(altair/bellatrix/capella..), `$IMAGE_NAME` (specifies the container to use), number of cores, preset type (mainnet/minimal), and test all forks flags. Ideally, this is the main way that users interact with the spec tests instead of running it locally with varying versions of dependencies.
+The `scripts/build_run_docker_tests.sh` script will cover most usecases. The script allows the user to configure the fork(altair/bellatrix/capella..), `$IMAGE_NAME` (specifies the container to use), preset type (mainnet/minimal), and test all forks flags. Ideally, this is the main way that users interact with the spec tests instead of running it locally with varying versions of dependencies.
 
 E.g:
-- `./build_run_test.sh --p mainnet --n 16` will run the mainnet preset tests with 16 threads
+- `./build_run_test.sh --p mainnet` will run the mainnet preset tests
 - `./build_run_test.sh --a` will run all the tests across all the forks
-- `./build_run_test.sh --f deneb --n 16` will only run deneb tests on 16 threads
+- `./build_run_test.sh --f deneb` will only run deneb tests
 
 Results are always placed in a folder called `./testResults`. The results are `.xml` files and contain the fork they represent and the date/time they were run at.

fork_choice/safe-block.md

@@ -7,7 +7,7 @@
 
 - [Introduction](#introduction)
 - [`get_safe_beacon_block_root`](#get_safe_beacon_block_root)
-- [`get_safe_execution_payload_hash`](#get_safe_execution_payload_hash)
+- [`get_safe_execution_block_hash`](#get_safe_execution_block_hash)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
 <!-- /TOC -->
@@ -31,10 +31,10 @@ def get_safe_beacon_block_root(store: Store) -> Root:
 *Note*: Currently safe block algorithm simply returns `store.justified_checkpoint.root`
 and is meant to be improved in the future.
 
-## `get_safe_execution_payload_hash`
+## `get_safe_execution_block_hash`
 
 ```python
-def get_safe_execution_payload_hash(store: Store) -> Hash32:
+def get_safe_execution_block_hash(store: Store) -> Hash32:
     safe_block_root = get_safe_beacon_block_root(store)
     safe_block = store.blocks[safe_block_root]
 

presets/mainnet/deneb.yaml

@@ -6,7 +6,5 @@
 FIELD_ELEMENTS_PER_BLOB: 4096
 # `uint64(2**12)` (= 4096)
 MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096
-# `uint64(6)`
-MAX_BLOBS_PER_BLOCK: 6
 # `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17
 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17

presets/mainnet/eip-7732.yaml

@@ -0,0 +1,9 @@
+# Mainnet preset - EIP7732
+
+# Execution
+# ---------------------------------------------------------------
+# 2**9 (= 512)
+PTC_SIZE: 512
+# 2**2 (= 4)
+MAX_PAYLOAD_ATTESTATIONS: 4
+KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_EIP7732: 13

presets/minimal/deneb.yaml

@@ -6,7 +6,5 @@
 FIELD_ELEMENTS_PER_BLOB: 4096
 # [customized]
 MAX_BLOB_COMMITMENTS_PER_BLOCK: 16
-# `uint64(6)`
-MAX_BLOBS_PER_BLOCK: 6
 # [customized] `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 4 = 9
 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9

presets/minimal/eip-7732.yaml

@@ -0,0 +1,9 @@
+# Minimal preset - EIP7732
+
+# Execution
+# ---------------------------------------------------------------
+# 2**1(= 2)
+PTC_SIZE: 2
+# 2**2 (= 4)
+MAX_PAYLOAD_ATTESTATIONS: 4
+KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_EIP7732: 13

pysetup/constants.py

@@ -8,6 +8,7 @@
 EIP7594 = 'eip7594'
 EIP6800 = 'eip6800'
 WHISK = 'whisk'
+EIP7732 = 'eip7732'
 
 
 # The helper functions that are used when defining constants

pysetup/helpers.py

@@ -124,13 +124,22 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
     # Keep engine from the most recent fork
     execution_engine_cls = reduce(lambda txt, builder: builder.execution_engine_cls() or txt, builders, "")
 
+    # Remove deprecated constants
+    deprecate_constants = reduce(lambda obj, builder: obj.union(builder.deprecate_constants()), builders, set())
+    # constant_vars = {k: v for k, v in spec_object.constant_vars.items() if k not in deprecate_constants}
+    filtered_ssz_dep_constants = {k: v for k, v in hardcoded_ssz_dep_constants.items() if k not in deprecate_constants}
+    # Remove deprecated presets
+    deprecate_presets = reduce(lambda obj, builder: obj.union(builder.deprecate_presets()), builders, set())
+    # preset_vars = {k: v for k, v in spec_object.constant_vars.items() if k not in deprecate_constants}
+    filtered_hardcoded_func_dep_presets = {k: v for k, v in hardcoded_func_dep_presets.items() if k not in deprecate_presets}
+
     constant_vars_spec = '# Constant vars\n' + '\n'.join(format_constant(k, v) for k, v in spec_object.constant_vars.items())
     preset_vars_spec = '# Preset vars\n' + '\n'.join(format_constant(k, v) for k, v in spec_object.preset_vars.items())
     ordered_class_objects_spec = '\n\n\n'.join(ordered_class_objects.values())
     ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
-    ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
+    ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), filtered_ssz_dep_constants))
     custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_custom_type_dep_constants[x]), hardcoded_custom_type_dep_constants))
-    func_dep_presets_verification = '\n'.join(map(lambda x: 'assert %s == %s  # noqa: E501' % (x, spec_object.func_dep_presets[x]), hardcoded_func_dep_presets))
+    func_dep_presets_verification = '\n'.join(map(lambda x: 'assert %s == %s  # noqa: E501' % (x, spec_object.func_dep_presets[x]), filtered_hardcoded_func_dep_presets))
     spec_strs = [
         imports,
         preparations,

pysetup/md_doc_paths.py

@@ -10,6 +10,7 @@
     WHISK,
     EIP7594,
     EIP6800,
+    EIP7732,
 )
 
 
@@ -23,6 +24,7 @@
     WHISK: CAPELLA,
     EIP7594: DENEB,
     EIP6800: DENEB,
+    EIP7732: ELECTRA,
 }
 
 ALL_FORKS = list(PREVIOUS_FORK_OF.keys())

pysetup/spec_builders/__init__.py

@@ -7,12 +7,13 @@
 from .whisk import WhiskSpecBuilder
 from .eip7594 import EIP7594SpecBuilder
 from .eip6800 import EIP6800SpecBuilder
+from .eip7732 import EIP7732SpecBuilder
 
 
 spec_builders = {
     builder.fork: builder
     for builder in (
         Phase0SpecBuilder, AltairSpecBuilder, BellatrixSpecBuilder, CapellaSpecBuilder, DenebSpecBuilder,
-        ElectraSpecBuilder, WhiskSpecBuilder, EIP7594SpecBuilder, EIP6800SpecBuilder,
+        ElectraSpecBuilder, WhiskSpecBuilder, EIP7594SpecBuilder, EIP6800SpecBuilder, EIP7732SpecBuilder,
     )
 }

pysetup/spec_builders/altair.py

@@ -35,7 +35,7 @@ def get_generalized_index(ssz_class: Any, *path: PyUnion[int, SSZVariableName])
 
 
 def compute_merkle_proof(object: SSZObject,
-                         index: GeneralizedIndex) -> Sequence[Bytes32]:
+                         index: GeneralizedIndex) -> list[Bytes32]:
     return build_proof(object.get_backing(), index)'''
 
 

pysetup/spec_builders/base.py

@@ -1,5 +1,5 @@
 from abc import ABC, abstractmethod
-from typing import Sequence, Dict
+from typing import Sequence, Dict, Set
 from pathlib import Path
 
 class BaseSpecBuilder(ABC):
@@ -54,3 +54,11 @@ def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
     @classmethod
     def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]:
         return functions
+
+    @classmethod
+    def deprecate_constants(cls) -> Set[str]:
+        return set()
+
+    @classmethod
+    def deprecate_presets(cls) -> Set[str]:
+        return set()

pysetup/spec_builders/deneb.py

@@ -70,7 +70,7 @@ def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:
         return {
             'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value,
             'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value,
-            'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value,
+            'MAX_BLOBS_PER_BLOCK': spec_object.config_vars['MAX_BLOBS_PER_BLOCK'].value,
             'MAX_BLOB_COMMITMENTS_PER_BLOCK': spec_object.preset_vars['MAX_BLOB_COMMITMENTS_PER_BLOCK'].value,
         }
 

pysetup/spec_builders/eip7732.py

@@ -0,0 +1,45 @@
+from typing import Dict, Set
+
+from .base import BaseSpecBuilder
+from ..constants import EIP7732
+
+
+class EIP7732SpecBuilder(BaseSpecBuilder):
+    fork: str = EIP7732
+
+    @classmethod
+    def imports(cls, preset_name: str):
+        return f'''
+from eth2spec.electra import {preset_name} as electra
+'''
+
+    @classmethod
+    def sundry_functions(cls) -> str:
+        return '''
+def concat_generalized_indices(*indices: GeneralizedIndex) -> GeneralizedIndex:
+    o = GeneralizedIndex(1)
+    for i in indices:
+        o = GeneralizedIndex(o * bit_floor(i) + (i - bit_floor(i)))
+    return o'''
+
+
+    @classmethod
+    def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:
+        return {
+            'PTC_SIZE': spec_object.preset_vars['PTC_SIZE'].value,
+            'MAX_PAYLOAD_ATTESTATIONS': spec_object.preset_vars['MAX_PAYLOAD_ATTESTATIONS'].value,
+            'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_EIP7732':
+                spec_object.preset_vars['KZG_COMMITMENT_INCLUSION_PROOF_DEPTH_EIP7732'].value,
+        }
+
+    @classmethod
+    def deprecate_constants(cls) -> Set[str]:
+        return set([
+            'EXECUTION_PAYLOAD_GINDEX',
+        ])
+
+    @classmethod
+    def deprecate_presets(cls) -> Set[str]:
+        return set([
+            'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH',
+        ])

pysetup/spec_builders/electra.py

@@ -12,12 +12,10 @@ def imports(cls, preset_name: str):
 from eth2spec.deneb import {preset_name} as deneb
 '''
 
-## TODO: deal with changed gindices
-
     @classmethod
     def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:
         return {
-            'FINALIZED_ROOT_GINDEX': 'GeneralizedIndex(169)',
-            'CURRENT_SYNC_COMMITTEE_GINDEX': 'GeneralizedIndex(86)',
-            'NEXT_SYNC_COMMITTEE_GINDEX': 'GeneralizedIndex(87)',
+            'FINALIZED_ROOT_GINDEX_ELECTRA': 'GeneralizedIndex(169)',
+            'CURRENT_SYNC_COMMITTEE_GINDEX_ELECTRA': 'GeneralizedIndex(86)',
+            'NEXT_SYNC_COMMITTEE_GINDEX_ELECTRA': 'GeneralizedIndex(87)',
         }

scripts/build_run_docker_tests.sh

@@ -13,25 +13,22 @@
 ALL_EXECUTABLE_SPECS=("phase0" "altair" "bellatrix" "capella" "deneb" "electra" "whisk")
 TEST_PRESET_TYPE=minimal
 FORK_TO_TEST=phase0
-NUMBER_OF_CORES=4
 WORKDIR="//consensus-specs//tests//core//pyspec"
 ETH2SPEC_FOLDER_NAME="eth2spec"
 CONTAINER_NAME="consensus-specs-tests"
 DATE=$(date +"%Y%m%d-%H-%M")
 # Default flag values
 version=$(git log --pretty=format:'%h' -n 1)
 IMAGE_NAME="consensus-specs:$version"
-number_of_core=4
 
 # displays the available options
 display_help() {
   echo "Run 'consensus-specs' tests from a container instance."
   echo "Be sure to launch Docker before running this script."
   echo
-  echo "Syntax: build_run_test.sh [--v TAG | --n NUMBER_OF_CORE | --f FORK_TO_TEST | --p PRESET_TYPE | --a | --h HELP]"
+  echo "Syntax: build_run_test.sh [--v TAG | --f FORK_TO_TEST | --p PRESET_TYPE | --a | --h HELP]"
     echo "  --f <fork>   Specify the fork to test"
     echo "  --i <image_name> Specify the docker image to use"
-    echo "  --n <number> Specify the number of cores"
     echo "  --p <type>   Specify the test preset type"
     echo "  --a          Test all forks"
     echo "  --h          Display this help and exit"
@@ -63,7 +60,6 @@ while [[ "$#" -gt 0 ]]; do
     case $1 in
         --f) FORK_TO_TEST="$2"; shift ;;
         --v) IMAGE_NAME="$2"; shift ;;
-        --n) NUMBER_OF_CORES="$2"; shift ;;
         --p) TEST_PRESET_TYPE="$2"; shift ;;
         --a) FORK_TO_TEST="all" ;;
         --h) display_help; exit 0 ;;
@@ -90,12 +86,12 @@ fi
 if [ "$FORK_TO_TEST" == "all" ]; then
   for fork in "${ALL_EXECUTABLE_SPECS[@]}"; do
     docker run --name $CONTAINER_NAME $IMAGE_NAME \
-      make citest fork=$fork TEST_PRESET_TYPE=$TEST_PRESET_TYPE NUMBER_OF_CORES=$NUMBER_OF_CORES
+      make citest fork=$fork TEST_PRESET_TYPE=$TEST_PRESET_TYPE
       copy_test_results $fork
   done
 else
   docker run --name $CONTAINER_NAME $IMAGE_NAME \
-      make citest fork=$FORK_TO_TEST TEST_PRESET_TYPE=$TEST_PRESET_TYPE NUMBER_OF_CORES=$NUMBER_OF_CORES
+      make citest fork=$FORK_TO_TEST TEST_PRESET_TYPE=$TEST_PRESET_TYPE
   copy_test_results $FORK_TO_TEST
 fi
 

setup.py

@@ -552,7 +552,7 @@ def run(self):
         "pycryptodome==3.15.0",
         "py_ecc==6.0.0",
         "milagro_bls_binding==1.9.0",
-        "remerkleable==0.1.27",
+        "remerkleable==0.1.28",
         "trie==2.0.2",
         RUAMEL_YAML_VERSION,
         "lru-dict==1.2.0",