Update python to 3.12.4 (#3880)

* update python to 3.12 * drop down to 3.12.4 for compatibility * setup rust toolchain for deps * fix typo * fix linter * fixing linter * adding rust dep * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying to fix circleci * trying new image * trying new image * trying new image * trying new image * invalidate cache * bump reqs * remove cache invalidation * fix cache name * update to use latest image
ethereum · Aug 21, 2024 · ad91a83 · ad91a83
1 parent 01aab85
commit ad91a83
Show file tree

Hide file tree

Showing 11 changed files with 62 additions and 59 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -60,7 +60,7 @@ commands:
 jobs:
   checkout_specs:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       # Restore git repo at point close to target branch/revision, to speed up checkout
@@ -80,19 +80,19 @@ jobs:
             - ~/specs-repo
   install_pyspec_test:
     docker:
-      - image: circleci/python:3.9
+      - image: ethpandaops/circleci-python-rust:latest
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
           key: v3-specs-repo-{{ .Branch }}-{{ .Revision }}
       - restore_pyspec_cached_venv
       - run:
           name: Install pyspec requirements
-          command: make install_test
+          command: make install_test && make preinstallation
       - save_pyspec_cached_venv
   test-phase0:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -105,7 +105,7 @@ jobs:
           path: tests/core/pyspec/test-reports
   test-altair:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -118,7 +118,7 @@ jobs:
           path: tests/core/pyspec/test-reports
   test-bellatrix:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -131,7 +131,7 @@ jobs:
           path: tests/core/pyspec/test-reports
   test-capella:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -144,7 +144,7 @@ jobs:
           path: tests/core/pyspec/test-reports
   test-deneb:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -157,7 +157,7 @@ jobs:
           path: tests/core/pyspec/test-reports
   test-electra:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -170,7 +170,7 @@ jobs:
           path: tests/core/pyspec/test-reports
   test-whisk:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -183,7 +183,7 @@ jobs:
           path: tests/core/pyspec/test-reports
   test-eip7594:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -205,7 +205,7 @@ jobs:
           command: sudo npm install -g [email protected] && make check_toc
   codespell:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - checkout
@@ -214,7 +214,7 @@ jobs:
           command: pip install 'codespell<3.0.0,>=2.0.0' --user &&  make codespell
   lint:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -270,7 +270,7 @@ jobs:
             - /nix
   install_deposit_contract_web3_tester:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:
@@ -282,7 +282,7 @@ jobs:
       - save_deposit_contract_tester_cached_venv
   test_deposit_contract_web3_tests:
     docker:
-      - image: circleci/python:3.9
+      - image: cimg/python:3.12.4
     working_directory: ~/specs-repo
     steps:
       - restore_cache:

diff --git a/.github/workflows/generate_vectors.yml b/.github/workflows/generate_vectors.yml
@@ -32,7 +32,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12.4'
           cache: ''
       - name: Clean up Spec Repository
         run: |

diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml
@@ -45,7 +45,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12.4'
           cache: ''
       - name: Check codespell
         run: make codespell
@@ -55,10 +55,12 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+      - name: Setup Rust for dependencies
+        uses: actions-rust-lang/setup-rust-toolchain@v1
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12.4'
           cache: ''
       - name: Install pyspec requirements
         run: make install_test
@@ -76,10 +78,12 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+      - name: Setup Rust for dependencies
+        uses: actions-rust-lang/setup-rust-toolchain@v1
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12.4'
           cache: ''
       - name: set TEST_PRESET_TYPE
         if: github.event.inputs.test_preset_type != ''

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -1,5 +1,5 @@
 # Rename the build stage from 'base' to 'builder' for clarification and code readability
-FROM python:3.11.0-slim-bullseye as builder
+FROM python:3.12.4-slim-bullseye as builder
 
 ENV DEBIAN_FRONTEND=noninteractive \
     WORKDIR=/consensus-specs  \

diff --git a/requirements_preinstallation.txt b/requirements_preinstallation.txt
@@ -1,4 +1,4 @@
-pip>=23.1.2
-wheel>=0.40.0
-setuptools>=68.0.0
-pylint>=3.0.0
+pip>=24.0.0
+wheel>=0.44.0
+setuptools>=72.0.0
+pylint>=3.2.0
diff --git a/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py b/tests/core/pyspec/eth2spec/gen_helpers/gen_base/gen_runner.py
@@ -132,7 +132,7 @@ def should_skip_case_dir(case_dir, is_force, diagnostics_obj):
             print(f'Skipping already existing test: {case_dir}')
             is_skip = True
         else:
-            print(f'Warning, output directory {case_dir} already exist,'
+            print(f'Warning, output directory {case_dir} already exist, '
                   ' old files will be deleted and it will generate test vector files with the latest version')
             # Clear the existing case_dir folder
             shutil.rmtree(case_dir)
@@ -356,7 +356,7 @@ def generate_test_vector(test_case, case_dir, log_file, file_mode):
     test_end = time.time()
     span = round(test_end - test_start, 2)
     if span > TIME_THRESHOLD_TO_PRINT:
-        print(f'    - generated in {span} seconds')
+        print(f'- generated in {span} seconds')
 
     return result
 

diff --git a/tests/generators/ssz_generic/ssz_basic_vector.py b/tests/generators/ssz_generic/ssz_basic_vector.py
@@ -33,7 +33,7 @@ def valid_cases():
         for length in [1, 2, 3, 4, 5, 8, 16, 31, 512, 513]:
             for mode in random_modes:
                 yield f'vec_{name}_{length}_{mode.to_name()}', \
-                      valid_test_case(lambda: basic_vector_case_fn(rng, mode, typ, length))
+                    valid_test_case(lambda: basic_vector_case_fn(rng, mode, typ, length))
 
 
 def invalid_cases():
@@ -52,14 +52,14 @@ def invalid_cases():
                 if length == 1:
                     # empty bytes, no elements. It may seem valid, but empty fixed-size elements are not valid SSZ.
                     yield f'vec_{name}_{length}_{mode.to_name()}_one_less', \
-                          invalid_test_case(lambda: b"")
+                        invalid_test_case(lambda: b"")
                 else:
                     yield f'vec_{name}_{length}_{mode.to_name()}_one_less', \
-                          invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length - 1)))
+                        invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length - 1)))
                 yield f'vec_{name}_{length}_{mode.to_name()}_one_more', \
-                      invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length + 1)))
+                    invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length + 1)))
                 yield f'vec_{name}_{length}_{mode.to_name()}_one_byte_less', \
-                      invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))[:-1])
+                    invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))[:-1])
                 yield f'vec_{name}_{length}_{mode.to_name()}_one_byte_more', \
-                      invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))
-                                        + serialize(basic_vector_case_fn(rng, mode, uint8, 1)))
+                    invalid_test_case(lambda: serialize(basic_vector_case_fn(rng, mode, typ, length))
+                                      + serialize(basic_vector_case_fn(rng, mode, uint8, 1)))
diff --git a/tests/generators/ssz_generic/ssz_bitlist.py b/tests/generators/ssz_generic/ssz_bitlist.py
@@ -22,7 +22,7 @@ def valid_cases():
                          RandomizationMode.mode_zero,
                          RandomizationMode.mode_max]:
                 yield f'bitlist_{size}_{mode.to_name()}_{variation}', \
-                      valid_test_case(lambda: bitlist_case_fn(rng, mode, size))
+                    valid_test_case(lambda: bitlist_case_fn(rng, mode, size))
 
 
 def invalid_cases():
@@ -33,5 +33,5 @@ def invalid_cases():
     for (typ_limit, test_limit) in [(1, 2), (1, 8), (1, 9), (2, 3), (3, 4), (4, 5),
                                     (5, 6), (8, 9), (32, 64), (32, 33), (512, 513)]:
         yield f'bitlist_{typ_limit}_but_{test_limit}', \
-              invalid_test_case(lambda: serialize(
-                  bitlist_case_fn(rng, RandomizationMode.mode_max_count, test_limit)))
+            invalid_test_case(lambda: serialize(
+                bitlist_case_fn(rng, RandomizationMode.mode_max_count, test_limit)))
diff --git a/tests/generators/ssz_generic/ssz_bitvector.py b/tests/generators/ssz_generic/ssz_bitvector.py
@@ -38,5 +38,5 @@ def invalid_cases():
                                   (5, 6), (8, 9), (9, 8), (16, 8), (32, 33), (512, 513)]:
         for mode in [RandomizationMode.mode_random, RandomizationMode.mode_zero, RandomizationMode.mode_max]:
             yield f'bitvec_{typ_size}_{mode.to_name()}_{test_size}', \
-                  invalid_test_case(lambda: serialize(bitvector_case_fn(rng, mode, test_size,
-                                                                        invalid_making_pos=typ_size)))
+                invalid_test_case(lambda: serialize(bitvector_case_fn(rng, mode, test_size,
+                                                    invalid_making_pos=typ_size)))
diff --git a/tests/generators/ssz_generic/ssz_container.py b/tests/generators/ssz_generic/ssz_container.py
@@ -46,7 +46,7 @@ class BitsStruct(Container):
     E: Bitvector[8]
 
 
-def container_case_fn(rng: Random, mode: RandomizationMode, typ: Type[View], chaos: bool=False):
+def container_case_fn(rng: Random, mode: RandomizationMode, typ: Type[View], chaos: bool = False):
     return get_random_ssz_object(rng, typ,
                                  max_bytes_length=2000,
                                  max_list_length=2000,
@@ -77,15 +77,15 @@ def valid_cases():
         for mode in modes:
             for variation in range(3):
                 yield f'{name}_{mode.to_name()}_chaos_{variation}', \
-                      valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True))
+                    valid_test_case(lambda: container_case_fn(rng, mode, typ, chaos=True))
         # Notes: Below is the second wave of iteration, and only the random mode is selected
         # for container without offset since ``RandomizationMode.mode_zero`` and ``RandomizationMode.mode_max``
         # are deterministic.
         modes = [RandomizationMode.mode_random] if len(offsets) == 0 else list(RandomizationMode)
         for mode in modes:
             for variation in range(10):
                 yield f'{name}_{mode.to_name()}_{variation}', \
-                      valid_test_case(lambda: container_case_fn(rng, mode, typ))
+                    valid_test_case(lambda: container_case_fn(rng, mode, typ))
 
 
 def mod_offset(b: bytes, offset_index: int, change: Callable[[int], int]):
@@ -100,8 +100,8 @@ def invalid_cases():
     for (name, (typ, offsets)) in PRESET_CONTAINERS.items():
         # using mode_max_count, so that the extra byte cannot be picked up as normal list content
         yield f'{name}_extra_byte', \
-              invalid_test_case(lambda: serialize(
-                  container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff')
+            invalid_test_case(lambda: serialize(
+                container_case_fn(rng, RandomizationMode.mode_max_count, typ)) + b'\xff')
 
         if len(offsets) != 0:
             # Note: there are many more ways to have invalid offsets,
@@ -112,23 +112,22 @@ def invalid_cases():
                          RandomizationMode.mode_max_count]:
                 for index, offset_index in enumerate(offsets):
                     yield f'{name}_{mode.to_name()}_offset_{offset_index}_plus_one', \
-                          invalid_test_case(lambda: mod_offset(
-                              b=serialize(container_case_fn(rng, mode, typ)),
-                              offset_index=offset_index,
-                              change=lambda x: x + 1
-                          ))
+                        invalid_test_case(lambda: mod_offset(
+                            b=serialize(container_case_fn(rng, mode, typ)),
+                            offset_index=offset_index,
+                            change=lambda x: x + 1))
                     yield f'{name}_{mode.to_name()}_offset_{offset_index}_zeroed', \
-                          invalid_test_case(lambda: mod_offset(
-                              b=serialize(container_case_fn(rng, mode, typ)),
-                              offset_index=offset_index,
-                              change=lambda x: 0
-                          ))
+                        invalid_test_case(lambda: mod_offset(
+                            b=serialize(container_case_fn(rng, mode, typ)),
+                            offset_index=offset_index,
+                            change=lambda x: 0
+                        ))
                     if index == 0:
                         yield f'{name}_{mode.to_name()}_offset_{offset_index}_minus_one', \
                             invalid_test_case(lambda: mod_offset(
-                              b=serialize(container_case_fn(rng, mode, typ)),
-                              offset_index=offset_index,
-                              change=lambda x: x - 1
+                                b=serialize(container_case_fn(rng, mode, typ)),
+                                offset_index=offset_index,
+                                change=lambda x: x - 1
                             ))
                     if mode == RandomizationMode.mode_max_count:
                         serialized = serialize(container_case_fn(rng, mode, typ))

diff --git a/tests/generators/ssz_generic/ssz_uints.py b/tests/generators/ssz_generic/ssz_uints.py
@@ -21,7 +21,7 @@ def valid_cases():
         mode = RandomizationMode.mode_random
         byte_len = uint_type.type_byte_length()
         yield f'uint_{byte_len * 8}_last_byte_empty', \
-              valid_test_case(lambda: uint_type((2 ** ((byte_len - 1) * 8)) - 1))
+            valid_test_case(lambda: uint_type((2 ** ((byte_len - 1) * 8)) - 1))
         for variation in range(5):
             yield f'uint_{byte_len * 8}_{mode.to_name()}_{variation}', \
                 valid_test_case(lambda: uint_case_fn(rng, mode, uint_type))
@@ -38,8 +38,8 @@ def invalid_cases():
     for uint_type in [uint8, uint16, uint32, uint64, uint128, uint256]:
         byte_len = uint_type.type_byte_length()
         yield f'uint_{byte_len * 8}_one_byte_longer', \
-              invalid_test_case(lambda: (2 ** (byte_len * 8) - 1).to_bytes(byte_len + 1, 'little'))
+            invalid_test_case(lambda: (2 ** (byte_len * 8) - 1).to_bytes(byte_len + 1, 'little'))
     for uint_type in [uint8, uint16, uint32, uint64, uint128, uint256]:
         byte_len = uint_type.type_byte_length()
         yield f'uint_{byte_len * 8}_one_byte_shorter', \
-              invalid_test_case(lambda: (2 ** ((byte_len - 1) * 8) - 1).to_bytes(byte_len - 1, 'little'))
+            invalid_test_case(lambda: (2 ** ((byte_len - 1) * 8) - 1).to_bytes(byte_len - 1, 'little'))