Merge branch 'master' into gpestana/403-treasury_mint_remainder

.config/taplo.toml

@@ -2,6 +2,7 @@
 
 # ignore zombienet as they do some deliberate custom toml stuff
 exclude = [
+	"bridges/testing/**",
 	"cumulus/zombienet/**",
 	"polkadot/node/malus/integrationtests/**",
 	"polkadot/zombienet_tests/**",

.github/scripts/check-workspace.py

@@ -27,6 +27,7 @@ def main(root, exclude):
 	all_crates = get_crates(root, exclude)
 	print(f'📦 Found {len(all_crates)} crates in total')
 
+	check_duplicates(workspace_crates)
 	check_missing(workspace_crates, all_crates)
 	check_links(all_crates)
 
@@ -89,6 +90,16 @@ def get_crates(workspace_dir, exclude_crates) -> dict:
 
 	return crates
 
+# Check that there are no duplicate entries in the workspace.
+def check_duplicates(workspace_crates):
+	print(f'🔎 Checking for duplicate crates')
+	found = {}
+	for path in workspace_crates:
+		if path in found:
+			print(f'❌ crate is listed twice in the workspace {path}')
+			sys.exit(1)
+		found[path] = True
+
 # Check that all crates are in the workspace.
 def check_missing(workspace_crates, all_crates):
 	print(f'🔎 Checking for missing crates')

.github/workflows/check-labels.yml

@@ -9,14 +9,6 @@ jobs:
   check-labels:
     runs-on: ubuntu-latest
     steps:
-      - name: Skip merge queue
-        if: ${{ contains(github.ref, 'gh-readonly-queue') }}
-        run: exit 0
-      - name: Pull image
-        env:
-          IMAGE: paritytech/ruled_labels:0.4.0
-        run: docker pull $IMAGE
-
       - name: Check labels
         env:
           IMAGE: paritytech/ruled_labels:0.4.0
@@ -28,6 +20,16 @@ jobs:
           RULES_PATH: labels/ruled_labels
           CHECK_SPECS: "specs_polkadot-sdk.yaml"
         run: |
+          if [ ${{ github.ref }} == "refs/heads/master" ]; then
+            echo "Skipping master"
+            exit 0
+          fi
+          if [ $(echo ${{ github.ref }} | grep -c "gh-readonly-queue") -eq 1 ]; then
+            echo "Skipping merge queue"
+            exit 0
+          fi
+
+          docker pull $IMAGE
           echo "REPO: ${REPO}"
           echo "GITHUB_PR: ${GITHUB_PR}"
 

.github/workflows/check-prdoc.yml

@@ -17,31 +17,25 @@ env:
 jobs:
   check-prdoc:
     runs-on: ubuntu-latest
+    if: github.event.pull_request.number != ''
     steps:
+      - name: Checkout repo
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1
       # we cannot show the version in this step (ie before checking out the repo)
       # due to https://github.com/paritytech/prdoc/issues/15
-      - name: Skip merge queue
-        if: ${{ contains(github.ref, 'gh-readonly-queue') }}
-        run: exit 0
-      - name: Pull image
+      - name: Check if PRdoc is required
+        id: get-labels
         run: |
           echo "Pulling $IMAGE"
           $ENGINE pull $IMAGE
 
-      - name: Check if PRdoc is required
-        id: get-labels
-        run: |
           # Fetch the labels for the PR under test
           echo "Fetch the labels for $API_BASE/${REPO}/pulls/${GITHUB_PR}"
           labels=$( curl -H "Authorization: token ${GITHUB_TOKEN}" -s "$API_BASE/${REPO}/pulls/${GITHUB_PR}" | jq '.labels | .[] | .name' | tr "\n" ",")
           echo "Labels: ${labels}"
           echo "labels=${labels}" >> "$GITHUB_OUTPUT"
 
-      - name: Checkout repo
-        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 #v4.1.1
-
-      - name: Check PRDoc version
-        run: |
+          echo "Checking PRdoc version"
           $ENGINE run --rm -v $PWD:/repo $IMAGE --version
 
       - name: Early exit if PR is silent

.github/workflows/check-workspace.yml

@@ -2,8 +2,6 @@ name: Check workspace
 
 on:
   pull_request:
-    paths:
-      - "*.toml"
   merge_group:
 
 jobs:
@@ -19,5 +17,5 @@ jobs:
         run: >
             python3 .github/scripts/check-workspace.py .
             --exclude
-            "substrate/frame/contracts/fixtures/build" 
+            "substrate/frame/contracts/fixtures/build"
             "substrate/frame/contracts/fixtures/contracts/common"

.github/workflows/fmt-check.yml

@@ -15,7 +15,7 @@ jobs:
         os: ["ubuntu-latest"]
     runs-on: ${{ matrix.os }}
     container:
-      image: paritytech/ci-unified:bullseye-1.74.0-2023-11-01-v20231204
+      image: docker.io/paritytech/ci-unified:bullseye-1.75.0-2024-01-22-v20240109
     steps:
       - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 

.github/workflows/gitspiegel-trigger.yml

@@ -13,14 +13,15 @@ on:
       - unlocked
       - ready_for_review
       - reopened
+  # doesn't work as intended, triggers "workflow_run" webhook in any case
   # the job doesn't check out any code, so it is relatively safe to run it on any event
-  pull_request_target:
-    types:
-      - opened
-      - synchronize
-      - unlocked
-      - ready_for_review
-      - reopened
+  # pull_request_target:
+  #   types:
+  #     - opened
+  #     - synchronize
+  #     - unlocked
+  #     - ready_for_review
+  #     - reopened
   merge_group:
 
 # drop all permissions for GITHUB_TOKEN

.github/workflows/review-bot.yml

@@ -23,7 +23,7 @@ jobs:
           app_id: ${{ secrets.REVIEW_APP_ID }}
           private_key: ${{ secrets.REVIEW_APP_KEY }}
       - name: "Evaluates PR reviews and assigns reviewers"
-        uses: paritytech/review-bot@v2.3.0
+        uses: paritytech/review-bot@v2.4.0
         with:
           repo-token: ${{ steps.app_token.outputs.token }}
           team-token: ${{ steps.app_token.outputs.token }}

.gitlab-ci.yml

@@ -121,22 +121,17 @@ default:
   before_script:
     - 'curl --header "PRIVATE-TOKEN: $FL_CI_GROUP_TOKEN" -o forklift  -L "${CI_API_V4_URL}/projects/676/packages/generic/forklift/${FL_FORKLIFT_VERSION}/forklift_${FL_FORKLIFT_VERSION}_linux_amd64"'
     - chmod +x forklift
-    - mkdir .forklift
-    - cp $FL_FORKLIFT_CONFIG .forklift/config.toml
-    - export FORKLIFT_PACKAGE_SUFFIX=${CI_JOB_NAME/ [0-9 \/]*}
+    - mkdir ~/.forklift
+    - cp $FL_FORKLIFT_CONFIG ~/.forklift/config.toml
     - shopt -s expand_aliases
     - export PATH=$PATH:$(pwd)
     - |
       if [ "$FORKLIFT_BYPASS" != "true" ]; then
       echo "FORKLIFT_BYPASS not set, creating alias cargo='forklift cargo'"
       alias cargo="forklift cargo"
       fi
-    - ls -al
-    - rm -f forklift.sock
-    - forklift clean
     #
     - echo "FL_FORKLIFT_VERSION ${FL_FORKLIFT_VERSION}"
-    - echo "FORKLIFT_PACKAGE_SUFFIX $FORKLIFT_PACKAGE_SUFFIX"
 
 .common-refs:
   rules:

.gitlab/check-each-crate.py

@@ -55,7 +55,7 @@
 
 	print(f"Checking {crates[crate][0]}", file=sys.stderr)
 
-	res = subprocess.run(["cargo", "check", "--locked"], cwd = crates[crate][1])
+	res = subprocess.run(["forklift", "cargo", "check", "--locked"], cwd = crates[crate][1])
 
 	if res.returncode != 0:
 		sys.exit(1)

.gitlab/pipeline/build.yml

@@ -314,8 +314,9 @@ build-linux-substrate:
     # tldr: we need to checkout the branch HEAD explicitly because of our dynamic versioning approach while building the substrate binary
     # see https://github.com/paritytech/ci_cd/issues/682#issuecomment-1340953589
     - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
+    - !reference [.forklift-cache, before_script]
   script:
-    - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release -p staging-node-cli
+    - time WASM_BUILD_NO_COLOR=1 cargo build --locked --release -p staging-node-cli
     - mv $CARGO_TARGET_DIR/release/substrate-node ./artifacts/substrate/substrate
     - echo -n "Substrate version = "
     - if [ "${CI_COMMIT_TAG}" ]; then
@@ -329,6 +330,17 @@ build-linux-substrate:
     # - printf '\n# building node-template\n\n'
     # - ./scripts/ci/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
 
+build-runtimes-polkavm:
+  stage: build
+  extends:
+    - .docker-env
+    - .common-refs
+  script:
+    - SUBSTRATE_RUNTIME_TARGET=riscv cargo check -p minimal-runtime
+    - SUBSTRATE_RUNTIME_TARGET=riscv cargo check -p westend-runtime
+    - SUBSTRATE_RUNTIME_TARGET=riscv cargo check -p rococo-runtime
+    - SUBSTRATE_RUNTIME_TARGET=riscv cargo check -p polkadot-test-runtime
+
 .build-subkey:
   stage: build
   extends:
@@ -341,9 +353,10 @@ build-linux-substrate:
     CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target"
   before_script:
     - mkdir -p ./artifacts/subkey
+    - !reference [.forklift-cache, before_script]
   script:
     - cd ./substrate/bin/utils/subkey
-    - SKIP_WASM_BUILD=1 time cargo build --locked --release
+    - time SKIP_WASM_BUILD=1 cargo build --locked --release
     # - cd -
     # - mv $CARGO_TARGET_DIR/release/subkey ./artifacts/subkey/.
     # - echo -n "Subkey version = "
@@ -382,3 +395,18 @@ build-subkey-linux:
 #   after_script: [""]
 #   tags:
 #     - osx
+
+# bridges
+
+# we need some non-binary artifacts in our bridges+zombienet image
+prepare-bridges-zombienet-artifacts:
+  stage: build
+  extends:
+    - .docker-env
+    - .common-refs
+    - .run-immediately
+    - .collect-artifacts
+  before_script:
+    - mkdir -p ./artifacts/bridges-polkadot-sdk/bridges
+  script:
+    - cp -r bridges/testing ./artifacts/bridges-polkadot-sdk/bridges/testing

.gitlab/pipeline/publish.yml

@@ -66,6 +66,8 @@ publish-rustdoc:
 # note: images are used not only in zombienet but also in rococo, wococo and versi
 .build-push-image:
   image: $BUILDAH_IMAGE
+  extends:
+    - .zombienet-refs
   variables:
     DOCKERFILE: "" # docker/path-to.Dockerfile
     IMAGE_NAME: "" # docker.io/paritypr/image_name
@@ -77,6 +79,7 @@ publish-rustdoc:
       --build-arg VCS_REF="${CI_COMMIT_SHA}"
       --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
       --build-arg IMAGE_NAME="${IMAGE_NAME}"
+      --build-arg ZOMBIENET_IMAGE="${ZOMBIENET_IMAGE}"
       --tag "$IMAGE_NAME:${DOCKER_IMAGES_VERSION}"
       --file ${DOCKERFILE} .
     - echo "$PARITYPR_PASS" |
@@ -163,3 +166,22 @@ build-push-image-substrate-pr:
   variables:
     DOCKERFILE: "docker/dockerfiles/substrate_injected.Dockerfile"
     IMAGE_NAME: "docker.io/paritypr/substrate"
+
+# unlike other images, bridges+zombienet image is based on Zombienet image that pulls required binaries
+# from other fresh images (polkadot and cumulus)
+build-push-image-bridges-zombienet-tests:
+  stage: publish
+  extends:
+    - .kubernetes-env
+    - .common-refs
+    - .build-push-image
+  needs:
+    - job: build-linux-stable
+      artifacts: true
+    - job: build-linux-stable-cumulus
+      artifacts: true
+    - job: prepare-bridges-zombienet-artifacts
+      artifacts: true
+  variables:
+    DOCKERFILE: "docker/dockerfiles/bridges_zombienet_tests_injected.Dockerfile"
+    IMAGE_NAME: "docker.io/paritypr/bridges-zombienet-tests"

.gitlab/pipeline/short-benchmarks.yml

@@ -84,6 +84,16 @@ short-benchmark-coretime-westend:
   variables:
     RUNTIME_CHAIN: coretime-westend-dev
 
+short-benchmark-people-rococo:
+  <<: *short-bench-cumulus
+  variables:
+    RUNTIME_CHAIN: people-rococo-dev
+
+short-benchmark-people-westend:
+  <<: *short-bench-cumulus
+  variables:
+    RUNTIME_CHAIN: people-westend-dev
+
 short-benchmark-glutton-westend:
   <<: *short-bench-cumulus
   variables:

.gitlab/pipeline/test.yml

@@ -224,6 +224,7 @@ cargo-check-benches:
       git merge --verbose --no-edit FETCH_HEAD;
       fi
       fi'
+    - !reference [.forklift-cache, before_script]
   parallel: 2
   script:
     - mkdir -p ./artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA

.gitlab/pipeline/zombienet.yml

@@ -10,3 +10,5 @@ include:
   - .gitlab/pipeline/zombienet/cumulus.yml
   # polkadot tests
   - .gitlab/pipeline/zombienet/polkadot.yml
+  # bridges tests
+  - .gitlab/pipeline/zombienet/bridges.yml

.gitlab/pipeline/zombienet/bridges.yml

@@ -0,0 +1,62 @@
+# This file is part of .gitlab-ci.yml
+# Here are all jobs that are executed during "zombienet" stage for bridges
+
+# common settings for all zombienet jobs
+.zombienet-bridges-common:
+  extends:
+    - .kubernetes-env
+    - .zombienet-refs
+  rules:
+    - !reference [.build-refs, rules]
+    - if: $CI_COMMIT_REF_NAME =~ /^gh-readonly-queue.*$/
+      variables:
+        DOCKER_IMAGES_VERSION: ${CI_COMMIT_SHORT_SHA}
+  before_script:
+    - echo "Zombienet Tests Config"
+    - echo "${ZOMBIENET_IMAGE}"
+    - echo "${GH_DIR}"
+    - echo "${LOCAL_DIR}"
+    - ls "${LOCAL_DIR}"
+    - export DEBUG=zombie,zombie::network-node
+    - export ZOMBIENET_INTEGRATION_TEST_IMAGE="${BRIDGES_ZOMBIENET_TESTS_IMAGE}":${BRIDGES_ZOMBIENET_TESTS_IMAGE_TAG}
+    - echo "${ZOMBIENET_INTEGRATION_TEST_IMAGE}"
+  stage: zombienet
+  image: "${BRIDGES_ZOMBIENET_TESTS_IMAGE}:${BRIDGES_ZOMBIENET_TESTS_IMAGE_TAG}"
+  needs:
+    - job: build-push-image-bridges-zombienet-tests
+      artifacts: true
+  variables:
+    BRIDGES_ZOMBIENET_TESTS_IMAGE_TAG: ${DOCKER_IMAGES_VERSION}
+    BRIDGES_ZOMBIENET_TESTS_IMAGE: "docker.io/paritypr/bridges-zombienet-tests"
+    GH_DIR: "https://github.com/paritytech/polkadot-sdk/tree/${CI_COMMIT_SHA}/bridges/testing"
+    LOCAL_DIR: "/builds/parity/mirrors/polkadot-sdk/bridges/testing"
+    FF_DISABLE_UMASK_FOR_DOCKER_EXECUTOR: 1
+    RUN_IN_CONTAINER: "1"
+  artifacts:
+    name: "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}_zombienet_bridge_tests"
+    when: always
+    expire_in: 2 days
+    paths:
+      - ./zombienet-logs
+  after_script:
+    - mkdir -p ./zombienet-logs
+    # copy general logs
+    - cp -r /tmp/bridges-tests-run-*/logs/* ./zombienet-logs/
+    # copy logs of rococo nodes
+    - cp -r /tmp/bridges-tests-run-*/bridge_hub_rococo_local_network/*.log ./zombienet-logs/
+    # copy logs of westend nodes
+    - cp -r /tmp/bridges-tests-run-*/bridge_hub_westend_local_network/*.log ./zombienet-logs/
+
+zombienet-bridges-0001-asset-transfer-works:
+  extends:
+    - .zombienet-bridges-common
+  script:
+    - /home/nonroot/bridges-polkadot-sdk/bridges/testing/run-new-test.sh 0001-asset-transfer --docker
+    - echo "Done"
+
+zombienet-bridges-0002-mandatory-headers-synced-while-idle:
+  extends:
+    - .zombienet-bridges-common
+  script:
+    - /home/nonroot/bridges-polkadot-sdk/bridges/testing/run-new-test.sh 0002-mandatory-headers-synced-while-idle --docker
+    - echo "Done"