tests-misc.yml

name: tests misc

on:
  push:
    branches:
      - master
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review]
  merge_group:
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

# Jobs in this workflow depend on each other, only for limiting peak amount of spawned workers

jobs:
  preflight:
    uses: ./.github/workflows/reusable-preflight.yml

  # more information about this job can be found here:
  # https://github.com/paritytech/substrate/pull/3778
  test-full-crypto-feature:
    needs: [preflight]
    runs-on: ${{ needs.preflight.outputs.RUNNER }}
    if: ${{ needs.preflight.outputs.changes_rust }}
    timeout-minutes: 60
    container:
      image: ${{ needs.preflight.outputs.IMAGE }}
    env:
      # Enable debug assertions since we are running optimized builds for testing
      # but still want to have debug assertions.
      RUSTFLAGS: "-C debug-assertions"
      RUST_BACKTRACE: 1
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: script
        run: |
          cd substrate/primitives/core/
          forklift cargo build --locked --no-default-features --features full_crypto
          cd ../application-crypto
          forklift cargo build --locked --no-default-features --features full_crypto

  test-frame-examples-compile-to-wasm:
    timeout-minutes: 20
    # into one job
    needs: [preflight, test-full-crypto-feature]
    runs-on: ${{ needs.preflight.outputs.RUNNER }}
    if: ${{ needs.preflight.outputs.changes_rust }}
    container:
      image: ${{ needs.preflight.outputs.IMAGE }}
    env:
      # Enable debug assertions since we are running optimized builds for testing
      # but still want to have debug assertions.
      RUSTFLAGS: "-C debug-assertions"
      RUST_BACKTRACE: 1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: script
        run: |
          cd substrate/frame/examples/offchain-worker/
          forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features
          cd ../basic
          forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features

  test-frame-ui:
    timeout-minutes: 60
    needs: [preflight]
    runs-on: ${{ needs.preflight.outputs.RUNNER }}
    if: ${{ needs.preflight.outputs.changes_rust }}
    container:
      image: ${{ needs.preflight.outputs.IMAGE }}
    env:
      # Enable debug assertions since we are running optimized builds for testing
      # but still want to have debug assertions.
      RUSTFLAGS: "-C debug-assertions -D warnings"
      RUST_BACKTRACE: 1
      SKIP_WASM_BUILD: 1
      # Ensure we run the UI tests.
      RUN_UI_TESTS: 1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: script
        run: |
          cargo version
          forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental ui
          forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental ui
          forklift cargo test --locked -q --profile testnet -p xcm-procedural ui
          forklift cargo test --locked -q --profile testnet -p frame-election-provider-solution-type ui
          forklift cargo test --locked -q --profile testnet -p sp-api-test ui
          # There is multiple version of sp-runtime-interface in the repo. So we point to the manifest.
          forklift cargo test --locked -q --profile testnet --manifest-path substrate/primitives/runtime-interface/Cargo.toml ui

  test-deterministic-wasm:
    timeout-minutes: 20
    needs: [preflight, test-frame-examples-compile-to-wasm]
    runs-on: ${{ needs.preflight.outputs.RUNNER }}
    if: ${{ needs.preflight.outputs.changes_rust }}
    container:
      image: ${{ needs.preflight.outputs.IMAGE }}
    env:
      WASM_BUILD_NO_COLOR: 1
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: script
        run: |
          # build runtime
          forklift cargo build -q --locked --release -p westend-runtime -p rococo-runtime
          # make checksum
          sha256sum target/release/wbuild/*-runtime/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256
          cargo clean
          # build again
          forklift cargo build -q --locked --release -p westend-runtime -p rococo-runtime
          # confirm checksum
          sha256sum -c checksum.sha256

  cargo-check-benches:
    needs: [preflight]
    if: ${{ github.event_name == 'pull_request' || github.event_name == 'merge_group' }}
    timeout-minutes: 60
    strategy:
      matrix:
        branch: [master, current]
    runs-on: ${{ needs.preflight.outputs.RUNNER }}
    container:
      image: ${{ needs.preflight.outputs.IMAGE }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          # if branch is master, use the branch, otherwise set empty string, so it uses the current context
          # either PR (including forks) or merge group (main repo)
          ref: ${{ matrix.branch == 'master' && matrix.branch || '' }}

      - name: script
        run: |
          ARTIFACTS_DIR=./artifacts
          BENCH_TRIE_READ=::trie::read::small
          BENCH_NODE_IMPORT=::node::import::sr25519::transfer_keep_alive::paritydb::small
          mkdir -p $ARTIFACTS_DIR

          SKIP_WASM_BUILD=1 forklift cargo check --locked --benches --all;
          forklift cargo run --locked --release -p node-bench -- $BENCH_TRIE_READ --json | tee $ARTIFACTS_DIR/bench_trie_read_small.json;
          forklift cargo run --locked --release -p node-bench -- $BENCH_NODE_IMPORT --json | tee $ARTIFACTS_DIR/bench_transfer_keep_alive.json

      - name: Upload artifacts
        uses: actions/upload-artifact@v4.3.6
        with:
          path: ./artifacts
          name: cargo-check-benches-${{ matrix.branch }}-${{ github.sha }}
          retention-days: 1

  node-bench-regression-guard:
    timeout-minutes: 20
    if: always() && !cancelled()
    runs-on: ubuntu-latest
    needs: [preflight, cargo-check-benches]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Download artifact (master run)
        uses: actions/download-artifact@v4.1.8
        continue-on-error: true
        with:
          name: cargo-check-benches-master-${{ github.sha }}
          path: ./artifacts/master

      - name: Download artifact (current run)
        uses: actions/download-artifact@v4.1.8
        continue-on-error: true
        with:
          name: cargo-check-benches-current-${{ github.sha }}
          path: ./artifacts/current

      - name: script
        id: compare
        run: |
          if [ "${{ github.ref_name }}" = "master" ]; then
            echo -e "Exiting on master branch"
            exit 0
          fi

          # fail if no artifacts
          if [ ! -d ./artifacts/master ] || [ ! -d ./artifacts/current ]; then
            echo "No artifacts found"
            exit 1
          fi

          docker run --rm \
          -v $PWD/artifacts/master:/artifacts/master \
          -v $PWD/artifacts/current:/artifacts/current \
          paritytech/node-bench-regression-guard:latest \
          node-bench-regression-guard --reference /artifacts/master --compare-with /artifacts/current

          if [ $? -ne 0 ]; then
            FAILED_MSG='### node-bench-regression-guard failed ❌, check the regression in *cargo-check-benches* job'
            echo $FAILED_MSG
            echo $FAILED_MSG >> $GITHUB_STEP_SUMMARY
            exit 1
          else
            echo "### node-bench-regression-guard passed ✅" >> $GITHUB_STEP_SUMMARY
          fi

  test-node-metrics:
    needs: [preflight]
    timeout-minutes: 30
    runs-on: ${{ needs.preflight.outputs.RUNNER }}
    if: ${{ needs.preflight.outputs.changes_rust }}
    container:
      image: ${{ needs.preflight.outputs.IMAGE }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Run tests
        id: tests
        env:
          RUST_TOOLCHAIN: stable
          # Enable debug assertions since we are running optimized builds for testing
          # but still want to have debug assertions.
          RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
        run: |
          forklift cargo build --bin polkadot-execute-worker --bin polkadot-prepare-worker --profile testnet --verbose --locked
          mkdir -p ./artifacts
          forklift cargo test --profile testnet --locked --features=runtime-metrics -p polkadot-node-metrics > ./artifacts/log.txt
          echo "Metrics test passed"

      - name: Upload artifacts if failed
        if: ${{ steps.tests.outcome != 'success' }}
        uses: actions/upload-artifact@v4.3.6
        with:
          name: node-metrics-failed
          path: ./artifacts

  # more information about this job can be found here:
  # https://github.com/paritytech/substrate/pull/6916
  check-tracing:
    timeout-minutes: 20
    needs: [preflight, test-node-metrics]
    runs-on: ${{ needs.preflight.outputs.RUNNER }}
    if: ${{ needs.preflight.outputs.changes_rust }}
    container:
      image: ${{ needs.preflight.outputs.IMAGE }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: script
        run: |
          forklift cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features
          forklift cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features --features=with-tracing

  check-metadata-hash:
    timeout-minutes: 20
    needs: [preflight, check-tracing]
    runs-on: ${{ needs.preflight.outputs.RUNNER }}
    if: ${{ needs.preflight.outputs.changes_rust }}
    container:
      image: ${{ needs.preflight.outputs.IMAGE }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: script
        run: |
          forklift cargo build --locked -p westend-runtime --features metadata-hash

  # disabled until https://github.com/paritytech/polkadot-sdk/issues/5812 is resolved
  # cargo-hfuzz:
  #   timeout-minutes: 20
  #   needs: [preflight, check-metadata-hash]
  #   runs-on: ${{ needs.preflight.outputs.RUNNER }}
  #   container:
  #     image: ${{ needs.preflight.outputs.IMAGE }}
  #   env:
  #     # max 10s per iteration, 60s per file
  #     HFUZZ_RUN_ARGS: |
  #       --exit_upon_crash
  #       --exit_code_upon_crash 1
  #       --timeout 10
  #       --run_time 60

  #     # use git version of honggfuzz-rs until v0.5.56 is out, we need a few recent changes:
  #     # https://github.com/rust-fuzz/honggfuzz-rs/pull/75 to avoid breakage on debian
  #     # https://github.com/rust-fuzz/honggfuzz-rs/pull/81 fix to the above pr
  #     # https://github.com/rust-fuzz/honggfuzz-rs/pull/82 fix for handling absolute CARGO_TARGET_DIR
  #     HFUZZ_BUILD_ARGS: |
  #       --config=patch.crates-io.honggfuzz.git="https://github.com/altaua/honggfuzz-rs"
  #       --config=patch.crates-io.honggfuzz.rev="205f7c8c059a0d98fe1cb912cdac84f324cb6981"
  #   steps:
  #     - name: Checkout
  #       uses: actions/checkout@v4

  #     - name: Run honggfuzz
  #       run: |
  #         cd substrate/primitives/arithmetic/fuzzer
  #         forklift cargo hfuzz build
  #         for target in $(cargo read-manifest | jq -r '.targets | .[] | .name');
  #         do
  #           forklift cargo hfuzz run "$target" || { printf "fuzzing failure for %s\n" "$target"; exit 1; };
  #         done

  #     - name: Upload artifacts
  #       uses: actions/upload-artifact@v4.3.6
  #       with:
  #         path: substrate/primitives/arithmetic/fuzzer/hfuzz_workspace/
  #         name: hfuzz-${{ github.sha }}

  cargo-check-each-crate:
    timeout-minutes: 70
    needs: [preflight]
    runs-on: ${{ needs.preflight.outputs.RUNNER }}
    if: ${{ needs.preflight.outputs.changes_rust }}
    container:
      image: ${{ needs.preflight.outputs.IMAGE }}
    env:
      RUSTFLAGS: "-D warnings"
      CI_JOB_NAME: cargo-check-each-crate
    strategy:
      matrix:
        index: [1, 2, 3, 4, 5, 6, 7] # 7 parallel jobs
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Check Rust
        run: |
          rustup show
          rustup +nightly show

      - name: script
        run: |
          mkdir -p /github/home/.forklift
          cp .forklift/config.toml /github/home/.forklift/config.toml
          PYTHONUNBUFFERED=x .github/scripts/check-each-crate.py ${{ matrix.index }} ${{ strategy.job-total }}

  cargo-check-all-crate-macos:
    timeout-minutes: 30
    needs: [preflight]
    runs-on: ${{ needs.preflight.outputs.RUNNER_MACOS }}
    if: ${{ needs.preflight.outputs.changes_rust }}
    env:
      SKIP_WASM_BUILD: 1
    steps:
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
      - name: Set rust version from env file
        run: |
          RUST_VERSION=$(cat .github/env | sed -E 's/.*ci-unified:([^-]+)-([^-]+).*/\2/')
          echo $RUST_VERSION
          echo "RUST_VERSION=${RUST_VERSION}" >> $GITHUB_ENV
      - name: Set up Homebrew
        uses: Homebrew/actions/setup-homebrew@1ccc07ccd54b6048295516a3eb89b192c35057dc # master from 12.09.2024
      - name: Install rust ${{ env.RUST_VERSION }}
        uses: actions-rust-lang/setup-rust-toolchain@11df97af8e8102fd60b60a77dfbf58d40cd843b8 # v1.10.1
        with:
          cache: false
          toolchain: ${{ env.RUST_VERSION }}
          target: wasm32-unknown-unknown
          components: cargo, clippy, rust-docs, rust-src, rustfmt, rustc, rust-std
      - name: Install protobuf
        run: brew install protobuf
      - name: cargo info
        run: |
          echo "######## rustup show ########"
          rustup show
          echo "######## cargo --version ########"
          cargo --version
      - name: Run cargo check
        run: cargo check --workspace --locked

  confirm-required-test-misc-jobs-passed:
    runs-on: ubuntu-latest
    name: All test misc tests passed
    # If any new job gets added, be sure to add it to this array
    needs:
      - test-full-crypto-feature
      - test-frame-examples-compile-to-wasm
      - test-frame-ui
      - cargo-check-benches
      - node-bench-regression-guard
      - test-node-metrics
      - check-tracing
      - cargo-check-each-crate
      - test-deterministic-wasm
      - cargo-check-all-crate-macos
      # - cargo-hfuzz remove from required for now, as it's flaky
    if: always() && !cancelled()
    steps:
      - run: |
          tee resultfile <<< '${{ toJSON(needs) }}'
          FAILURES=$(cat resultfile | grep '"result": "failure"' | wc -l)
          if [ $FAILURES -gt 0 ]; then
            echo "### At least one required job failed ❌" >> $GITHUB_STEP_SUMMARY
            exit 1
          else
            echo '### Good job! All the required jobs passed 🚀' >> $GITHUB_STEP_SUMMARY
          fi