From e1ad2d401d14104d8aa522ec11ac764028558761 Mon Sep 17 00:00:00 2001 From: EgorPopelyaev Date: Tue, 19 Dec 2023 11:43:13 +0100 Subject: [PATCH 1/5] add srtool gha --- .github/scripts/common/lib.sh | 33 +++++++ .github/workflows/srtool.yml | 156 ++++++++++++++++++++++++++++++++++ 2 files changed, 189 insertions(+) create mode 100644 .github/workflows/srtool.yml diff --git a/.github/scripts/common/lib.sh b/.github/scripts/common/lib.sh index e499dfabd643..446423edc55f 100755 --- a/.github/scripts/common/lib.sh +++ b/.github/scripts/common/lib.sh @@ -307,3 +307,36 @@ function increment_rc_tag() { ((suffix++)) echo $suffix } + +# Find all the runtimes, it returns the result as JSON object, compatible to be +# used as Github Workflow Matrix. This call is exposed by the `scan` command and can be used as: +# podman run --rm -it -v /.../fellowship-runtimes:/build docker.io/chevdor/srtool:1.70.0-0.11.1 scan +function find_runtimes() { + libs=($(git grep -I -r --cached --max-depth 20 --files-with-matches 'construct_runtime!' -- '*lib.rs')) + re=".*-runtime$" + JSON=$(jq --null-input '{ "include": [] }') + + # EXCLUDED_RUNTIMES is a space separated list of runtime names (without the -runtime postfix) + # EXCLUDED_RUNTIMES=${EXCLUDED_RUNTIMES:-"substrate-test"} + IFS=' ' read -r -a exclusions <<< "$EXCLUDED_RUNTIMES" + + for lib in "${libs[@]}"; do + crate_dir=$(dirname "$lib") + cargo_toml="$crate_dir/../Cargo.toml" + + name=$(toml get -r $cargo_toml 'package.name') + chain=${name//-runtime/} + + if [[ "$name" =~ $re ]] && ! [[ ${exclusions[@]} =~ $chain ]]; then + lib_dir=$(dirname "$lib") + runtime_dir=$(relative_parent "$lib_dir/..") + ITEM=$(jq --null-input \ + --arg chain "$chain" \ + --arg name "$name" \ + --arg runtime_dir "$runtime_dir" \ + '{ "chain": $chain, "crate": $name, "runtime_dir": $runtime_dir }') + JSON=$(echo $JSON | jq ".include += [$ITEM]") + fi + done + echo $JSON +} diff --git a/.github/workflows/srtool.yml b/.github/workflows/srtool.yml new file mode 100644 index 000000000000..68a7bf9d2e85 --- /dev/null +++ b/.github/workflows/srtool.yml @@ -0,0 +1,156 @@ +name: Srtool build + +env: + SUBWASM_VERSION: 0.20.0 + TOML_CLI_VERSION: 0.2.4 + +on: + push: + tags: + - "*" + branches: + - "release*" + + schedule: + - cron: "00 02 * * 1" # 2AM weekly on monday + + workflow_dispatch: + +jobs: + find-runtimes: + name: Scan repo paritytech/polkadot-sdk + outputs: + runtime: ${{ steps.get_runtimes_list.outputs.runtime }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + with: + fetch-depth: 0 + path: sdk + + - name: Install tooling + run: | + URL=https://github.com/chevdor/toml-cli/releases/download/v${{ env.TOML_CLI_VERSION }}/toml_linux_amd64_v${{ env.TOML_CLI_VERSION }}.deb + curl -L $URL --output toml.deb + sudo dpkg -i toml.deb + toml --version; jq --version + + - name: Scan runtimes + env: + EXCLUDED_RUNTIMES: "substrate-test" + run: | + . ./.github/scripts/common/lib.sh + + echo "Github workspace: ${{ github.workspace }}" + echo "Current folder: $(pwd)"; ls -al + cd sdk; ls -al + + MATRIX=$(find_runtimes | tee runtimes_list.json) + echo $MATRIX + + - name: Get runtimes list + id: get_runtimes_list + run: | + cd sdk; ls -al + MATRIX=$(cat runtimes_list.json) + echo $MATRIX + echo "runtime=$MATRIX" >> $GITHUB_OUTPUT + + srtool: + runs-on: ubuntu-latest + needs: + - find-runtimes + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.find-runtimes.outputs.runtime) }} + # strategy: + # matrix: + # include: + # - category: assets + # runtime: asset-hub-westend + # - category: bridge-hubs + # runtime: bridge-hub-polkadot + # - category: bridge-hubs + # runtime: bridge-hub-kusama + # - category: bridge-hubs + # runtime: bridge-hub-rococo + # - category: collectives + # runtime: collectives-polkadot + # - category: contracts + # runtime: contracts-rococo + # - category: starters + # runtime: seedling + # - category: starters + # runtime: shell + # - category: testing + # runtime: rococo-parachain + steps: + - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + with: + fetch-depth: 0 + path: sdk + + - name: Srtool build + id: srtool_build + uses: chevdor/srtool-actions@v0.9.1 + with: + workdir: sdk + chain: ${{ matrix.chain }} + runtime_dir: ${{ matrix.runtime_dir }} + + - name: Summary + run: | + echo '${{ steps.srtool_build.outputs.json }}' | jq > ${{ matrix.chain }}-srtool-digest.json + cat ${{ matrix.chain }}-srtool-digest.json + echo "Compact Runtime: ${{ steps.srtool_build.outputs.wasm }}" + echo "Compressed Runtime: ${{ steps.srtool_build.outputs.wasm_compressed }}" + + # it takes a while to build the runtime, so let's save the artifact as soon as we have it + - name: Archive Artifacts for ${{ matrix.chain }} + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{ matrix.chain }}-runtime + path: | + ${{ steps.srtool_build.outputs.wasm }} + ${{ steps.srtool_build.outputs.wasm_compressed }} + ${{ matrix.chain }}-srtool-digest.json + + # We now get extra information thanks to subwasm + - name: Install subwasm + run: | + wget https://github.com/chevdor/subwasm/releases/download/v${{ env.SUBWASM_VERSION }}/subwasm_linux_amd64_v${{ env.SUBWASM_VERSION }}.deb + sudo dpkg -i subwasm_linux_amd64_v${{ env.SUBWASM_VERSION }}.deb + subwasm --version + + - name: Show Runtime information + shell: bash + run: | + subwasm info ${{ steps.srtool_build.outputs.wasm }} + subwasm info ${{ steps.srtool_build.outputs.wasm_compressed }} + subwasm --json info ${{ steps.srtool_build.outputs.wasm }} > ${{ matrix.chain }}-info.json + subwasm --json info ${{ steps.srtool_build.outputs.wasm_compressed }} > ${{ matrix.chain }}-compressed-info.json + + - name: Extract the metadata + shell: bash + run: | + subwasm meta ${{ steps.srtool_build.outputs.wasm }} + subwasm --json meta ${{ steps.srtool_build.outputs.wasm }} > ${{ matrix.chain }}-metadata.json + + - name: Check the metadata diff + shell: bash + # the following subwasm call will error for chains that are not known and/or live, that includes shell for instance + run: | + subwasm diff ${{ steps.srtool_build.outputs.wasm }} --chain-b ${{ matrix.chain }} || \ + echo "Subwasm call failed, check the logs. This is likely because ${{ matrix.chain }} is not known by subwasm" | \ + tee ${{ matrix.chain }}-diff.txt + + - name: Archive Subwasm results + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{ matrix.chain }}-runtime + path: | + ${{ matrix.chain }}-info.json + ${{ matrix.chain }}-compressed-info.json + ${{ matrix.chain }}-metadata.json + ${{ matrix.chain }}-diff.txt From d168c2fb05310830a9c883d7d3a90734943da58f Mon Sep 17 00:00:00 2001 From: EgorPopelyaev Date: Tue, 19 Dec 2023 11:53:24 +0100 Subject: [PATCH 2/5] change working directory --- .github/workflows/srtool.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/srtool.yml b/.github/workflows/srtool.yml index 68a7bf9d2e85..b01a3069fcfe 100644 --- a/.github/workflows/srtool.yml +++ b/.github/workflows/srtool.yml @@ -27,7 +27,6 @@ jobs: uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 with: fetch-depth: 0 - path: sdk - name: Install tooling run: | @@ -44,7 +43,7 @@ jobs: echo "Github workspace: ${{ github.workspace }}" echo "Current folder: $(pwd)"; ls -al - cd sdk; ls -al + ls -al MATRIX=$(find_runtimes | tee runtimes_list.json) echo $MATRIX @@ -52,7 +51,7 @@ jobs: - name: Get runtimes list id: get_runtimes_list run: | - cd sdk; ls -al + ls -al MATRIX=$(cat runtimes_list.json) echo $MATRIX echo "runtime=$MATRIX" >> $GITHUB_OUTPUT @@ -89,13 +88,11 @@ jobs: - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 with: fetch-depth: 0 - path: sdk - name: Srtool build id: srtool_build uses: chevdor/srtool-actions@v0.9.1 with: - workdir: sdk chain: ${{ matrix.chain }} runtime_dir: ${{ matrix.runtime_dir }} From 437417167851a7d3c6ec5bf2e54ca0e90e45290f Mon Sep 17 00:00:00 2001 From: EgorPopelyaev Date: Tue, 19 Dec 2023 12:14:30 +0100 Subject: [PATCH 3/5] add missing function --- .github/scripts/common/lib.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/scripts/common/lib.sh b/.github/scripts/common/lib.sh index 446423edc55f..bd12d9c6e6ff 100755 --- a/.github/scripts/common/lib.sh +++ b/.github/scripts/common/lib.sh @@ -308,6 +308,10 @@ function increment_rc_tag() { echo $suffix } +function relative_parent() { + echo "$1" | sed -E 's/(.*)\/(.*)\/\.\./\1/g' +} + # Find all the runtimes, it returns the result as JSON object, compatible to be # used as Github Workflow Matrix. This call is exposed by the `scan` command and can be used as: # podman run --rm -it -v /.../fellowship-runtimes:/build docker.io/chevdor/srtool:1.70.0-0.11.1 scan From 635eb9fdd3480d744a05d751f9f5d6c0fe59615a Mon Sep 17 00:00:00 2001 From: EgorPopelyaev Date: Tue, 19 Dec 2023 14:18:03 +0100 Subject: [PATCH 4/5] change release branches trigger --- .github/workflows/srtool.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/srtool.yml b/.github/workflows/srtool.yml index b01a3069fcfe..70e6ea7b5b2c 100644 --- a/.github/workflows/srtool.yml +++ b/.github/workflows/srtool.yml @@ -9,7 +9,9 @@ on: tags: - "*" branches: - - "release*" + - release-v[0-9]+.[0-9]+.[0-9]+* + - release-cumulus-v[0-9]+* + - release-polkadot-v[0-9]+* schedule: - cron: "00 02 * * 1" # 2AM weekly on monday From 33a1d95987e6c71df0cb53e0b3a0f5b4425958f6 Mon Sep 17 00:00:00 2001 From: EgorPopelyaev Date: Tue, 19 Dec 2023 14:25:33 +0100 Subject: [PATCH 5/5] cleanup --- .github/workflows/srtool.yml | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/.github/workflows/srtool.yml b/.github/workflows/srtool.yml index 70e6ea7b5b2c..89659399fc64 100644 --- a/.github/workflows/srtool.yml +++ b/.github/workflows/srtool.yml @@ -65,27 +65,7 @@ jobs: strategy: fail-fast: false matrix: ${{ fromJSON(needs.find-runtimes.outputs.runtime) }} - # strategy: - # matrix: - # include: - # - category: assets - # runtime: asset-hub-westend - # - category: bridge-hubs - # runtime: bridge-hub-polkadot - # - category: bridge-hubs - # runtime: bridge-hub-kusama - # - category: bridge-hubs - # runtime: bridge-hub-rococo - # - category: collectives - # runtime: collectives-polkadot - # - category: contracts - # runtime: contracts-rococo - # - category: starters - # runtime: seedling - # - category: starters - # runtime: shell - # - category: testing - # runtime: rococo-parachain + steps: - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 with: