diff --git a/.github/scripts/common/lib.sh b/.github/scripts/common/lib.sh index e499dfabd643..bd12d9c6e6ff 100755 --- a/.github/scripts/common/lib.sh +++ b/.github/scripts/common/lib.sh @@ -307,3 +307,40 @@ function increment_rc_tag() { ((suffix++)) echo $suffix } + +function relative_parent() { + echo "$1" | sed -E 's/(.*)\/(.*)\/\.\./\1/g' +} + +# Find all the runtimes, it returns the result as JSON object, compatible to be +# used as Github Workflow Matrix. This call is exposed by the `scan` command and can be used as: +# podman run --rm -it -v /.../fellowship-runtimes:/build docker.io/chevdor/srtool:1.70.0-0.11.1 scan +function find_runtimes() { + libs=($(git grep -I -r --cached --max-depth 20 --files-with-matches 'construct_runtime!' -- '*lib.rs')) + re=".*-runtime$" + JSON=$(jq --null-input '{ "include": [] }') + + # EXCLUDED_RUNTIMES is a space separated list of runtime names (without the -runtime postfix) + # EXCLUDED_RUNTIMES=${EXCLUDED_RUNTIMES:-"substrate-test"} + IFS=' ' read -r -a exclusions <<< "$EXCLUDED_RUNTIMES" + + for lib in "${libs[@]}"; do + crate_dir=$(dirname "$lib") + cargo_toml="$crate_dir/../Cargo.toml" + + name=$(toml get -r $cargo_toml 'package.name') + chain=${name//-runtime/} + + if [[ "$name" =~ $re ]] && ! [[ ${exclusions[@]} =~ $chain ]]; then + lib_dir=$(dirname "$lib") + runtime_dir=$(relative_parent "$lib_dir/..") + ITEM=$(jq --null-input \ + --arg chain "$chain" \ + --arg name "$name" \ + --arg runtime_dir "$runtime_dir" \ + '{ "chain": $chain, "crate": $name, "runtime_dir": $runtime_dir }') + JSON=$(echo $JSON | jq ".include += [$ITEM]") + fi + done + echo $JSON +} diff --git a/.github/workflows/srtool.yml b/.github/workflows/srtool.yml new file mode 100644 index 000000000000..89659399fc64 --- /dev/null +++ b/.github/workflows/srtool.yml @@ -0,0 +1,135 @@ +name: Srtool build + +env: + SUBWASM_VERSION: 0.20.0 + TOML_CLI_VERSION: 0.2.4 + +on: + push: + tags: + - "*" + branches: + - release-v[0-9]+.[0-9]+.[0-9]+* + - release-cumulus-v[0-9]+* + - release-polkadot-v[0-9]+* + + schedule: + - cron: "00 02 * * 1" # 2AM weekly on monday + + workflow_dispatch: + +jobs: + find-runtimes: + name: Scan repo paritytech/polkadot-sdk + outputs: + runtime: ${{ steps.get_runtimes_list.outputs.runtime }} + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + with: + fetch-depth: 0 + + - name: Install tooling + run: | + URL=https://github.com/chevdor/toml-cli/releases/download/v${{ env.TOML_CLI_VERSION }}/toml_linux_amd64_v${{ env.TOML_CLI_VERSION }}.deb + curl -L $URL --output toml.deb + sudo dpkg -i toml.deb + toml --version; jq --version + + - name: Scan runtimes + env: + EXCLUDED_RUNTIMES: "substrate-test" + run: | + . ./.github/scripts/common/lib.sh + + echo "Github workspace: ${{ github.workspace }}" + echo "Current folder: $(pwd)"; ls -al + ls -al + + MATRIX=$(find_runtimes | tee runtimes_list.json) + echo $MATRIX + + - name: Get runtimes list + id: get_runtimes_list + run: | + ls -al + MATRIX=$(cat runtimes_list.json) + echo $MATRIX + echo "runtime=$MATRIX" >> $GITHUB_OUTPUT + + srtool: + runs-on: ubuntu-latest + needs: + - find-runtimes + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.find-runtimes.outputs.runtime) }} + + steps: + - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4.0.0 + with: + fetch-depth: 0 + + - name: Srtool build + id: srtool_build + uses: chevdor/srtool-actions@v0.9.1 + with: + chain: ${{ matrix.chain }} + runtime_dir: ${{ matrix.runtime_dir }} + + - name: Summary + run: | + echo '${{ steps.srtool_build.outputs.json }}' | jq > ${{ matrix.chain }}-srtool-digest.json + cat ${{ matrix.chain }}-srtool-digest.json + echo "Compact Runtime: ${{ steps.srtool_build.outputs.wasm }}" + echo "Compressed Runtime: ${{ steps.srtool_build.outputs.wasm_compressed }}" + + # it takes a while to build the runtime, so let's save the artifact as soon as we have it + - name: Archive Artifacts for ${{ matrix.chain }} + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{ matrix.chain }}-runtime + path: | + ${{ steps.srtool_build.outputs.wasm }} + ${{ steps.srtool_build.outputs.wasm_compressed }} + ${{ matrix.chain }}-srtool-digest.json + + # We now get extra information thanks to subwasm + - name: Install subwasm + run: | + wget https://github.com/chevdor/subwasm/releases/download/v${{ env.SUBWASM_VERSION }}/subwasm_linux_amd64_v${{ env.SUBWASM_VERSION }}.deb + sudo dpkg -i subwasm_linux_amd64_v${{ env.SUBWASM_VERSION }}.deb + subwasm --version + + - name: Show Runtime information + shell: bash + run: | + subwasm info ${{ steps.srtool_build.outputs.wasm }} + subwasm info ${{ steps.srtool_build.outputs.wasm_compressed }} + subwasm --json info ${{ steps.srtool_build.outputs.wasm }} > ${{ matrix.chain }}-info.json + subwasm --json info ${{ steps.srtool_build.outputs.wasm_compressed }} > ${{ matrix.chain }}-compressed-info.json + + - name: Extract the metadata + shell: bash + run: | + subwasm meta ${{ steps.srtool_build.outputs.wasm }} + subwasm --json meta ${{ steps.srtool_build.outputs.wasm }} > ${{ matrix.chain }}-metadata.json + + - name: Check the metadata diff + shell: bash + # the following subwasm call will error for chains that are not known and/or live, that includes shell for instance + run: | + subwasm diff ${{ steps.srtool_build.outputs.wasm }} --chain-b ${{ matrix.chain }} || \ + echo "Subwasm call failed, check the logs. This is likely because ${{ matrix.chain }} is not known by subwasm" | \ + tee ${{ matrix.chain }}-diff.txt + + - name: Archive Subwasm results + uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2 + with: + name: ${{ matrix.chain }}-runtime + path: | + ${{ matrix.chain }}-info.json + ${{ matrix.chain }}-compressed-info.json + ${{ matrix.chain }}-metadata.json + ${{ matrix.chain }}-diff.txt