[CI] Add weights verification jobs (#6996)

* add weights verification job * switch to a github action * rename script.. * add swc check * fix perms... * debugging * fix comments, remove artifact. I hate CI * switch to swc compare files * sigh * switch back to compare commits * fix output * fix output... again. yay markdown! * remove test version number * remove TODO * switch to docker image * Revert "remove TODO" This reverts commit c313afd. * fix docker registry url * revert docker experiment too janky. will fix upstream stuff and fix as a separate PR reverts commits: - 3be2043 - fd17a0f - 59f0ebf --------- Co-authored-by: parity-processbot <> Co-authored-by: Martin <[email protected]>
paritytech · Apr 13, 2023 · e164da6 · e164da6
1 parent 0960b14
commit e164da6
Show file tree

Hide file tree

Showing 3 changed files with 125 additions and 0 deletions.
diff --git a/.github/workflows/check-weights.yml b/.github/workflows/check-weights.yml
@@ -0,0 +1,49 @@
+name: Check updated weights
+
+on:
+  pull_request:
+    paths:
+      - 'runtime/*/src/weights/**'
+
+jobs:
+  check_weights_files:
+    strategy:
+      fail-fast: false
+      matrix:
+        runtime: [westend, kusama, polkadot, rococo]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+      - name: Check weights files
+        shell: bash
+        run: |
+          scripts/ci/github/verify_updated_weights.sh ${{ matrix.runtime }}
+
+  # This job uses https://github.com/ggwpez/substrate-weight-compare to compare the weights of the current
+  # release with the last release, then adds them as a comment to the PR.
+  check_weight_changes:
+    strategy:
+      fail-fast: false
+      matrix:
+        runtime: [westend, kusama, polkadot, rococo]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get latest release
+        run: |
+          LAST_RELEASE=$(curl -s https://api.github.com/repos/paritytech/polkadot/releases/latest | jq -r .tag_name)
+          echo "LAST_RELEASE=$LAST_RELEASE" >> $GITHUB_ENV
+      - name: Checkout current sources
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Check weight changes
+        shell: bash
+        run: |
+          cargo install --git https://github.com/ggwpez/substrate-weight-compare swc
+          ./scripts/ci/github/check_weights_swc.sh ${{ matrix.runtime }} "$LAST_RELEASE" | tee swc_output_${{ matrix.runtime }}.md
+      - name: Add comment
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          filePath: ./swc_output_${{ matrix.runtime }}.md
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
diff --git a/scripts/ci/github/check_weights_swc.sh b/scripts/ci/github/check_weights_swc.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# Need to set globstar for ** magic
+shopt -s globstar
+
+RUNTIME=$1
+VERSION=$2
+echo "<details>"
+echo "<summary>Weight changes for $RUNTIME</summary>"
+echo
+swc compare commits \
+    --method asymptotic \
+    --offline \
+    --path-pattern "./runtime/$RUNTIME/src/weights/**/*.rs" \
+    --no-color \
+    --format markdown \
+    --strip-path-prefix "runtime/$RUNTIME/src/weights/" \
+    "$VERSION"
+    #--ignore-errors
+echo
+echo "</details>"
diff --git a/scripts/ci/github/verify_updated_weights.sh b/scripts/ci/github/verify_updated_weights.sh
@@ -0,0 +1,56 @@
+#!/bin/bash
+
+ROOT="$(dirname "$0")/../../.."
+RUNTIME="$1"
+
+# If we're on a mac, use gdate for date command (requires coreutils installed via brew)
+if [[ "$OSTYPE" == "darwin"* ]]; then
+  DATE="gdate"
+else
+  DATE="date"
+fi
+
+function check_date() {
+  # Get the dates as input arguments
+  LAST_RUN="$1"
+  TODAY="$($DATE +%Y-%m-%d)"
+  # Calculate the date two days before today
+  CUTOFF=$($DATE -d "$TODAY - 2 days" +%Y-%m-%d)
+
+  if [[ "$LAST_RUN" > "$CUTOFF" ]]; then
+    return 0
+  else
+    return 1
+  fi
+}
+
+check_weights(){
+    FILE=$1
+    CUR_DATE=$2
+    DATE_REGEX='[0-9]{4}-[0-9]{2}-[0-9]{2}'
+    LAST_UPDATE="$(grep -E "//! DATE: $DATE_REGEX" "$FILE" | sed -r "s/.*DATE: ($DATE_REGEX).*/\1/")"
+    # If the file does not contain a date, flag it as an error.
+    if [ -z "$LAST_UPDATE" ]; then
+        echo "Skipping $FILE, no date found."
+        return 0
+    fi
+    if ! check_date "$LAST_UPDATE" ; then
+        echo "ERROR: $FILE was not updated for the current date. Last update: $LAST_UPDATE"
+        return 1
+    fi
+    # echo "OK: $FILE"
+}
+
+echo "Checking weights for $RUNTIME"
+CUR_DATE="$(date +%Y-%m-%d)"
+HAS_ERROR=0
+for FILE in "$ROOT"/runtime/"$RUNTIME"/src/weights/*.rs; do
+    if ! check_weights "$FILE" "$CUR_DATE"; then
+        HAS_ERROR=1
+    fi
+done
+
+if [ $HAS_ERROR -eq 1 ]; then
+    echo "ERROR: One or more weights files were not updated during the last benchmark run. Check the logs above."
+    exit 1
+fi