chore(ci): Extend compiler memory report to external repos (#6768)

Co-authored-by: Tom French <[email protected]> Co-authored-by: TomAFrench <[email protected]>
noir-lang · Dec 11, 2024 · 2a840ca · 2a840ca
1 parent a2d031f
commit 2a840ca
Show file tree

Hide file tree

Showing 3 changed files with 164 additions and 42 deletions.
diff --git a/.github/scripts/merge-bench-reports.sh b/.github/scripts/merge-bench-reports.sh
@@ -0,0 +1,27 @@
+#!/bin/bash
+set -eu
+
+echo "Merging reports"
+
+REPORT_NAME=$1
+NAME_PLURAL=""$REPORT_NAME"s"
+
+combined_reports="[]"
+
+# Iterate over each report and merge them
+for report in ./reports/*; do
+  # The report is saved under ./$REPORT_NAME_{ matrix_report }/$REPORT_NAME_{ matrix_report }.json
+  FILE_PATH=$(echo $(ls $report)) 
+
+  # Extract the $NAME_PLURAL array from each report and merge it
+  combined_reports=$(jq '[."'"$NAME_PLURAL"'"[]] + '"$combined_reports" <<< "$(cat "$report/$FILE_PATH")")
+done
+
+combined_reports=$(jq '[."'$NAME_PLURAL'"[]] + '"$combined_reports" <<< "$(cat ./$REPORT_NAME.json)")
+
+# Wrap the merged memory reports into a new object as to keep the $NAME_PLURAL key
+final_report="{\"$NAME_PLURAL\": $combined_reports}"
+
+echo "$final_report" > $REPORT_NAME.json
+
+cat $REPORT_NAME.json
diff --git a/.github/workflows/reports.yml b/.github/workflows/reports.yml
@@ -217,21 +217,13 @@ jobs:
           ./memory_report.sh
           mv memory_report.json ../memory_report.json
 
-      - name: Parse memory report
-        id: memory_report
-        uses: noir-lang/noir-bench-report@0d7464a8c39170523932d7846b6e6b458a294aea
-        with:
-          report: memory_report.json
-          header: |
-            # Memory Report
-          memory_report: true
-
-      - name: Add memory report to sticky comment
-        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
-        uses: marocchino/sticky-pull-request-comment@v2
+      - name: Upload memory report
+        uses: actions/upload-artifact@v4
         with:
-          header: memory
-          message: ${{ steps.memory_report.outputs.markdown }}
+          name: in_progress_memory_report
+          path: memory_report.json
+          retention-days: 3
+          overwrite: true
 
   generate_compilation_report:
     name: Compilation time
@@ -283,10 +275,11 @@ jobs:
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail }
           - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset }
           # TODO: Bring these back once they no longer time out
-          #- project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private }
-          #- project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public }
+          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-private }
+          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/rollup-base-public }
 
     name: External repo compilation report - ${{ matrix.project.repo }}/${{ matrix.project.path }}
     steps:
@@ -367,26 +360,8 @@ jobs:
 
       - name: Merge compilation reports using jq
         run: |
-          echo "Merging reports" 
-
-          combined_reports="[]"
-
-          # Iterate over each report and merge them
-          for report in ./reports/*; do
-            # The report is saved under ./compilation_report_{ matrix_report }/compilation_report_{ matrix_report }.json
-            REPORT_NAME=$(echo $(ls $report)) 
-            # Extract the 'compilation_reports' array from each report and merge it
-            combined_reports=$(jq '[.compilation_reports[]] + '"$combined_reports" <<< "$(cat "$report/$REPORT_NAME")")
-          done
-
-          combined_reports=$(jq '[.compilation_reports[]] + '"$combined_reports" <<< "$(cat ./compilation_report.json)")
-
-          # Wrap the merged compilation reports into a new object as to keep the 'compilation_reports' key
-          final_report="{\"compilation_reports\": $combined_reports}"
-
-          echo "$final_report" > compilation_report.json
-
-          cat compilation_report.json
+          mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh
+          ./merge-bench-reports.sh compilation_report
 
       - name: Parse compilation report
         id: compilation_report
@@ -403,3 +378,116 @@ jobs:
         with:
           header: compilation
           message: ${{ steps.compilation_report.outputs.markdown }}
+
+  external_repo_memory_report:
+    needs: [build-nargo]
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # TODO: Bring this report back under a flag. The `noir-contracts` report takes just under 30 minutes.
+          # - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-contracts }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/parity-root }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-inner }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-reset }
+          - project: { repo: AztecProtocol/aztec-packages, path: noir-projects/noir-protocol-circuits/crates/private-kernel-tail }
+
+    name: External repo memory report - ${{ matrix.project.repo }}/${{ matrix.project.path }}
+    steps:
+      - name: Download nargo binary
+        uses: actions/download-artifact@v4
+        with:
+          name: nargo
+          path: ./nargo
+
+      - name: Set nargo on PATH
+        run: |
+          nargo_binary="${{ github.workspace }}/nargo/nargo"
+          chmod +x $nargo_binary
+          echo "$(dirname $nargo_binary)" >> $GITHUB_PATH
+          export PATH="$PATH:$(dirname $nargo_binary)"
+          nargo -V
+
+      - uses: actions/checkout@v4
+        with:
+          path: scripts
+          sparse-checkout: |
+            test_programs/memory_report.sh
+          sparse-checkout-cone-mode: false
+
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ matrix.project.repo }}
+          path: test-repo
+          ref: ${{ matrix.project.ref }}
+
+      - name: Generate compilation report
+        working-directory: ./test-repo/${{ matrix.project.path }}
+        run: |
+          mv /home/runner/work/noir/noir/scripts/test_programs/memory_report.sh ./memory_report.sh
+          chmod +x ./memory_report.sh
+          ./memory_report.sh 1
+
+      - name: Move compilation report
+        id: report
+        shell: bash
+        run: |
+          PACKAGE_NAME=${{ matrix.project.path }}
+          PACKAGE_NAME=$(basename $PACKAGE_NAME)
+          mv ./test-repo/${{ matrix.project.path }}/memory_report.json ./memory_report_$PACKAGE_NAME.json
+          echo "memory_report_name=$PACKAGE_NAME" >> $GITHUB_OUTPUT
+
+      - name: Upload memory report
+        uses: actions/upload-artifact@v4
+        with:
+          name: memory_report_${{ steps.report.outputs.memory_report_name }}
+          path: memory_report_${{ steps.report.outputs.memory_report_name }}.json
+          retention-days: 3
+          overwrite: true
+
+  upload_memory_report:
+    name: Upload memory report 
+    needs: [generate_memory_report, external_repo_memory_report]
+    # We want this job to run even if one variation of the matrix in `external_repo_memory_report` fails
+    if: always() 
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: write
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download initial memory report
+        uses: actions/download-artifact@v4
+        with:
+          name: in_progress_memory_report
+
+      - name: Download matrix memory reports
+        uses: actions/download-artifact@v4
+        with:
+          pattern: memory_report_*
+          path: ./reports
+
+      - name: Merge memory reports using jq
+        run: |
+          mv ./.github/scripts/merge-bench-reports.sh merge-bench-reports.sh
+          ./merge-bench-reports.sh memory_report
+
+      - name: Parse memory report
+        id: memory_report
+        uses: noir-lang/noir-bench-report@0d7464a8c39170523932d7846b6e6b458a294aea
+        with:
+          report: memory_report.json
+          header: |
+            # Memory Report
+          memory_report: true
+
+      - name: Add memory report to sticky comment
+        if: github.event_name == 'pull_request' || github.event_name == 'pull_request_target'
+        uses: marocchino/sticky-pull-request-comment@v2
+        with:
+          header: memory
+          message: ${{ steps.memory_report.outputs.markdown }}
diff --git a/test_programs/memory_report.sh b/test_programs/memory_report.sh
@@ -5,29 +5,36 @@ sudo apt-get install heaptrack
 
 NARGO="nargo"
 
-
 # Tests to be profiled for memory report
 tests_to_profile=("keccak256" "workspace" "regression_4709" "ram_blowup_regression")
 
 current_dir=$(pwd)
-execution_success_path="$current_dir/execution_success"
-test_dirs=$(ls $execution_success_path)
+base_path="$current_dir/execution_success"
+
+# If there is an argument that means we want to generate a report for only the current directory
+if [ "$#" -ne 0 ]; then
+  base_path="$current_dir"
+  tests_to_profile=(".")
+fi
 
 FIRST="1"
 
 echo "{\"memory_reports\": [ " > memory_report.json
 
-
 for test_name in ${tests_to_profile[@]}; do    
-        full_path=$execution_success_path"/"$test_name
-        cd $full_path
+        cd $base_path/$test_name
 
         if [ $FIRST = "1" ]
         then
             FIRST="0"
         else
             echo " ," >> $current_dir"/memory_report.json"
         fi
+
+        if [ "$#" -ne 0 ]; then
+            test_name=$(basename $current_dir)
+        fi
+
         heaptrack --output $current_dir/$test_name"_heap" $NARGO compile --force
         if test -f $current_dir/$test_name"_heap.gz"; 
         then