From 937ef2efd4d319932cfd108e636ba5379a2301c9 Mon Sep 17 00:00:00 2001 From: Arseny Kapoulkine Date: Thu, 29 Sep 2022 15:42:23 -0700 Subject: [PATCH] Cleanup benchmark builds a little bit (#691) We don't need to run any cachegrind benchmarks in benchmark-dev, since benchmark uses our new callgrind setup instead. Also removes prototyping filters that we no longer need from all builds. --- .github/workflows/benchmark-dev.yml | 200 ---------------------------- .github/workflows/benchmark.yml | 1 - .github/workflows/build.yml | 2 - .github/workflows/release.yml | 1 - scripts/run-with-cachegrind.sh | 109 --------------- 5 files changed, 313 deletions(-) delete mode 100644 scripts/run-with-cachegrind.sh diff --git a/.github/workflows/benchmark-dev.yml b/.github/workflows/benchmark-dev.yml index 210a38e5b..d78cb0ba6 100644 --- a/.github/workflows/benchmark-dev.yml +++ b/.github/workflows/benchmark-dev.yml @@ -10,7 +10,6 @@ on: - "papers/**" - "rfcs/**" - "*.md" - - "prototyping/**" jobs: windows: @@ -25,8 +24,6 @@ jobs: script: "run-benchmarks", timeout: 12, title: "Luau Benchmarks", - cachegrindTitle: "Performance", - cachegrindIterCount: 20, } benchResultsRepo: - { name: "luau-lang/benchmark-data", branch: "main" } @@ -117,8 +114,6 @@ jobs: script: "run-benchmarks", timeout: 12, title: "Luau Benchmarks", - cachegrindTitle: "Performance", - cachegrindIterCount: 20, } benchResultsRepo: - { name: "luau-lang/benchmark-data", branch: "main" } @@ -145,19 +140,6 @@ jobs: run: | python bench/bench.py | tee ${{ matrix.bench.script }}-output.txt - - name: Install valgrind - if: matrix.os == 'ubuntu-latest' - run: | - sudo apt-get install valgrind - - - name: Run ${{ matrix.bench.title }} (Cold Cachegrind) - if: matrix.os == 'ubuntu-latest' - run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 | tee -a ${{ matrix.bench.script }}-output.txt - - - name: Run ${{ matrix.bench.title }} (Warm Cachegrind) - if: matrix.os == 'ubuntu-latest' - run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/bench.py "${{ matrix.bench.cachegrindTitle }}" ${{ matrix.bench.cachegrindIterCount }} | tee -a ${{ matrix.bench.script }}-output.txt - - name: Push benchmark results id: pushBenchmarkAttempt1 continue-on-error: true @@ -201,185 +183,3 @@ jobs: bench_tool: "benchmarkluau" bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - - name: Push Cachegrind benchmark results - if: matrix.os == 'ubuntu-latest' - id: pushBenchmarkCachegrindAttempt1 - continue-on-error: true - uses: ./.github/workflows/push-results - with: - repository: ${{ matrix.benchResultsRepo.name }} - branch: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" - bench_name: ${{ matrix.bench.title }} (CacheGrind) - bench_tool: "roblox" - bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" - bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - - name: Push Cachegrind benchmark results (Attempt 2) - if: matrix.os == 'ubuntu-latest' && steps.pushBenchmarkCachegrindAttempt1.outcome == 'failure' - id: pushBenchmarkCachegrindAttempt2 - continue-on-error: true - uses: ./.github/workflows/push-results - with: - repository: ${{ matrix.benchResultsRepo.name }} - branch: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" - bench_name: ${{ matrix.bench.title }} (CacheGrind) - bench_tool: "roblox" - bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" - bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - - name: Push Cachegrind benchmark results (Attempt 3) - if: matrix.os == 'ubuntu-latest' && steps.pushBenchmarkCachegrindAttempt2.outcome == 'failure' - id: pushBenchmarkCachegrindAttempt3 - continue-on-error: true - uses: ./.github/workflows/push-results - with: - repository: ${{ matrix.benchResultsRepo.name }} - branch: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" - bench_name: ${{ matrix.bench.title }} (CacheGrind) - bench_tool: "roblox" - bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" - bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - static-analysis: - name: luau-analyze - strategy: - fail-fast: false - matrix: - os: [ubuntu-latest] - bench: - - { - script: "run-analyze", - timeout: 12, - title: "Luau Analyze", - cachegrindTitle: "Performance", - cachegrindIterCount: 20, - } - benchResultsRepo: - - { name: "luau-lang/benchmark-data", branch: "main" } - - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v3 - with: - token: "${{ secrets.BENCH_GITHUB_TOKEN }}" - - - name: Build Luau - run: make config=release luau luau-analyze - - - uses: actions/setup-python@v4 - with: - python-version: "3.9" - architecture: "x64" - - - name: Install python dependencies - run: | - sudo pip install requests numpy scipy matplotlib ipython jupyter pandas sympy nose - - - name: Install valgrind - run: | - sudo apt-get install valgrind - - - name: Run Luau Analyze on static file - run: sudo python ./bench/measure_time.py ./build/release/luau-analyze bench/other/LuauPolyfillMap.lua | tee ${{ matrix.bench.script }}-output.txt - - - name: Run ${{ matrix.bench.title }} (Cold Cachegrind) - run: sudo ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}Cold" 1 ./build/release/luau-analyze bench/other/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt - - - name: Run ${{ matrix.bench.title }} (Warm Cachegrind) - run: sudo bash ./scripts/run-with-cachegrind.sh python ./bench/measure_time.py "${{ matrix.bench.cachegrindTitle}}" 1 ./build/release/luau-analyze bench/other/LuauPolyfillMap.lua | tee -a ${{ matrix.bench.script }}-output.txt - - - name: Push static analysis results - id: pushStaticAnalysisAttempt1 - continue-on-error: true - uses: ./.github/workflows/push-results - with: - repository: ${{ matrix.benchResultsRepo.name }} - branch: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" - bench_name: ${{ matrix.bench.title }} - bench_tool: "roblox" - bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" - bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - - name: Push static analysis results (Attempt 2) - if: steps.pushStaticAnalysisAttempt1.outcome == 'failure' - id: pushStaticAnalysisAttempt2 - continue-on-error: true - uses: ./.github/workflows/push-results - with: - repository: ${{ matrix.benchResultsRepo.name }} - branch: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" - bench_name: ${{ matrix.bench.title }} - bench_tool: "roblox" - bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" - bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - - name: Push static analysis results (Attempt 3) - if: steps.pushStaticAnalysisAttempt2.outcome == 'failure' - id: pushStaticAnalysisAttempt3 - continue-on-error: true - uses: ./.github/workflows/push-results - with: - repository: ${{ matrix.benchResultsRepo.name }} - branch: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" - bench_name: ${{ matrix.bench.title }} - bench_tool: "roblox" - bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" - bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - - name: Push static analysis Cachegrind results - if: matrix.os == 'ubuntu-latest' - id: pushStaticAnalysisCachegrindAttempt1 - continue-on-error: true - uses: ./.github/workflows/push-results - with: - repository: ${{ matrix.benchResultsRepo.name }} - branch: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" - bench_name: ${{ matrix.bench.title }} - bench_tool: "roblox" - bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" - bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - - name: Push static analysis Cachegrind results (Attempt 2) - if: matrix.os == 'ubuntu-latest' && steps.pushStaticAnalysisCachegrindAttempt1.outcome == 'failure' - id: pushStaticAnalysisCachegrindAttempt2 - continue-on-error: true - uses: ./.github/workflows/push-results - with: - repository: ${{ matrix.benchResultsRepo.name }} - branch: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" - bench_name: ${{ matrix.bench.title }} - bench_tool: "roblox" - bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" - bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" - - - name: Push static analysis Cachegrind results (Attempt 2) - if: matrix.os == 'ubuntu-latest' && steps.pushStaticAnalysisCachegrindAttempt2.outcome == 'failure' - id: pushStaticAnalysisCachegrindAttempt3 - continue-on-error: true - uses: ./.github/workflows/push-results - with: - repository: ${{ matrix.benchResultsRepo.name }} - branch: ${{ matrix.benchResultsRepo.branch }} - token: ${{ secrets.BENCH_GITHUB_TOKEN }} - path: "./gh-pages" - bench_name: ${{ matrix.bench.title }} - bench_tool: "roblox" - bench_output_file_path: "./${{ matrix.bench.script }}-output.txt" - bench_external_data_json_path: "./gh-pages/dev/bench/data-${{ matrix.os }}.json" diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index deb8be865..80a4ba267 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -9,7 +9,6 @@ on: - "papers/**" - "rfcs/**" - "*.md" - - "prototyping/**" jobs: callgrind: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 0958747d5..78c3a34d2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,14 +9,12 @@ on: - 'papers/**' - 'rfcs/**' - '*.md' - - 'prototyping/**' pull_request: paths-ignore: - 'docs/**' - 'papers/**' - 'rfcs/**' - '*.md' - - 'prototyping/**' jobs: unix: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2bf796a2c..e5cd952b1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -9,7 +9,6 @@ on: - 'papers/**' - 'rfcs/**' - '*.md' - - 'prototyping/**' jobs: build: diff --git a/scripts/run-with-cachegrind.sh b/scripts/run-with-cachegrind.sh deleted file mode 100644 index 787043ffa..000000000 --- a/scripts/run-with-cachegrind.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/bin/bash -set -euo pipefail -IFS=$'\n\t' - -declare -A event_map -event_map[Ir]="TotalInstructionsExecuted,executions\n" -event_map[I1mr]="L1_InstrReadCacheMisses,misses/op\n" -event_map[ILmr]="LL_InstrReadCacheMisses,misses/op\n" -event_map[Dr]="TotalMemoryReads,reads\n" -event_map[D1mr]="L1_DataReadCacheMisses,misses/op\n" -event_map[DLmr]="LL_DataReadCacheMisses,misses/op\n" -event_map[Dw]="TotalMemoryWrites,writes\n" -event_map[D1mw]="L1_DataWriteCacheMisses,misses/op\n" -event_map[DLmw]="LL_DataWriteCacheMisses,misses/op\n" -event_map[Bc]="ConditionalBranchesExecuted,executions\n" -event_map[Bcm]="ConditionalBranchMispredictions,mispredictions/op\n" -event_map[Bi]="IndirectBranchesExecuted,executions\n" -event_map[Bim]="IndirectBranchMispredictions,mispredictions/op\n" - -now_ms() { - echo -n $(date +%s%N | cut -b1-13) -} - -# Run cachegrind on a given benchmark and echo the results. -ITERATION_COUNT=$4 -START_TIME=$(now_ms) - -ARGS=( "$@" ) -REST_ARGS="${ARGS[@]:4}" - -valgrind \ - --quiet \ - --tool=cachegrind \ - "$1" "$2" $REST_ARGS>/dev/null - -ARGS=( "$@" ) -REST_ARGS="${ARGS[@]:4}" - - -TIME_ELAPSED=$(bc <<< "$(now_ms) - ${START_TIME}") - -# Generate report using cg_annotate and extract the header and totals of the -# recorded events valgrind was configured to record. -CG_RESULTS=$(cg_annotate $(ls -t cachegrind.out.* | head -1)) -CG_HEADERS=$(grep -B2 'PROGRAM TOTALS$' <<< "$CG_RESULTS" | head -1 | sed -E 's/\s+/\n/g' | sed '/^$/d') -CG_TOTALS=$(grep 'PROGRAM TOTALS$' <<< "$CG_RESULTS" | head -1 | grep -Po '[0-9,]+\s' | tr -d ', ') - -TOTALS_ARRAY=($CG_TOTALS) -HEADERS_ARRAY=($CG_HEADERS) - -declare -A header_map -for i in "${!TOTALS_ARRAY[@]}"; do - header_map[${HEADERS_ARRAY[$i]}]=$i -done - -# Map the results to the format that the benchmark script expects. -for i in "${!TOTALS_ARRAY[@]}"; do - TOTAL=${TOTALS_ARRAY[$i]} - - # Labels and unit descriptions are packed together in the map. - EVENT_TUPLE=${event_map[${HEADERS_ARRAY[$i]}]} - IFS=$',' read -d '\n' -ra EVENT_VALUES < <(printf "%s" "$EVENT_TUPLE") - EVENT_NAME="${EVENT_VALUES[0]}" - UNIT="${EVENT_VALUES[1]}" - - case ${HEADERS_ARRAY[$i]} in - I1mr | ILmr) - REF=${TOTALS_ARRAY[header_map["Ir"]]} - OPS_PER_SEC=$(bc -l <<< "$TOTAL / $REF") - ;; - - D1mr | DLmr) - REF=${TOTALS_ARRAY[header_map["Dr"]]} - OPS_PER_SEC=$(bc -l <<< "$TOTAL / $REF") - ;; - - D1mw | DLmw) - REF=${TOTALS_ARRAY[header_map["Dw"]]} - OPS_PER_SEC=$(bc -l <<< "$TOTAL / $REF") - ;; - - Bcm) - REF=${TOTALS_ARRAY[header_map["Bc"]]} - OPS_PER_SEC=$(bc -l <<< "$TOTAL / $REF") - ;; - - Bim) - REF=${TOTALS_ARRAY[header_map["Bi"]]} - OPS_PER_SEC=$(bc -l <<< "$TOTAL / $REF") - ;; - - *) - OPS_PER_SEC=$(bc -l <<< "$TOTAL") - ;; - esac - - STD_DEV="0%" - RUNS="1" - - if [[ $OPS_PER_SEC =~ ^[+-]?[0-9]*$ ]] - then # $OPS_PER_SEC is integer - printf "%s#%s x %.0f %s ±%s (%d runs sampled)\n" \ - "$3" "$EVENT_NAME" "$OPS_PER_SEC" "$UNIT" "$STD_DEV" "$RUNS" - else # $OPS_PER_SEC is float - printf "%s#%s x %.10f %s ±%s (%d runs sampled)\n" \ - "$3" "$EVENT_NAME" "$OPS_PER_SEC" "$UNIT" "$STD_DEV" "$RUNS" - fi - -done