-
Notifications
You must be signed in to change notification settings - Fork 257
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
chore(ci): Run benchmarks on Earthly (#6089)
Migrates benchmark jobs from CCI to Earthly+GA. Introduces a new `yarn-project/scripts/Earthfile` with jobs for downloading logs, aggregating them, and commenting, which are called from a new `bench-summary` job in GA's CI workflow that runs after `e2e`. Adds a new `UPLOAD_LOGS` function in the `yarn-projects/end-to-end/Earthfile` to upload logs to S3, gets called from all bench e2e jobs. Perhaps we can ditch S3 for storing logs (not historic benches, since those are needed for comparisons) and use Earthly artifacts for that, but we can do that later down the road.
- Loading branch information
1 parent
7933f0f
commit c985c73
Showing
14 changed files
with
340 additions
and
180 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,3 +18,7 @@ cmake-build-debug | |
| .DS_Store | ||
|
|
||
| **/*.dockerignore | ||
|
|
||
| # Earthly | ||
| .arg | ||
| .secret | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file was deleted.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| #!/usr/bin/env bash | ||
| # Checks that all logs needed for assembling aggregate benchmarks have been retrieved. | ||
|
|
||
| [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace | ||
| set -eu | ||
|
|
||
| LOG_FOLDER="${LOG_FOLDER:-log}" | ||
| E2E_SRC_FOLDER=/usr/src/yarn-project/end-to-end/src | ||
|
|
||
| echo "Checking log files in $LOG_FOLDER" | ||
|
|
||
| # Only generate the aggregated benchmark if we've managed to retrieve all the needed log files | ||
| # If these runs were skipped due to no changes in their rebuild-patterns, then there's no need | ||
| # to recompute the aggregated benchmark. Note that if one benchmark did run but others didn't, | ||
| # this skips the whole aggregation. For now, that's fine because all benchmark files have the | ||
| # same rebuild pattern rules. But if that changes, then we'd need to go up in the commit history | ||
| # to find the latest log files for the unchanged benchmarks. | ||
| EXPECTED_LOGS_COUNT=$(find $E2E_SRC_FOLDER -type f -name "bench*.test.ts" | wc -l) | ||
| DOWNLOADED_LOGS_COUNT=$(find $LOG_FOLDER -type f -name "*.jsonl" | wc -l) | ||
| if [ "$DOWNLOADED_LOGS_COUNT" -lt "$EXPECTED_LOGS_COUNT" ]; then | ||
| echo Found only $DOWNLOADED_LOGS_COUNT out of $EXPECTED_LOGS_COUNT benchmark log files in S3. | ||
| echo Files found: $(find $LOG_FOLDER -type f -name "*.jsonl") | ||
| exit 1 | ||
| fi | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,33 @@ | ||
| #!/usr/bin/env bash | ||
| # Downloads base benchmarks from S3 to compare with the current benchmarks via bench-comment | ||
|
|
||
| [ -n "${BUILD_SYSTEM_DEBUG:-}" ] && set -x # conditionally trace | ||
| set -eu | ||
|
|
||
| BUCKET_NAME="aztec-ci-artifacts" | ||
| BENCH_FOLDER="${BENCH_FOLDER:-bench}" | ||
| COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" | ||
| BASE_BENCHMARK_FILE_JSON="${BENCH_FOLDER}/base-benchmark.json" | ||
|
|
||
| # If on a pull request, get the data from the most recent commit on master where it's available to generate a comment comparing them | ||
| if [ -n "${PULL_REQUEST:-}" ]; then | ||
| MASTER_COMMIT_HASH=$(curl -s "https://api.github.com/repos/AztecProtocol/aztec-packages/pulls/${PULL_REQUEST##*/}" | jq -r '.base.sha') | ||
| MASTER_COMMIT_HASHES=($(git log $MASTER_COMMIT_HASH --format="%H" -n 50)) | ||
|
|
||
| mkdir -p $BENCH_FOLDER | ||
|
|
||
| set +e | ||
| echo "Searching for base benchmark data starting from commit $MASTER_COMMIT_HASH" | ||
| for commit_hash in "${MASTER_COMMIT_HASHES[@]}"; do | ||
| aws s3 cp "s3://${BUCKET_NAME}/benchmarks-v1/master/$commit_hash.json" $BASE_BENCHMARK_FILE_JSON | ||
| if [ $? -eq 0 ]; then | ||
| echo "Downloaded base data from commit $commit_hash" | ||
| exit 0 | ||
| fi | ||
| done | ||
| set -e | ||
|
|
||
| echo "No base commit data found" | ||
| else | ||
| echo "Not on a pull request, skipping download of base benchmark data" | ||
| fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| #!/usr/bin/env bash | ||
| # Downloads the log files uploaded in upload_logs_to_s3 | ||
|
|
||
| set -eu | ||
|
|
||
| BUCKET_NAME="aztec-ci-artifacts" | ||
| LOG_FOLDER="${LOG_FOLDER:-log}" | ||
| COMMIT_HASH="${COMMIT_HASH:-$(git rev-parse HEAD)}" | ||
|
|
||
| echo "Downloading logs from S3 for commit $COMMIT_HASH in branch ${BRANCH:-} at pull request ${PULL_REQUEST:-none}" | ||
|
|
||
| # Paths from upload_logs_to_s3 | ||
| if [ "${BRANCH:-}" = "master" ]; then | ||
| LOG_SOURCE_FOLDER="logs-v1/master/$COMMIT_HASH" | ||
| BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/master/$COMMIT_HASH" | ||
| BENCHMARK_TARGET_FILE="benchmarks-v1/master/$COMMIT_HASH.json" | ||
| BENCHMARK_LATEST_FILE="benchmarks-v1/latest.json" | ||
| elif [ -n "${PULL_REQUEST:-}" ]; then | ||
| LOG_SOURCE_FOLDER="logs-v1/pulls/${PULL_REQUEST##*/}" | ||
| BARRETENBERG_BENCH_SOURCE_FOLDER="barretenberg-bench-v1/pulls/${PULL_REQUEST##*/}" | ||
| BENCHMARK_TARGET_FILE="benchmarks-v1/pulls/${PULL_REQUEST##*/}.json" | ||
| else | ||
| echo "Skipping benchmark run on branch ${BRANCH:-unknown}." | ||
| exit 0 | ||
| fi | ||
|
|
||
| mkdir -p $LOG_FOLDER | ||
|
|
||
| # Download benchmark log files from S3 LOG_SOURCE_FOLDER into local LOG_FOLDER | ||
| echo "Downloading benchmark log files from $BUCKET_NAME/$LOG_SOURCE_FOLDER to $LOG_FOLDER" | ||
| aws s3 cp "s3://${BUCKET_NAME}/${LOG_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include 'bench*.jsonl' --recursive | ||
|
|
||
| # Download barretenberg log files, these are direct benchmarks and separate from the above | ||
| aws s3 cp "s3://${BUCKET_NAME}/${BARRETENBERG_BENCH_SOURCE_FOLDER}/" $LOG_FOLDER --exclude '*' --include '*_bench.json' --recursive | ||
|
|
||
| echo "Downloaded log files $(ls $LOG_FOLDER)" |
Oops, something went wrong.