feat: Add OpenLineage support for CloudSQLExecuteQueryOperator and some SQLtoGCS transfer operators #116457
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# Licensed to the Apache Software Foundation (ASF) under one | |
# or more contributor license agreements. See the NOTICE file | |
# distributed with this work for additional information | |
# regarding copyright ownership. The ASF licenses this file | |
# to you under the Apache License, Version 2.0 (the | |
# "License"); you may not use this file except in compliance | |
# with the License. You may obtain a copy of the License at | |
# | |
# http://www.apache.org/licenses/LICENSE-2.0 | |
# | |
# Unless required by applicable law or agreed to in writing, | |
# software distributed under the License is distributed on an | |
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
# KIND, either express or implied. See the License for the | |
# specific language governing permissions and limitations | |
# under the License. | |
# | |
--- | |
name: Tests | |
on: # yamllint disable-line rule:truthy | |
schedule: | |
- cron: '28 1,7,13,19 * * *' | |
push: | |
branches: | |
- v[0-9]+-[0-9]+-test | |
- providers-[a-z]+-?[a-z]*/v[0-9]+-[0-9]+ | |
pull_request: | |
branches: | |
- main | |
- v[0-9]+-[0-9]+-test | |
- v[0-9]+-[0-9]+-stable | |
- providers-[a-z]+-?[a-z]*/v[0-9]+-[0-9]+ | |
workflow_dispatch: | |
permissions: | |
# All other permissions are set to none | |
contents: read | |
# Technically read access while waiting for images should be more than enough. However, | |
# there is a bug in GitHub Actions/Packages and in case private repositories are used, you get a permission | |
# denied error when attempting to just pull private image, changing the token permission to write solves the | |
# issue. This is not dangerous, because if it is for "apache/airflow", only maintainers can push ci.yml | |
# changes. If it is for a fork, then the token is read-only anyway. | |
packages: write | |
env: | |
GITHUB_REPOSITORY: ${{ github.repository }} | |
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
GITHUB_USERNAME: ${{ github.actor }} | |
IMAGE_TAG: "${{ github.event.pull_request.head.sha || github.sha }}" | |
SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} | |
VERBOSE: "true" | |
concurrency: | |
group: ci-${{ github.event.pull_request.number || github.ref }} | |
cancel-in-progress: true | |
jobs: | |
build-info: | |
name: "Build info" | |
# At build-info stage we do not yet have outputs so we need to hard-code the runs-on to public runners | |
runs-on: ["ubuntu-22.04"] | |
env: | |
GITHUB_CONTEXT: ${{ toJson(github) }} | |
outputs: | |
all-python-versions-list-as-string: >- | |
${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} | |
basic-checks-only: ${{ steps.selective-checks.outputs.basic-checks-only }} | |
build-job-description: ${{ steps.source-run-info.outputs.build-job-description }} | |
canary-run: ${{ steps.source-run-info.outputs.canary-run }} | |
chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} | |
ci-image-build: ${{ steps.selective-checks.outputs.ci-image-build }} | |
core-test-types-list-as-string: >- | |
${{ steps.selective-checks.outputs.core-test-types-list-as-string }} | |
debug-resources: ${{ steps.selective-checks.outputs.debug-resources }} | |
default-branch: ${{ steps.selective-checks.outputs.default-branch }} | |
default-constraints-branch: ${{ steps.selective-checks.outputs.default-constraints-branch }} | |
default-helm-version: ${{ steps.selective-checks.outputs.default-helm-version }} | |
default-kind-version: ${{ steps.selective-checks.outputs.default-kind-version }} | |
default-kubernetes-version: ${{ steps.selective-checks.outputs.default-kubernetes-version }} | |
default-mysql-version: ${{ steps.selective-checks.outputs.default-mysql-version }} | |
default-postgres-version: ${{ steps.selective-checks.outputs.default-postgres-version }} | |
default-python-version: ${{ steps.selective-checks.outputs.default-python-version }} | |
disable-airflow-repo-cache: ${{ steps.selective-checks.outputs.disable-airflow-repo-cache }} | |
docker-cache: ${{ steps.selective-checks.outputs.docker-cache }} | |
docs-build: ${{ steps.selective-checks.outputs.docs-build }} | |
docs-list-as-string: ${{ steps.selective-checks.outputs.docs-list-as-string }} | |
excluded-providers-as-string: ${{ steps.selective-checks.outputs.excluded-providers-as-string }} | |
force-pip: ${{ steps.selective-checks.outputs.force-pip }} | |
full-tests-needed: ${{ steps.selective-checks.outputs.full-tests-needed }} | |
has-migrations: ${{ steps.selective-checks.outputs.has-migrations }} | |
helm-test-packages: ${{ steps.selective-checks.outputs.helm-test-packages }} | |
image-tag: ${{ github.event.pull_request.head.sha || github.sha }} | |
in-workflow-build: ${{ steps.source-run-info.outputs.in-workflow-build }} | |
include-success-outputs: ${{ steps.selective-checks.outputs.include-success-outputs }} | |
individual-providers-test-types-list-as-string: >- | |
${{ steps.selective-checks.outputs.individual-providers-test-types-list-as-string }} | |
is-airflow-runner: ${{ steps.selective-checks.outputs.is-airflow-runner }} | |
is-amd-runner: ${{ steps.selective-checks.outputs.is-amd-runner }} | |
is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} | |
is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} | |
is-self-hosted-runner: ${{ steps.selective-checks.outputs.is-self-hosted-runner }} | |
is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} | |
kubernetes-combos-list-as-string: >- | |
${{ steps.selective-checks.outputs.kubernetes-combos-list-as-string }} | |
kubernetes-versions-list-as-string: >- | |
${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} | |
latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} | |
mypy-checks: ${{ steps.selective-checks.outputs.mypy-checks }} | |
mysql-exclude: ${{ steps.selective-checks.outputs.mysql-exclude }} | |
mysql-versions: ${{ steps.selective-checks.outputs.mysql-versions }} | |
needs-api-codegen: ${{ steps.selective-checks.outputs.needs-api-codegen }} | |
needs-api-tests: ${{ steps.selective-checks.outputs.needs-api-tests }} | |
needs-helm-tests: ${{ steps.selective-checks.outputs.needs-helm-tests }} | |
needs-mypy: ${{ steps.selective-checks.outputs.needs-mypy }} | |
only-new-ui-files: ${{ steps.selective-checks.outputs.only-new-ui-files }} | |
postgres-exclude: ${{ steps.selective-checks.outputs.postgres-exclude }} | |
postgres-versions: ${{ steps.selective-checks.outputs.postgres-versions }} | |
prod-image-build: ${{ steps.selective-checks.outputs.prod-image-build }} | |
# yamllint disable rule:line-length | |
providers-compatibility-tests-matrix: ${{ steps.selective-checks.outputs.providers-compatibility-tests-matrix }} | |
providers-test-types-list-as-string: >- | |
${{ steps.selective-checks.outputs.providers-test-types-list-as-string }} | |
pull-request-labels: ${{ steps.source-run-info.outputs.pr-labels }} | |
python-versions-list-as-string: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} | |
python-versions: ${{ steps.selective-checks.outputs.python-versions }} | |
run-amazon-tests: ${{ steps.selective-checks.outputs.run-amazon-tests }} | |
run-coverage: ${{ steps.source-run-info.outputs.run-coverage }} | |
run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} | |
run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }} | |
run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }} | |
run-tests: ${{ steps.selective-checks.outputs.run-tests }} | |
run-ui-tests: ${{ steps.selective-checks.outputs.run-ui-tests }} | |
run-www-tests: ${{ steps.selective-checks.outputs.run-www-tests }} | |
runs-on-as-json-default: ${{ steps.selective-checks.outputs.runs-on-as-json-default }} | |
runs-on-as-json-docs-build: ${{ steps.selective-checks.outputs.runs-on-as-json-docs-build }} | |
runs-on-as-json-public: ${{ steps.selective-checks.outputs.runs-on-as-json-public }} | |
runs-on-as-json-self-hosted-asf: ${{ steps.selective-checks.outputs.runs-on-as-json-self-hosted-asf }} | |
runs-on-as-json-self-hosted: ${{ steps.selective-checks.outputs.runs-on-as-json-self-hosted }} | |
selected-providers-list-as-string: >- | |
${{ steps.selective-checks.outputs.selected-providers-list-as-string }} | |
skip-pre-commits: ${{ steps.selective-checks.outputs.skip-pre-commits }} | |
skip-providers-tests: ${{ steps.selective-checks.outputs.skip-providers-tests }} | |
source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} | |
sqlite-exclude: ${{ steps.selective-checks.outputs.sqlite-exclude }} | |
test-groups: ${{ steps.selective-checks.outputs.test-groups }} | |
testable-core-integrations: ${{ steps.selective-checks.outputs.testable-core-integrations }} | |
testable-providers-integrations: ${{ steps.selective-checks.outputs.testable-providers-integrations }} | |
upgrade-to-newer-dependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} | |
steps: | |
- name: "Cleanup repo" | |
shell: bash | |
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
uses: actions/checkout@v4 | |
with: | |
persist-credentials: false | |
- name: "Cleanup docker" | |
run: ./scripts/ci/cleanup_docker.sh | |
- name: Fetch incoming commit ${{ github.sha }} with its parent | |
uses: actions/checkout@v4 | |
with: | |
ref: ${{ github.sha }} | |
fetch-depth: 2 | |
persist-credentials: false | |
- name: "Install Breeze" | |
uses: ./.github/actions/breeze | |
- name: "Get information about the Workflow" | |
id: source-run-info | |
run: breeze ci get-workflow-info 2>> ${GITHUB_OUTPUT} | |
env: | |
SKIP_BREEZE_SELF_UPGRADE_CHECK: "true" | |
- name: Selective checks | |
id: selective-checks | |
env: | |
PR_LABELS: "${{ steps.source-run-info.outputs.pr-labels }}" | |
COMMIT_REF: "${{ github.sha }}" | |
VERBOSE: "false" | |
run: breeze ci selective-check 2>> ${GITHUB_OUTPUT} | |
- name: env | |
run: printenv | |
env: | |
PR_LABELS: ${{ steps.source-run-info.outputs.pr-labels }} | |
GITHUB_CONTEXT: ${{ toJson(github) }} | |
basic-tests: | |
name: "Basic tests" | |
needs: [build-info] | |
uses: ./.github/workflows/basic-tests.yml | |
with: | |
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} | |
run-ui-tests: ${{needs.build-info.outputs.run-ui-tests}} | |
run-www-tests: ${{needs.build-info.outputs.run-www-tests}} | |
needs-api-codegen: ${{needs.build-info.outputs.needs-api-codegen}} | |
default-python-version: ${{needs.build-info.outputs.default-python-version}} | |
basic-checks-only: ${{needs.build-info.outputs.basic-checks-only}} | |
skip-pre-commits: ${{needs.build-info.outputs.skip-pre-commits}} | |
canary-run: ${{needs.build-info.outputs.canary-run}} | |
latest-versions-only: ${{needs.build-info.outputs.latest-versions-only}} | |
build-ci-images: | |
name: > | |
${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' || 'Skip building' }} | |
CI images in-workflow | |
needs: [build-info] | |
uses: ./.github/workflows/ci-image-build.yml | |
permissions: | |
contents: read | |
# This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs | |
# from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. | |
# For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the | |
# "in-workflow-build" condition | |
packages: write | |
secrets: inherit | |
with: | |
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} | |
runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }} | |
do-build: ${{ needs.build-info.outputs.in-workflow-build }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
platform: "linux/amd64" | |
python-versions: ${{ needs.build-info.outputs.python-versions }} | |
branch: ${{ needs.build-info.outputs.default-branch }} | |
use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }} | |
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
docker-cache: ${{ needs.build-info.outputs.docker-cache }} | |
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} | |
wait-for-ci-images: | |
timeout-minutes: 120 | |
name: "Wait for CI images" | |
runs-on: ${{ fromJSON(needs.build-info.outputs.runs-on-as-json-public) }} | |
needs: [build-info, build-ci-images] | |
if: needs.build-info.outputs.ci-image-build == 'true' | |
env: | |
BACKEND: sqlite | |
# Force more parallelism for pull even on public images | |
PARALLELISM: 6 | |
INCLUDE_SUCCESS_OUTPUTS: "${{needs.build-info.outputs.include-success-outputs}}" | |
steps: | |
- name: "Cleanup repo" | |
shell: bash | |
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
if: needs.build-info.outputs.in-workflow-build == 'false' | |
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
uses: actions/checkout@v4 | |
with: | |
persist-credentials: false | |
if: needs.build-info.outputs.in-workflow-build == 'false' | |
- name: "Cleanup docker" | |
run: ./scripts/ci/cleanup_docker.sh | |
if: needs.build-info.outputs.in-workflow-build == 'false' | |
- name: "Install Breeze" | |
uses: ./.github/actions/breeze | |
if: needs.build-info.outputs.in-workflow-build == 'false' | |
- name: Login to ghcr.io | |
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin | |
if: needs.build-info.outputs.in-workflow-build == 'false' | |
- name: Wait for CI images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} | |
id: wait-for-images | |
run: breeze ci-image pull --run-in-parallel --wait-for-image --tag-as-latest | |
env: | |
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} | |
if: needs.build-info.outputs.in-workflow-build == 'false' | |
additional-ci-image-checks: | |
name: "Additional CI image checks" | |
needs: [build-info, wait-for-ci-images] | |
uses: ./.github/workflows/additional-ci-image-checks.yml | |
if: needs.build-info.outputs.canary-run == 'true' | |
with: | |
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} | |
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} | |
runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
python-versions: ${{ needs.build-info.outputs.python-versions }} | |
branch: ${{ needs.build-info.outputs.default-branch }} | |
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} | |
docker-cache: ${{ needs.build-info.outputs.docker-cache }} | |
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} | |
canary-run: ${{ needs.build-info.outputs.canary-run }} | |
latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} | |
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }} | |
generate-constraints: | |
name: "Generate constraints" | |
needs: [build-info, wait-for-ci-images] | |
uses: ./.github/workflows/generate-constraints.yml | |
if: > | |
needs.build-info.outputs.ci-image-build == 'true' && | |
needs.build-info.outputs.only-new-ui-files != 'true' | |
with: | |
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} | |
python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
# generate no providers constraints only in canary builds - they take quite some time to generate | |
# they are not needed for regular builds, they are only needed to update constraints in canaries | |
generate-no-providers-constraints: ${{ needs.build-info.outputs.canary-run }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
ci-image-checks: | |
name: "CI image checks" | |
needs: [build-info, wait-for-ci-images] | |
uses: ./.github/workflows/ci-image-checks.yml | |
secrets: inherit | |
with: | |
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} | |
runs-on-as-json-docs-build: ${{ needs.build-info.outputs.runs-on-as-json-docs-build }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
needs-mypy: ${{ needs.build-info.outputs.needs-mypy }} | |
mypy-checks: ${{ needs.build-info.outputs.mypy-checks }} | |
python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
branch: ${{ needs.build-info.outputs.default-branch }} | |
canary-run: ${{ needs.build-info.outputs.canary-run }} | |
default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
docs-list-as-string: ${{ needs.build-info.outputs.docs-list-as-string }} | |
latest-versions-only: ${{ needs.build-info.outputs.latest-versions-only }} | |
basic-checks-only: ${{ needs.build-info.outputs.basic-checks-only }} | |
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
skip-pre-commits: ${{ needs.build-info.outputs.skip-pre-commits }} | |
chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
ci-image-build: ${{ needs.build-info.outputs.ci-image-build }} | |
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
docs-build: ${{ needs.build-info.outputs.docs-build }} | |
needs-api-codegen: ${{ needs.build-info.outputs.needs-api-codegen }} | |
default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} | |
run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
providers: | |
name: "Provider packages tests" | |
uses: ./.github/workflows/test-provider-packages.yml | |
needs: [build-info, wait-for-ci-images] | |
permissions: | |
contents: read | |
packages: read | |
secrets: inherit | |
if: > | |
needs.build-info.outputs.skip-providers-tests != 'true' && | |
needs.build-info.outputs.latest-versions-only != 'true' | |
with: | |
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
canary-run: ${{ needs.build-info.outputs.canary-run }} | |
default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
selected-providers-list-as-string: ${{ needs.build-info.outputs.selected-providers-list-as-string }} | |
# yamllint disable rule:line-length | |
providers-compatibility-tests-matrix: ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }} | |
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} | |
python-versions: ${{ needs.build-info.outputs.python-versions }} | |
providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }} | |
tests-helm: | |
name: "Helm tests" | |
uses: ./.github/workflows/helm-tests.yml | |
needs: [build-info, wait-for-ci-images] | |
permissions: | |
contents: read | |
packages: read | |
secrets: inherit | |
with: | |
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} | |
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} | |
helm-test-packages: ${{ needs.build-info.outputs.helm-test-packages }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
if: > | |
needs.build-info.outputs.needs-helm-tests == 'true' && | |
needs.build-info.outputs.default-branch == 'main' && | |
needs.build-info.outputs.latest-versions-only != 'true' | |
tests-postgres: | |
name: "Postgres tests" | |
uses: ./.github/workflows/run-unit-tests.yml | |
needs: [build-info, wait-for-ci-images] | |
permissions: | |
contents: read | |
packages: read | |
secrets: inherit | |
with: | |
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} | |
backend: "postgres" | |
test-name: "Postgres" | |
test-scope: "DB" | |
test-groups: ${{ needs.build-info.outputs.test-groups }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
python-versions: ${{ needs.build-info.outputs.python-versions }} | |
backend-versions: ${{ needs.build-info.outputs.postgres-versions }} | |
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
excludes: ${{ needs.build-info.outputs.postgres-exclude }} | |
core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }} | |
providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }} | |
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
run-migration-tests: "true" | |
run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
if: needs.build-info.outputs.run-tests == 'true' | |
tests-mysql: | |
name: "MySQL tests" | |
uses: ./.github/workflows/run-unit-tests.yml | |
needs: [build-info, wait-for-ci-images] | |
permissions: | |
contents: read | |
packages: read | |
secrets: inherit | |
with: | |
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} | |
backend: "mysql" | |
test-name: "MySQL" | |
test-scope: "DB" | |
test-groups: ${{ needs.build-info.outputs.test-groups }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
python-versions: ${{ needs.build-info.outputs.python-versions }} | |
backend-versions: ${{ needs.build-info.outputs.mysql-versions }} | |
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
excludes: ${{ needs.build-info.outputs.mysql-exclude }} | |
core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }} | |
providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }} | |
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
run-migration-tests: "true" | |
debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
if: needs.build-info.outputs.run-tests == 'true' | |
tests-sqlite: | |
name: "Sqlite tests" | |
uses: ./.github/workflows/run-unit-tests.yml | |
needs: [build-info, wait-for-ci-images] | |
permissions: | |
contents: read | |
packages: read | |
secrets: inherit | |
with: | |
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} | |
backend: "sqlite" | |
test-name: "Sqlite" | |
test-name-separator: "" | |
test-scope: "DB" | |
test-groups: ${{ needs.build-info.outputs.test-groups }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
python-versions: ${{ needs.build-info.outputs.python-versions }} | |
# No versions for sqlite | |
backend-versions: "['']" | |
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
excludes: ${{ needs.build-info.outputs.sqlite-exclude }} | |
core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }} | |
providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }} | |
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
run-migration-tests: "true" | |
debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
if: needs.build-info.outputs.run-tests == 'true' | |
tests-non-db: | |
name: "Non-DB tests" | |
uses: ./.github/workflows/run-unit-tests.yml | |
needs: [build-info, wait-for-ci-images] | |
permissions: | |
contents: read | |
packages: read | |
secrets: inherit | |
with: | |
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} | |
backend: "sqlite" | |
test-name: "" | |
test-name-separator: "" | |
test-scope: "Non-DB" | |
test-groups: ${{ needs.build-info.outputs.test-groups }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
python-versions: ${{ needs.build-info.outputs.python-versions }} | |
# No versions for non-db | |
backend-versions: "['']" | |
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
excludes: ${{ needs.build-info.outputs.sqlite-exclude }} | |
core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }} | |
providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }} | |
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
if: needs.build-info.outputs.run-tests == 'true' | |
tests-special: | |
name: "Special tests" | |
uses: ./.github/workflows/special-tests.yml | |
needs: [build-info, wait-for-ci-images] | |
permissions: | |
contents: read | |
packages: read | |
secrets: inherit | |
if: > | |
needs.build-info.outputs.run-tests == 'true' && | |
(needs.build-info.outputs.canary-run == 'true' || | |
needs.build-info.outputs.upgrade-to-newer-dependencies != 'false' || | |
needs.build-info.outputs.full-tests-needed == 'true') | |
with: | |
test-groups: ${{ needs.build-info.outputs.test-groups }} | |
default-branch: ${{ needs.build-info.outputs.default-branch }} | |
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }} | |
providers-test-types-list-as-string: ${{ needs.build-info.outputs.providers-test-types-list-as-string }} | |
run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
python-versions: ${{ needs.build-info.outputs.python-versions }} | |
default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} | |
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
canary-run: ${{ needs.build-info.outputs.canary-run }} | |
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
tests-integration-system: | |
name: Integration and System Tests | |
needs: [build-info, wait-for-ci-images] | |
uses: ./.github/workflows/integration-system-tests.yml | |
permissions: | |
contents: read | |
packages: read | |
secrets: inherit | |
with: | |
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
testable-core-integrations: ${{ needs.build-info.outputs.testable-core-integrations }} | |
testable-providers-integrations: ${{ needs.build-info.outputs.testable-providers-integrations }} | |
run-system-tests: ${{ needs.build-info.outputs.run-tests }} | |
default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
default-postgres-version: ${{ needs.build-info.outputs.default-postgres-version }} | |
default-mysql-version: ${{ needs.build-info.outputs.default-mysql-version }} | |
skip-providers-tests: ${{ needs.build-info.outputs.skip-providers-tests }} | |
run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
if: needs.build-info.outputs.run-tests == 'true' | |
tests-with-lowest-direct-resolution: | |
name: "Lowest direct dependency providers tests" | |
needs: [build-info, wait-for-ci-images] | |
uses: ./.github/workflows/run-unit-tests.yml | |
permissions: | |
contents: read | |
packages: read | |
secrets: inherit | |
if: > | |
needs.build-info.outputs.run-tests == 'true' | |
with: | |
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} | |
test-name: "LowestDeps-Postgres" | |
force-lowest-dependencies: "true" | |
test-scope: "All" | |
test-groups: ${{ needs.build-info.outputs.test-groups }} | |
backend: "postgres" | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
python-versions: ${{ needs.build-info.outputs.python-versions }} | |
backend-versions: "['${{ needs.build-info.outputs.default-postgres-version }}']" | |
excluded-providers-as-string: ${{ needs.build-info.outputs.excluded-providers-as-string }} | |
excludes: "[]" | |
core-test-types-list-as-string: ${{ needs.build-info.outputs.core-test-types-list-as-string }} | |
# yamllint disable rule:line-length | |
providers-test-types-list-as-string: ${{ needs.build-info.outputs.individual-providers-test-types-list-as-string }} | |
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
run-coverage: ${{ needs.build-info.outputs.run-coverage }} | |
debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
monitor-delay-time-in-seconds: 120 | |
build-prod-images: | |
name: > | |
${{ needs.build-info.outputs.in-workflow-build == 'true' && 'Build' || 'Skip building' }} | |
PROD images in-workflow | |
needs: [build-info, build-ci-images, generate-constraints] | |
uses: ./.github/workflows/prod-image-build.yml | |
permissions: | |
contents: read | |
# This write is only given here for `push` events from "apache/airflow" repo. It is not given for PRs | |
# from forks. This is to prevent malicious PRs from creating images in the "apache/airflow" repo. | |
# For regular build for PRS this "build-prod-images" workflow will be skipped anyway by the | |
# "in-workflow-build" condition | |
packages: write | |
secrets: inherit | |
with: | |
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} | |
build-type: "Regular" | |
do-build: ${{ needs.build-info.outputs.in-workflow-build }} | |
upload-package-artifact: "true" | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
platform: "linux/amd64" | |
python-versions: ${{ needs.build-info.outputs.python-versions }} | |
default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
branch: ${{ needs.build-info.outputs.default-branch }} | |
push-image: "true" | |
use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }} | |
build-provider-packages: ${{ needs.build-info.outputs.default-branch == 'main' }} | |
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
docker-cache: ${{ needs.build-info.outputs.docker-cache }} | |
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} | |
wait-for-prod-images: | |
timeout-minutes: 80 | |
name: "Wait for PROD images" | |
runs-on: ${{ fromJSON(needs.build-info.outputs.runs-on-as-json-public) }} | |
needs: [build-info, wait-for-ci-images, build-prod-images] | |
if: needs.build-info.outputs.prod-image-build == 'true' | |
env: | |
BACKEND: sqlite | |
PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" | |
# Force more parallelism for pull on public images | |
PARALLELISM: 6 | |
INCLUDE_SUCCESS_OUTPUTS: "${{needs.build-info.outputs.include-success-outputs}}" | |
IMAGE_TAG: ${{ needs.build-info.outputs.image-tag }} | |
steps: | |
- name: "Cleanup repo" | |
shell: bash | |
run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" | |
if: needs.build-info.outputs.in-workflow-build == 'false' | |
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" | |
uses: actions/checkout@v4 | |
with: | |
persist-credentials: false | |
if: needs.build-info.outputs.in-workflow-build == 'false' | |
- name: "Cleanup docker" | |
run: ./scripts/ci/cleanup_docker.sh | |
if: needs.build-info.outputs.in-workflow-build == 'false' | |
- name: "Install Breeze" | |
uses: ./.github/actions/breeze | |
if: needs.build-info.outputs.in-workflow-build == 'false' | |
- name: Login to ghcr.io | |
run: echo "${{ env.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin | |
if: needs.build-info.outputs.in-workflow-build == 'false' | |
- name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ needs.build-info.outputs.image-tag }} | |
# We wait for the images to be available either from "build-images.yml' run as pull_request_target | |
# or from build-prod-images (or build-prod-images-release-branch) above. | |
# We are utilising single job to wait for all images because this job merely waits | |
# For the images to be available. | |
run: breeze prod-image pull --wait-for-image --run-in-parallel | |
env: | |
PYTHON_VERSIONS: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
DEBUG_RESOURCES: ${{ needs.build-info.outputs.debug-resources }} | |
if: needs.build-info.outputs.in-workflow-build == 'false' | |
additional-prod-image-tests: | |
name: "Additional PROD image tests" | |
needs: [build-info, wait-for-prod-images, generate-constraints] | |
uses: ./.github/workflows/additional-prod-image-tests.yml | |
with: | |
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} | |
default-branch: ${{ needs.build-info.outputs.default-branch }} | |
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
chicken-egg-providers: ${{ needs.build-info.outputs.chicken-egg-providers }} | |
docker-cache: ${{ needs.build-info.outputs.docker-cache }} | |
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} | |
default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
canary-run: ${{ needs.build-info.outputs.canary-run }} | |
if: needs.build-info.outputs.prod-image-build == 'true' | |
tests-kubernetes: | |
name: "Kubernetes tests" | |
uses: ./.github/workflows/k8s-tests.yml | |
needs: [build-info, wait-for-prod-images] | |
permissions: | |
contents: read | |
packages: read | |
secrets: inherit | |
with: | |
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
kubernetes-versions-list-as-string: ${{ needs.build-info.outputs.kubernetes-versions-list-as-string }} | |
kubernetes-combos-list-as-string: ${{ needs.build-info.outputs.kubernetes-combos-list-as-string }} | |
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
use-uv: ${{ needs.build-info.outputs.force-pip == 'true' && 'false' || 'true' }} | |
debug-resources: ${{ needs.build-info.outputs.debug-resources }} | |
if: > | |
( needs.build-info.outputs.run-kubernetes-tests == 'true' || | |
needs.build-info.outputs.needs-helm-tests == 'true') | |
tests-task-sdk: | |
name: "Task SDK tests" | |
uses: ./.github/workflows/task-sdk-tests.yml | |
needs: [build-info, wait-for-ci-images] | |
permissions: | |
contents: read | |
packages: read | |
secrets: inherit | |
with: | |
runs-on-as-json-default: ${{ needs.build-info.outputs.runs-on-as-json-default }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
python-versions: ${{ needs.build-info.outputs.python-versions }} | |
run-task-sdk-tests: ${{ needs.build-info.outputs.run-task-sdk-tests }} | |
if: > | |
( needs.build-info.outputs.run-task-sdk-tests == 'true' || | |
needs.build-info.outputs.run-tests == 'true' && | |
needs.build-info.outputs.only-new-ui-files != 'true') | |
finalize-tests: | |
name: Finalize tests | |
permissions: | |
contents: write | |
packages: write | |
secrets: inherit | |
needs: | |
- build-info | |
- generate-constraints | |
- wait-for-ci-images | |
- wait-for-prod-images | |
- ci-image-checks | |
- tests-sqlite | |
- tests-mysql | |
- tests-postgres | |
- tests-non-db | |
- tests-integration-system | |
uses: ./.github/workflows/finalize-tests.yml | |
with: | |
runs-on-as-json-public: ${{ needs.build-info.outputs.runs-on-as-json-public }} | |
runs-on-as-json-self-hosted: ${{ needs.build-info.outputs.runs-on-as-json-self-hosted }} | |
image-tag: ${{ needs.build-info.outputs.image-tag }} | |
python-versions: ${{ needs.build-info.outputs.python-versions }} | |
python-versions-list-as-string: ${{ needs.build-info.outputs.python-versions-list-as-string }} | |
branch: ${{ needs.build-info.outputs.default-branch }} | |
constraints-branch: ${{ needs.build-info.outputs.default-constraints-branch }} | |
default-python-version: ${{ needs.build-info.outputs.default-python-version }} | |
in-workflow-build: ${{ needs.build-info.outputs.in-workflow-build }} | |
upgrade-to-newer-dependencies: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} | |
include-success-outputs: ${{ needs.build-info.outputs.include-success-outputs }} | |
docker-cache: ${{ needs.build-info.outputs.docker-cache }} | |
disable-airflow-repo-cache: ${{ needs.build-info.outputs.disable-airflow-repo-cache }} | |
canary-run: ${{ needs.build-info.outputs.canary-run }} | |
notify-slack-failure: | |
name: "Notify Slack on Failure" | |
needs: | |
- basic-tests | |
- additional-ci-image-checks | |
- providers | |
- tests-helm | |
- tests-special | |
- tests-with-lowest-direct-resolution | |
- additional-prod-image-tests | |
- tests-kubernetes | |
- tests-task-sdk | |
- finalize-tests | |
if: github.event_name == 'schedule' && failure() && github.run_attempt == 1 | |
runs-on: ["ubuntu-22.04"] | |
steps: | |
- name: Notify Slack | |
id: slack | |
uses: slackapi/slack-github-action@485a9d42d3a73031f12ec201c457e2162c45d02d # v2.0.0 | |
with: | |
method: chat.postMessage | |
token: ${{ env.SLACK_BOT_TOKEN }} | |
# yamllint disable rule:line-length | |
payload: | | |
channel: "internal-airflow-ci-cd" | |
text: "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>" | |
blocks: | |
- type: "section" | |
text: | |
type: "mrkdwn" | |
text: "🚨🕒 Scheduled CI Failure Alert 🕒🚨\n\n*Details:* <https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}|View the failure log>" | |
# yamllint enable rule:line-length |