From 470e0851934e7b71c1ffbc895abc1175d3af27ce Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Fri, 1 Dec 2023 12:15:43 +0100 Subject: [PATCH] Add support for chicken-egg providers to dockerhub release process --- .github/workflows/release_dockerhub_image.yml | 17 +++++ .../commands/release_management_commands.py | 30 ++++++--- .../release_management_commands_config.py | 1 + .../airflow_breeze/utils/common_options.py | 8 +++ .../src/airflow_breeze/utils/versions.py | 6 ++ ...release-management_release-prod-images.svg | 62 ++++++++++++------- ...release-management_release-prod-images.txt | 2 +- 7 files changed, 96 insertions(+), 30 deletions(-) diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 6889539387f6a..a5c44df03f48d 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -46,6 +46,7 @@ jobs: pythonVersions: ${{ steps.selective-checks.outputs.python-versions }} allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} + chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} skipLatest: ${{ github.event.inputs.skipLatest == '' && ' ' || '--skip-latest' }} limitPlatform: ${{ github.repository == 'apache/airflow' && ' ' || '--limit-platform linux/amd64' }} env: @@ -107,6 +108,20 @@ jobs: run: > echo ${{ secrets.DOCKERHUB_TOKEN }} | docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }} + - name: "Prepare chicken-eggs provider packages" + # In case of provider packages which use latest dev0 version of providers, we should prepare them + # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency + # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0 + shell: bash + run: > + breeze release-management prepare-provider-packages + --package-format wheel + --version-suffix-for-pypi dev0 ${{ needs.build-info.outputs.chicken-egg-providers }} + if: inputs.chicken-egg-providers != '' + - name: "Copy dist packages to docker-context files" + shell: bash + run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files + if: needs.build-info.outputs.chicken-egg-providers != '' - name: > Release regular images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }} run: > @@ -116,6 +131,7 @@ jobs: ${{ needs.build-info.outputs.skipLatest }} ${{ needs.build-info.outputs.limitPlatform }} --limit-python ${{ matrix.python-version }} + --chicken-egg-providers "${{ needs.build-info.outputs.chicken-egg-providers }}" env: COMMIT_SHA: ${{ github.sha }} - name: > @@ -127,6 +143,7 @@ jobs: ${{ needs.build-info.outputs.skipLatest }} ${{ needs.build-info.outputs.limitPlatform }} --limit-python ${{ matrix.python-version }} --slim-images + --chicken-egg-providers "${{ needs.build-info.outputs.chicken-egg-providers }}" env: COMMIT_SHA: ${{ github.sha }} - name: "Stop ARM instance" diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 5aa1f65bf114d..9d4b266f9fff1 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -70,6 +70,7 @@ option_airflow_extras, option_airflow_site_directory, option_answer, + option_chicken_egg_providers, option_commit_sha, option_debug_resources, option_dry_run, @@ -138,6 +139,7 @@ run_compile_www_assets, ) from airflow_breeze.utils.shared_options import get_dry_run, get_verbose +from airflow_breeze.utils.versions import is_pre_release option_debug_release_management = click.option( "--debug", @@ -684,14 +686,7 @@ def run_generate_constraints_in_parallel( @option_image_tag_for_running @option_debug_release_management @option_airflow_constraints_mode_ci -@click.option( - "--chicken-egg-providers", - default="", - help="List of chicken-egg provider packages - " - "those that have airflow_version >= current_version and should " - "be installed in CI from locally built packages with >= current_version.dev0 ", - envvar="CHICKEN_EGG_PROVIDERS", -) +@option_chicken_egg_providers @option_github_repository @option_verbose @option_dry_run @@ -1206,6 +1201,17 @@ def add_back_references( start_generating_back_references(site_path, list(expand_all_provider_packages(doc_packages))) +def _add_chicken_egg_providers_to_build_args( + python_build_args: dict[str, str], chicken_egg_providers: str, airflow_version: str +): + if chicken_egg_providers and is_pre_release(airflow_version): + get_console().print( + f"[info]Adding chicken egg providers to build args as {airflow_version} is " + f"pre release and we have chicken-egg packages '{chicken_egg_providers}' defined[/]" + ) + python_build_args["INSTALL_PACKAGES_FROM_CONTEXT"] = "true" + + @release_management.command( name="release-prod-images", help="Release production images to DockerHub (needs DockerHub permissions)." ) @@ -1242,6 +1248,7 @@ def add_back_references( "rc/alpha/beta images are built.", ) @option_commit_sha +@option_chicken_egg_providers @option_verbose @option_dry_run def release_prod_images( @@ -1252,6 +1259,7 @@ def release_prod_images( limit_python: str | None, commit_sha: str | None, skip_latest: bool, + chicken_egg_providers: str, ): perform_environment_checks() check_remote_ghcr_io_commands() @@ -1307,6 +1315,9 @@ def release_prod_images( get_console().print(f"[info]Building slim {airflow_version} image for Python {python}[/]") python_build_args = deepcopy(slim_build_args) slim_image_name = f"{dockerhub_repo}:slim-{airflow_version}-python{python}" + _add_chicken_egg_providers_to_build_args( + python_build_args, chicken_egg_providers, airflow_version + ) docker_buildx_command = [ "docker", "buildx", @@ -1336,6 +1347,9 @@ def release_prod_images( } if commit_sha: regular_build_args["COMMIT_SHA"] = commit_sha + _add_chicken_egg_providers_to_build_args( + regular_build_args, chicken_egg_providers, airflow_version + ) docker_buildx_command = [ "docker", "buildx", diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py index ff0c9cdcd0426..fb60e12463e9b 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py @@ -161,6 +161,7 @@ "--limit-platform", "--skip-latest", "--commit-sha", + "--chicken-egg-providers", ], } ], diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index cf59df8bc09a3..d053a00514195 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -787,3 +787,11 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, is_flag=True, envvar="RESTART", ) +option_chicken_egg_providers = click.option( + "--chicken-egg-providers", + default="", + help="List of chicken-egg provider packages - " + "those that have airflow_version >= current_version and should " + "be installed in CI from locally built packages with >= current_version.dev0 ", + envvar="CHICKEN_EGG_PROVIDERS", +) diff --git a/dev/breeze/src/airflow_breeze/utils/versions.py b/dev/breeze/src/airflow_breeze/utils/versions.py index 88c5986f7d975..70dc6ad77d38b 100644 --- a/dev/breeze/src/airflow_breeze/utils/versions.py +++ b/dev/breeze/src/airflow_breeze/utils/versions.py @@ -34,3 +34,9 @@ def get_version_tag(version: str, provider_package_id: str, version_suffix: str if version_suffix is None: version_suffix = "" return f"providers-{provider_package_id.replace('.','-')}/{version}{version_suffix}" + + +def is_pre_release(version: str) -> bool: + from packaging.version import Version + + return Version(version).is_prerelease diff --git a/images/breeze/output_release-management_release-prod-images.svg b/images/breeze/output_release-management_release-prod-images.svg index 76922057e1f60..f1e6d24f2c072 100644 --- a/images/breeze/output_release-management_release-prod-images.svg +++ b/images/breeze/output_release-management_release-prod-images.svg @@ -1,4 +1,4 @@ - +