Add support for chicken-egg providers to dockerhub release process

.github/workflows/release_dockerhub_image.yml

@@ -46,6 +46,7 @@ jobs:
       pythonVersions: ${{ steps.selective-checks.outputs.python-versions }}
       allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }}
       defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }}
+      chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }}
       skipLatest: ${{ github.event.inputs.skipLatest == '' && ' ' || '--skip-latest' }}
       limitPlatform: ${{ github.repository == 'apache/airflow' && ' ' || '--limit-platform linux/amd64' }}
     env:
@@ -107,6 +108,20 @@ jobs:
         run: >
           echo ${{ secrets.DOCKERHUB_TOKEN }} |
           docker login --password-stdin --username ${{ secrets.DOCKERHUB_USER }}
+      - name: "Prepare chicken-eggs provider packages"
+        # In case of provider packages which use latest dev0 version of providers, we should prepare them
+        # from the source code, not from the PyPI because they have apache-airflow>=X.Y.Z dependency
+        # And when we prepare them from sources they will have apache-airflow>=X.Y.Z.dev0
+        shell: bash
+        run: >
+          breeze release-management prepare-provider-packages
+          --package-format wheel
+          --version-suffix-for-pypi dev0 ${{ needs.build-info.outputs.chicken-egg-providers }}
+        if: inputs.chicken-egg-providers != ''
+      - name: "Copy dist packages to docker-context files"
+        shell: bash
+        run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files
+        if: needs.build-info.outputs.chicken-egg-providers != ''
       - name: >
           Release regular images: ${{ github.event.inputs.airflowVersion }}, ${{ matrix.python-version }}
         run: >
@@ -116,6 +131,7 @@ jobs:
           ${{ needs.build-info.outputs.skipLatest }}
           ${{ needs.build-info.outputs.limitPlatform }}
           --limit-python ${{ matrix.python-version }}
+          --chicken-egg-providers "${{ needs.build-info.outputs.chicken-egg-providers }}"
         env:
           COMMIT_SHA: ${{ github.sha }}
       - name: >
@@ -127,6 +143,7 @@ jobs:
           ${{ needs.build-info.outputs.skipLatest }}
           ${{ needs.build-info.outputs.limitPlatform }}
           --limit-python ${{ matrix.python-version }} --slim-images
+          --chicken-egg-providers "${{ needs.build-info.outputs.chicken-egg-providers }}"
         env:
           COMMIT_SHA: ${{ github.sha }}
       - name: "Stop ARM instance"

dev/breeze/src/airflow_breeze/commands/release_management_commands.py

@@ -70,6 +70,7 @@
     option_airflow_extras,
     option_airflow_site_directory,
     option_answer,
+    option_chicken_egg_providers,
     option_commit_sha,
     option_debug_resources,
     option_dry_run,
@@ -138,6 +139,7 @@
     run_compile_www_assets,
 )
 from airflow_breeze.utils.shared_options import get_dry_run, get_verbose
+from airflow_breeze.utils.versions import is_pre_release
 
 option_debug_release_management = click.option(
     "--debug",
@@ -684,14 +686,7 @@ def run_generate_constraints_in_parallel(
 @option_image_tag_for_running
 @option_debug_release_management
 @option_airflow_constraints_mode_ci
-@click.option(
-    "--chicken-egg-providers",
-    default="",
-    help="List of chicken-egg provider packages - "
-    "those that have airflow_version >= current_version and should "
-    "be installed in CI from locally built packages with >= current_version.dev0 ",
-    envvar="CHICKEN_EGG_PROVIDERS",
-)
+@option_chicken_egg_providers
 @option_github_repository
 @option_verbose
 @option_dry_run
@@ -1206,6 +1201,17 @@ def add_back_references(
     start_generating_back_references(site_path, list(expand_all_provider_packages(doc_packages)))
 
 
+def _add_chicken_egg_providers_to_build_args(
+    python_build_args: dict[str, str], chicken_egg_providers: str, airflow_version: str
+):
+    if chicken_egg_providers and is_pre_release(airflow_version):
+        get_console().print(
+            f"[info]Adding chicken egg providers to build args as {airflow_version} is "
+            f"pre release and we have chicken-egg packages '{chicken_egg_providers}' defined[/]"
+        )
+        python_build_args["INSTALL_PACKAGES_FROM_CONTEXT"] = "true"
+
+
 @release_management.command(
     name="release-prod-images", help="Release production images to DockerHub (needs DockerHub permissions)."
 )
@@ -1242,6 +1248,7 @@ def add_back_references(
     "rc/alpha/beta images are built.",
 )
 @option_commit_sha
+@option_chicken_egg_providers
 @option_verbose
 @option_dry_run
 def release_prod_images(
@@ -1252,6 +1259,7 @@ def release_prod_images(
     limit_python: str | None,
     commit_sha: str | None,
     skip_latest: bool,
+    chicken_egg_providers: str,
 ):
     perform_environment_checks()
     check_remote_ghcr_io_commands()
@@ -1307,6 +1315,9 @@ def release_prod_images(
             get_console().print(f"[info]Building slim {airflow_version} image for Python {python}[/]")
             python_build_args = deepcopy(slim_build_args)
             slim_image_name = f"{dockerhub_repo}:slim-{airflow_version}-python{python}"
+            _add_chicken_egg_providers_to_build_args(
+                python_build_args, chicken_egg_providers, airflow_version
+            )
             docker_buildx_command = [
                 "docker",
                 "buildx",
@@ -1336,6 +1347,9 @@ def release_prod_images(
             }
             if commit_sha:
                 regular_build_args["COMMIT_SHA"] = commit_sha
+            _add_chicken_egg_providers_to_build_args(
+                regular_build_args, chicken_egg_providers, airflow_version
+            )
             docker_buildx_command = [
                 "docker",
                 "buildx",

dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py

@@ -161,6 +161,7 @@
                 "--limit-platform",
                 "--skip-latest",
                 "--commit-sha",
+                "--chicken-egg-providers",
             ],
         }
     ],

dev/breeze/src/airflow_breeze/utils/common_options.py

@@ -787,3 +787,11 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option,
     is_flag=True,
     envvar="RESTART",
 )
+option_chicken_egg_providers = click.option(
+    "--chicken-egg-providers",
+    default="",
+    help="List of chicken-egg provider packages - "
+    "those that have airflow_version >= current_version and should "
+    "be installed in CI from locally built packages with >= current_version.dev0 ",
+    envvar="CHICKEN_EGG_PROVIDERS",
+)

dev/breeze/src/airflow_breeze/utils/versions.py

@@ -34,3 +34,9 @@ def get_version_tag(version: str, provider_package_id: str, version_suffix: str
     if version_suffix is None:
         version_suffix = ""
     return f"providers-{provider_package_id.replace('.','-')}/{version}{version_suffix}"
+
+
+def is_pre_release(version: str) -> bool:
+    from packaging.version import Version
+
+    return Version(version).is_prerelease