diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 560f3852ecb0f..f33fdad2ef491 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -22,6 +22,7 @@ import subprocess import sys import time +from copy import deepcopy from functools import partial from pathlib import Path from typing import TYPE_CHECKING, Any, Callable @@ -32,10 +33,10 @@ from airflow_breeze.utils.ci_group import ci_group from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.common_options import ( + option_additional_airflow_extras, option_additional_dev_apt_command, option_additional_dev_apt_deps, option_additional_dev_apt_env, - option_additional_extras, option_additional_pip_install_flags, option_additional_python_deps, option_airflow_constraints_location, @@ -53,7 +54,6 @@ option_docker_cache, option_dry_run, option_eager_upgrade_additional_requirements, - option_force_build, option_github_repository, option_github_token, option_image_name, @@ -77,7 +77,7 @@ option_upgrade_to_newer_dependencies, option_verbose, option_verify, - option_version_suffix_for_pypi, + option_version_suffix_for_pypi_ci, option_wait_for_image, ) from airflow_breeze.utils.confirm import STANDARD_TIMEOUT, Answer, user_confirm @@ -99,7 +99,6 @@ from airflow_breeze.utils.registry import login_to_github_docker_registry from airflow_breeze.utils.run_tests import verify_an_image from airflow_breeze.utils.run_utils import ( - filter_out_none, fix_group_permissions, instruct_build_image, is_repo_rebased, @@ -224,12 +223,6 @@ def get_exitcode(status: int) -> int: @ci_image.command(name="build") @option_python @option_debian_version -@option_run_in_parallel -@option_parallelism -@option_skip_cleanup -@option_debug_resources -@option_include_success_outputs -@option_python_versions @option_upgrade_to_newer_dependencies @option_upgrade_on_failure @option_platform_multiple @@ -239,18 +232,16 @@ def get_exitcode(status: int) -> int: @option_prepare_buildx_cache @option_push @option_install_providers_from_sources -@option_additional_extras +@option_additional_airflow_extras @option_additional_dev_apt_deps @option_additional_python_deps @option_additional_dev_apt_command @option_additional_dev_apt_env @option_builder @option_build_progress -@option_build_timeout_minutes @option_commit_sha @option_dev_apt_command @option_dev_apt_deps -@option_force_build @option_python_image @option_eager_upgrade_additional_requirements @option_airflow_constraints_location @@ -259,19 +250,58 @@ def get_exitcode(status: int) -> int: @option_tag_as_latest @option_additional_pip_install_flags @option_github_repository -@option_version_suffix_for_pypi +@option_version_suffix_for_pypi_ci +@option_build_timeout_minutes +@option_run_in_parallel +@option_parallelism +@option_skip_cleanup +@option_debug_resources +@option_include_success_outputs +@option_python_versions @option_verbose @option_dry_run @option_answer def build( + # Build options + python: str, + debian_version: str, + upgrade_to_newer_dependencies: bool, + upgrade_on_failure: bool, + platform: str | None, + github_token: str | None, + docker_cache: str, + image_tag: str, + prepare_buildx_cache: bool, + push: bool, + install_providers_from_sources: bool, + additional_airflow_extras: str | None, + additional_dev_apt_deps: str | None, + additional_python_deps: str | None, + additional_dev_apt_command: str | None, + additional_dev_apt_env: str | None, + builder: str, + build_progress: str, + commit_sha: str | None, + dev_apt_command: str | None, + dev_apt_deps: str | None, + eager_upgrade_additional_requirements: str | None, + airflow_constraints_location: str | None, + airflow_constraints_mode: str, + airflow_constraints_reference: str, + tag_as_latest: bool, + additional_pip_install_flags: str | None, + github_repository: str, + python_image: str | None, + version_suffix_for_pypi: str, + # Parallel building run_in_parallel: bool, parallelism: int, skip_cleanup: bool, debug_resources: bool, include_success_outputs, python_versions: str, + # Other options build_timeout_minutes: int | None, - **kwargs: dict[str, Any], ): """Build CI image. Include building multiple images for all python versions.""" @@ -306,16 +336,51 @@ def run_build(ci_image_params: BuildCiParams) -> None: perform_environment_checks() check_remote_ghcr_io_commands() - parameters_passed = filter_out_none(**kwargs) - parameters_passed["force_build"] = True fix_group_permissions() + base_build_params = BuildCiParams( + force_build=True, + python=python, + debian_version=debian_version, + upgrade_to_newer_dependencies=upgrade_to_newer_dependencies, + upgrade_on_failure=upgrade_on_failure, + github_token=github_token, + docker_cache=docker_cache, + image_tag=image_tag, + prepare_buildx_cache=prepare_buildx_cache, + push=push, + install_providers_from_sources=install_providers_from_sources, + additional_airflow_extras=additional_airflow_extras, + additional_python_deps=additional_python_deps, + additional_dev_apt_command=additional_dev_apt_command, + additional_dev_apt_env=additional_dev_apt_env, + builder=builder, + build_progress=build_progress, + commit_sha=commit_sha, + dev_apt_command=dev_apt_command, + dev_apt_deps=dev_apt_deps, + eager_upgrade_additional_requirements=eager_upgrade_additional_requirements, + airflow_constraints_location=airflow_constraints_location, + airflow_constraints_mode=airflow_constraints_mode, + airflow_constraints_reference=airflow_constraints_reference, + tag_as_latest=tag_as_latest, + additional_pip_install_flags=additional_pip_install_flags, + github_repository=github_repository, + python_image=python_image, + version_suffix_for_pypi=version_suffix_for_pypi, + ) + if platform: + base_build_params.platform = platform + if additional_dev_apt_deps: + # For CI image we only set additional_dev_apt_deps when we explicitly pass it + base_build_params.additional_dev_apt_deps = additional_dev_apt_deps + if run_in_parallel: python_version_list = get_python_version_list(python_versions) params_list: list[BuildCiParams] = [] for python in python_version_list: - params = BuildCiParams(**parameters_passed) - params.python = python - params_list.append(params) + build_params = deepcopy(base_build_params) + build_params.python = python + params_list.append(build_params) prepare_for_building_ci_image(params=params_list[0]) run_build_in_parallel( image_params_list=params_list, @@ -326,9 +391,8 @@ def run_build(ci_image_params: BuildCiParams) -> None: debug_resources=debug_resources, ) else: - params = BuildCiParams(**parameters_passed) - prepare_for_building_ci_image(params=params) - run_build(ci_image_params=params) + prepare_for_building_ci_image(params=base_build_params) + run_build(ci_image_params=base_build_params) @ci_image.command(name="pull") @@ -550,13 +614,6 @@ def run_build_ci_image( :param ci_image_params: CI image parameters :param output: output redirection """ - if not ci_image_params.version_suffix_for_pypi: - # We need that to handle the >= 2.7.0 limit we have for openlineage provider at least until - # Airflow 2.7.0 release is out, in order to avoid conflicting dependencies while building the image - # We are setting version_suffix_for_pypi to dev0 for CI builds where cache is prepared, so in - # order to have the cache used effectively, we should also locally force the version_suffix_for_pypi - # to dev0. We might evan leave it as default value in the future (to be decided after 2.7.0 release) - ci_image_params.version_suffix_for_pypi = "dev0" if ( ci_image_params.is_multi_platform() and not ci_image_params.push @@ -645,8 +702,6 @@ def rebuild_or_pull_ci_image_if_needed(command_params: ShellParams | BuildCiPara Rebuilds CI image if needed and user confirms it. :param command_params: parameters of the command to execute - - """ build_ci_image_check_cache = Path( BUILD_CACHE_DIR, command_params.airflow_branch, f".built_{command_params.python}" diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py index 87e6ab6e915cd..e128aa035b37d 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands_config.py @@ -35,7 +35,7 @@ "--image-tag", "--tag-as-latest", "--docker-cache", - "--force-build", + "--version-suffix-for-pypi", "--build-progress", ], }, @@ -51,24 +51,33 @@ ], }, { - "name": "Advanced options (for power users)", + "name": "Advanced build options (for power users)", "options": [ "--debian-version", + "--python-image", + "--commit-sha", + "--additional-pip-install-flags", "--install-providers-from-sources", + ], + }, + { + "name": "Selecting constraint location (for power users)", + "options": [ "--airflow-constraints-location", "--airflow-constraints-mode", "--airflow-constraints-reference", - "--python-image", - "--additional-python-deps", + ], + }, + { + "name": "Choosing dependencies and extras (for power users)", + "options": [ "--additional-airflow-extras", - "--additional-pip-install-flags", - "--additional-dev-apt-deps", - "--additional-dev-apt-env", - "--additional-dev-apt-command", + "--additional-python-deps", "--dev-apt-deps", + "--additional-dev-apt-deps", "--dev-apt-command", - "--version-suffix-for-pypi", - "--commit-sha", + "--additional-dev-apt-command", + "--additional-dev-apt-env", ], }, { diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 9f132aa3e7fac..ca1780738db33 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -18,7 +18,7 @@ import os import sys -from typing import Any +from copy import deepcopy import click @@ -27,10 +27,10 @@ from airflow_breeze.utils.ci_group import ci_group from airflow_breeze.utils.click_utils import BreezeGroup from airflow_breeze.utils.common_options import ( + option_additional_airflow_extras, option_additional_dev_apt_command, option_additional_dev_apt_deps, option_additional_dev_apt_env, - option_additional_extras, option_additional_pip_install_flags, option_additional_python_deps, option_additional_runtime_apt_command, @@ -39,6 +39,7 @@ option_airflow_constraints_location, option_airflow_constraints_mode_prod, option_airflow_constraints_reference_build, + option_answer, option_build_progress, option_builder, option_commit_sha, @@ -90,7 +91,7 @@ from airflow_breeze.utils.python_versions import get_python_version_list from airflow_breeze.utils.registry import login_to_github_docker_registry from airflow_breeze.utils.run_tests import verify_an_image -from airflow_breeze.utils.run_utils import filter_out_none, fix_group_permissions, run_command +from airflow_breeze.utils.run_utils import fix_group_permissions, run_command from airflow_breeze.utils.shared_options import get_dry_run, get_verbose @@ -151,26 +152,38 @@ def prod_image(): @prod_image.command(name="build") @option_python @option_debian_version -@option_run_in_parallel -@option_parallelism -@option_skip_cleanup -@option_debug_resources -@option_include_success_outputs -@option_python_versions @option_platform_multiple @option_github_token @option_docker_cache @option_image_tag_for_building @option_prepare_buildx_cache @option_push +@option_install_providers_from_sources +@click.option("-V", "--install-airflow-version", help="Install version of Airflow from PyPI.") +@option_additional_airflow_extras +@option_additional_dev_apt_deps +@option_additional_runtime_apt_deps +@option_additional_python_deps +@option_additional_dev_apt_command +@option_additional_dev_apt_env +@option_additional_runtime_apt_env +@option_additional_runtime_apt_command +@option_builder +@option_build_progress +@option_commit_sha +@option_dev_apt_command +@option_dev_apt_deps +@option_runtime_apt_command +@option_runtime_apt_deps @option_airflow_constraints_location @option_airflow_constraints_mode_prod @click.option( "--installation-method", help="Install Airflow from: sources or PyPI.", type=BetterChoice(ALLOWED_INSTALLATION_METHODS), + default=ALLOWED_INSTALLATION_METHODS[0], + show_default=True, ) -@option_install_providers_from_sources @click.option( "--install-packages-from-context", help="Install wheels from local docker-context-files when building image. " @@ -208,37 +221,72 @@ def prod_image(): help="Install Airflow using GitHub tag or branch.", ) @option_airflow_constraints_reference_build -@click.option("-V", "--install-airflow-version", help="Install version of Airflow from PyPI.") -@option_additional_extras -@option_additional_dev_apt_deps -@option_additional_runtime_apt_deps -@option_additional_python_deps -@option_additional_dev_apt_command -@option_additional_dev_apt_env -@option_additional_runtime_apt_env -@option_additional_runtime_apt_command -@option_builder -@option_build_progress -@option_dev_apt_command -@option_dev_apt_deps -@option_python_image -@option_runtime_apt_command -@option_runtime_apt_deps @option_tag_as_latest @option_additional_pip_install_flags @option_github_repository +@option_python_image @option_version_suffix_for_pypi -@option_commit_sha +@option_run_in_parallel +@option_parallelism +@option_skip_cleanup +@option_debug_resources +@option_include_success_outputs +@option_python_versions @option_verbose @option_dry_run +@option_answer def build( + # build options + python: str, + debian_version: str, + platform: str | None, + github_token: str | None, + docker_cache: str, + image_tag: str, + prepare_buildx_cache: bool, + push: bool, + install_providers_from_sources: bool, + install_airflow_version: str | None, + additional_airflow_extras: str | None, + additional_dev_apt_deps: str | None, + additional_runtime_apt_deps: str | None, + additional_python_deps: str | None, + additional_dev_apt_command: str | None, + additional_dev_apt_env: str | None, + additional_runtime_apt_command: str | None, + additional_runtime_apt_env: str | None, + builder: str, + build_progress: str, + commit_sha: str | None, + dev_apt_command: str | None, + dev_apt_deps: str | None, + runtime_apt_command: str | None, + runtime_apt_deps: str | None, + airflow_constraints_location: str | None, + airflow_constraints_mode: str, + installation_method: str, + install_packages_from_context: bool, + use_constraints_for_context_packages: bool, + cleanup_context: bool, + airflow_extras: str, + disable_mysql_client_installation: bool, + disable_mssql_client_installation: bool, + disable_postgres_client_installation: bool, + disable_airflow_repo_cache: bool, + install_airflow_reference: str | None, + airflow_constraints_reference: str | None, + tag_as_latest: bool, + additional_pip_install_flags: str | None, + github_repository: str, + python_image: str | None, + version_suffix_for_pypi: str, + # Parallel building run_in_parallel: bool, parallelism: int, skip_cleanup: bool, debug_resources: bool, - include_success_outputs: bool, + include_success_outputs, python_versions: str, - **kwargs: dict[str, Any], ): """ Build Production image. Include building multiple images for all or selected Python versions sequentially. @@ -252,14 +300,59 @@ def run_build(prod_image_params: BuildProdParams) -> None: perform_environment_checks() check_remote_ghcr_io_commands() - parameters_passed = filter_out_none(**kwargs) + base_build_params = BuildProdParams( + python=python, + debian_version=debian_version, + github_token=github_token, + docker_cache=docker_cache, + image_tag=image_tag, + prepare_buildx_cache=prepare_buildx_cache, + push=push, + install_providers_from_sources=install_providers_from_sources, + install_airflow_version=install_airflow_version, + additional_airflow_extras=additional_airflow_extras, + additional_dev_apt_deps=additional_dev_apt_deps, + additional_runtime_apt_deps=additional_runtime_apt_deps, + additional_python_deps=additional_python_deps, + additional_dev_apt_command=additional_dev_apt_command, + additional_dev_apt_env=additional_dev_apt_env, + additional_runtime_apt_command=additional_runtime_apt_command, + additional_runtime_apt_env=additional_runtime_apt_env, + builder=builder, + build_progress=build_progress, + commit_sha=commit_sha, + dev_apt_command=dev_apt_command, + dev_apt_deps=dev_apt_deps, + runtime_apt_command=runtime_apt_command, + runtime_apt_deps=runtime_apt_deps, + airflow_constraints_location=airflow_constraints_location, + airflow_constraints_mode=airflow_constraints_mode, + installation_method=installation_method, + install_packages_from_context=install_packages_from_context, + use_constraints_for_context_packages=use_constraints_for_context_packages, + cleanup_context=cleanup_context, + airflow_extras=airflow_extras, + disable_mysql_client_installation=disable_mysql_client_installation, + disable_mssql_client_installation=disable_mssql_client_installation, + disable_postgres_client_installation=disable_postgres_client_installation, + disable_airflow_repo_cache=disable_airflow_repo_cache, + install_airflow_reference=install_airflow_reference, + airflow_constraints_reference=airflow_constraints_reference, + tag_as_latest=tag_as_latest, + additional_pip_install_flags=additional_pip_install_flags, + github_repository=github_repository, + python_image=python_image, + version_suffix_for_pypi=version_suffix_for_pypi, + ) + if platform: + base_build_params.platform = platform fix_group_permissions() if run_in_parallel: python_version_list = get_python_version_list(python_versions) params_list: list[BuildProdParams] = [] for python in python_version_list: - params = BuildProdParams(**parameters_passed) + params = deepcopy(base_build_params) params.python = python params_list.append(params) prepare_for_building_prod_image(prod_image_params=params_list[0]) @@ -272,9 +365,8 @@ def run_build(prod_image_params: BuildProdParams) -> None: include_success_outputs=include_success_outputs, ) else: - params = BuildProdParams(**parameters_passed) - prepare_for_building_prod_image(prod_image_params=params) - run_build(prod_image_params=params) + prepare_for_building_prod_image(prod_image_params=base_build_params) + run_build(prod_image_params=base_build_params) @prod_image.command(name="pull") diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py index 713bd0bfa42c5..52291268f86e8 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands_config.py @@ -34,6 +34,7 @@ "--image-tag", "--tag-as-latest", "--docker-cache", + "--version-suffix-for-pypi", "--build-progress", ], }, @@ -49,44 +50,53 @@ ], }, { - "name": "Options for customizing images", + "name": "Advanced build options (for power users)", "options": [ + "--debian-version", + "--python-image", + "--commit-sha", + "--additional-pip-install-flags", "--install-providers-from-sources", - "--airflow-extras", + ], + }, + { + "name": "Selecting constraint location (for power users)", + "options": [ "--airflow-constraints-location", "--airflow-constraints-mode", "--airflow-constraints-reference", - "--python-image", + ], + }, + { + "name": "Choosing dependencies and extras (for power users)", + "options": [ + "--airflow-extras", "--additional-airflow-extras", - "--additional-pip-install-flags", "--additional-python-deps", - "--additional-runtime-apt-deps", - "--additional-runtime-apt-env", - "--additional-runtime-apt-command", + "--dev-apt-deps", "--additional-dev-apt-deps", - "--additional-dev-apt-env", + "--dev-apt-command", "--additional-dev-apt-command", + "--additional-dev-apt-env", "--runtime-apt-deps", + "--additional-runtime-apt-deps", "--runtime-apt-command", - "--dev-apt-deps", - "--dev-apt-command", - "--version-suffix-for-pypi", - "--commit-sha", + "--additional-runtime-apt-command", + "--additional-runtime-apt-env", ], }, { "name": "Advanced customization options (for specific customization needs)", "options": [ - "--debian-version", + "--installation-method", + "--install-airflow-reference", "--install-packages-from-context", - "--use-constraints-for-context-packages", "--cleanup-context", + "--use-constraints-for-context-packages", + "--disable-airflow-repo-cache", "--disable-mysql-client-installation", "--disable-mssql-client-installation", "--disable-postgres-client-installation", - "--disable-airflow-repo-cache", - "--install-airflow-reference", - "--installation-method", ], }, { diff --git a/dev/breeze/src/airflow_breeze/params/build_ci_params.py b/dev/breeze/src/airflow_breeze/params/build_ci_params.py index a713c280b54d9..b29ebacff0ae3 100644 --- a/dev/breeze/src/airflow_breeze/params/build_ci_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_ci_params.py @@ -16,6 +16,7 @@ # under the License. from __future__ import annotations +import random from dataclasses import dataclass from pathlib import Path @@ -38,7 +39,7 @@ class BuildCiParams(CommonBuildParams): force_build: bool = False upgrade_to_newer_dependencies: bool = False upgrade_on_failure: bool = False - eager_upgrade_additional_requirements: str = "" + eager_upgrade_additional_requirements: str | None = None skip_provider_dependencies_check: bool = False @property @@ -49,72 +50,52 @@ def airflow_version(self): def image_type(self) -> str: return "CI" - @property - def extra_docker_build_flags(self) -> list[str]: - extra_ci_flags = [] - extra_ci_flags.extend( - ["--build-arg", f"AIRFLOW_CONSTRAINTS_REFERENCE={self.airflow_constraints_reference}"] - ) - if self.airflow_constraints_location: - extra_ci_flags.extend( - ["--build-arg", f"AIRFLOW_CONSTRAINTS_LOCATION={self.airflow_constraints_location}"] - ) - if self.upgrade_to_newer_dependencies: - eager_upgrade_arg = self.eager_upgrade_additional_requirements.strip().replace("\n", "") - if eager_upgrade_arg: - extra_ci_flags.extend( - [ - "--build-arg", - f"EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS={eager_upgrade_arg}", - ] - ) - return super().extra_docker_build_flags + extra_ci_flags - @property def md5sum_cache_dir(self) -> Path: return Path(BUILD_CACHE_DIR, self.airflow_branch, self.python, "CI") - @property - def required_image_args(self) -> list[str]: - return [ - "airflow_branch", - "airflow_constraints_mode", - "airflow_constraints_reference", - "airflow_extras", - "airflow_image_date_created", - "airflow_image_repository", - "airflow_pre_cached_pip_packages", - "airflow_version", - "build_id", - "constraints_github_repository", - "python_base_image", - "upgrade_to_newer_dependencies", - ] - - @property - def optional_image_args(self) -> list[str]: - return [ - "additional_airflow_extras", - "additional_dev_apt_command", - "additional_dev_apt_deps", - "additional_dev_apt_env", - "additional_pip_install_flags", - "additional_python_deps", - "additional_runtime_apt_command", - "additional_runtime_apt_deps", - "additional_runtime_apt_env", - "dev_apt_command", - "dev_apt_deps", - "additional_dev_apt_command", - "additional_dev_apt_deps", - "additional_dev_apt_env", - "additional_airflow_extras", - "additional_pip_install_flags", - "additional_python_deps", - "version_suffix_for_pypi", - "commit_sha", - "build_progress", - ] - - def __post_init__(self): - pass + def prepare_arguments_for_docker_build_command(self) -> list[str]: + self.build_arg_values: list[str] = [] + # Required build args + self._req_arg("AIRFLOW_BRANCH", self.airflow_branch) + self._req_arg("AIRFLOW_CONSTRAINTS_MODE", self.airflow_constraints_mode) + self._req_arg("AIRFLOW_CONSTRAINTS_REFERENCE", self.airflow_constraints_reference) + self._req_arg("AIRFLOW_EXTRAS", self.airflow_extras) + self._req_arg("AIRFLOW_IMAGE_DATE_CREATED", self.airflow_image_date_created) + self._req_arg("AIRFLOW_IMAGE_REPOSITORY", self.airflow_image_repository) + self._req_arg("AIRFLOW_PRE_CACHED_PIP_PACKAGES", self.airflow_pre_cached_pip_packages) + self._req_arg("AIRFLOW_VERSION", self.airflow_version) + self._req_arg("BUILD_ID", self.build_id) + self._req_arg("CONSTRAINTS_GITHUB_REPOSITORY", self.constraints_github_repository) + self._req_arg("PYTHON_BASE_IMAGE", self.python_base_image) + if self.upgrade_to_newer_dependencies: + self._opt_arg("UPGRADE_TO_NEWER_DEPENDENCIES", f"{random.randrange(2**32):x}") + if self.eager_upgrade_additional_requirements: + # in case eager upgrade additional requirements have EOL, connect them together + self._opt_arg( + "EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS", + self.eager_upgrade_additional_requirements.replace("\n", ""), + ) + # optional build args + self._opt_arg("AIRFLOW_CONSTRAINTS_LOCATION", self.airflow_constraints_location) + self._opt_arg("ADDITIONAL_AIRFLOW_EXTRAS", self.additional_airflow_extras) + self._opt_arg("ADDITIONAL_DEV_APT_COMMAND", self.additional_dev_apt_command) + self._opt_arg("ADDITIONAL_DEV_APT_DEPS", self.additional_dev_apt_deps) + self._opt_arg("ADDITIONAL_DEV_APT_ENV", self.additional_dev_apt_env) + self._opt_arg("ADDITIONAL_PIP_INSTALL_FLAGS", self.additional_pip_install_flags) + self._opt_arg("ADDITIONAL_PYTHON_DEPS", self.additional_python_deps) + self._opt_arg("DEV_APT_COMMAND", self.dev_apt_command) + self._opt_arg("DEV_APT_DEPS", self.dev_apt_deps) + self._opt_arg("ADDITIONAL_DEV_APT_COMMAND", self.additional_dev_apt_command) + self._opt_arg("ADDITIONAL_DEV_APT_DEPS", self.additional_dev_apt_deps) + self._opt_arg("ADDITIONAL_DEV_APT_ENV", self.additional_dev_apt_env) + self._opt_arg("ADDITIONAL_AIRFLOW_EXTRAS", self.additional_airflow_extras) + self._opt_arg("ADDITIONAL_PIP_INSTALL_FLAGS", self.additional_pip_install_flags) + self._opt_arg("ADDITIONAL_PYTHON_DEPS", self.additional_python_deps) + self._opt_arg("VERSION_SUFFIX_FOR_PYPI", self.version_suffix_for_pypi) + self._opt_arg("COMMIT_SHA", self.commit_sha) + self._opt_arg("BUILD_PROGRESS", self.build_progress) + # Convert to build args + build_args = self._to_build_args() + # Add cache directive + return build_args diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index 585c7dc0c9690..ed9a6b674106f 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -38,9 +38,9 @@ class BuildProdParams(CommonBuildParams): PROD build parameters. Those parameters are used to determine command issued to build PROD image. """ - additional_runtime_apt_command: str = "" - additional_runtime_apt_deps: str = "" - additional_runtime_apt_env: str = "" + additional_runtime_apt_command: str | None = None + additional_runtime_apt_deps: str | None = None + additional_runtime_apt_env: str | None = None airflow_constraints_mode: str = "constraints" airflow_constraints_reference: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH cleanup_context: bool = False @@ -49,13 +49,13 @@ class BuildProdParams(CommonBuildParams): disable_mssql_client_installation: bool = False disable_mysql_client_installation: bool = False disable_postgres_client_installation: bool = False - install_airflow_reference: str = "" - install_airflow_version: str = "" + install_airflow_reference: str | None = None + install_airflow_version: str | None = None install_packages_from_context: bool = False use_constraints_for_context_packages: bool = False installation_method: str = "." - runtime_apt_command: str = "" - runtime_apt_deps: str = "" + runtime_apt_command: str | None = None + runtime_apt_deps: str | None = None @property def airflow_version(self) -> str: @@ -110,24 +110,20 @@ def args_for_remote_install(self) -> list: self.airflow_branch_for_pypi_preloading = AIRFLOW_BRANCH return build_args - @property - def extra_docker_build_flags(self) -> list[str]: + def _extra_prod_docker_build_flags(self) -> list[str]: extra_build_flags = [] if self.install_airflow_reference: - AIRFLOW_INSTALLATION_METHOD = ( - "https://github.com/apache/airflow/archive/" - + self.install_airflow_reference - + ".tar.gz#egg=apache-airflow" - ) extra_build_flags.extend( [ "--build-arg", - AIRFLOW_INSTALLATION_METHOD, + "https://github.com/apache/airflow/archive/" + + self.install_airflow_reference + + ".tar.gz#egg=apache-airflow", ] ) extra_build_flags.extend(self.args_for_remote_install) elif self.install_airflow_version: - if not re.match(r"^[0-9\.]+((a|b|rc|alpha|beta|pre)[0-9]+)?$", self.install_airflow_version): + if not re.match(r"^[0-9.]+((a|b|rc|alpha|beta|pre)[0-9]+)?$", self.install_airflow_version): get_console().print( f"\n[error]ERROR: Bad value for install-airflow-version:{self.install_airflow_version}" ) @@ -175,7 +171,7 @@ def extra_docker_build_flags(self) -> list[str]: f"io.artifacthub.package.logo-url={logo_url}", ] ) - return super().extra_docker_build_flags + extra_build_flags + return extra_build_flags @property def airflow_pre_cached_pip_packages(self) -> str: @@ -201,46 +197,45 @@ def docker_context_files(self) -> str: def airflow_image_kubernetes(self) -> str: return f"{self.airflow_image_name}-kubernetes" - @property - def required_image_args(self) -> list[str]: - return [ - "airflow_branch", - "airflow_constraints_mode", - "airflow_extras", - "airflow_image_date_created", - "airflow_image_readme_url", - "airflow_image_repository", - "airflow_pre_cached_pip_packages", - "airflow_version", - "build_id", - "constraints_github_repository", - "docker_context_files", - "install_mssql_client", - "install_mysql_client", - "install_packages_from_context", - "install_postgres_client", - "install_providers_from_sources", - "python_base_image", - ] - - @property - def optional_image_args(self) -> list[str]: - return [ - "additional_airflow_extras", - "additional_dev_apt_command", - "additional_dev_apt_deps", - "additional_dev_apt_env", - "additional_pip_install_flags", - "additional_python_deps", - "additional_runtime_apt_command", - "additional_runtime_apt_deps", - "additional_runtime_apt_env", - "dev_apt_command", - "dev_apt_deps", - "runtime_apt_command", - "runtime_apt_deps", - "version_suffix_for_pypi", - "commit_sha", - "build_progress", - "use_constraints_for_context_packages", - ] + def prepare_arguments_for_docker_build_command(self) -> list[str]: + self.build_arg_values: list[str] = [] + # Required build args + self._req_arg("AIRFLOW_BRANCH", self.airflow_branch) + self._req_arg("AIRFLOW_CONSTRAINTS_MODE", self.airflow_constraints_mode) + self._req_arg("AIRFLOW_EXTRAS", self.airflow_extras) + self._req_arg("AIRFLOW_IMAGE_DATE_CREATED", self.airflow_image_date_created) + self._req_arg("AIRFLOW_IMAGE_README_URL", self.airflow_image_readme_url) + self._req_arg("AIRFLOW_IMAGE_REPOSITORY", self.airflow_image_repository) + self._req_arg("AIRFLOW_PRE_CACHED_PIP_PACKAGES", self.airflow_pre_cached_pip_packages) + self._req_arg("AIRFLOW_VERSION", self.airflow_version) + self._req_arg("BUILD_ID", self.build_id) + self._req_arg("CONSTRAINTS_GITHUB_REPOSITORY", self.constraints_github_repository) + self._req_arg("DOCKER_CONTEXT_FILES", self.docker_context_files) + self._req_arg("INSTALL_MSSQL_CLIENT", self.install_mssql_client) + self._req_arg("INSTALL_MYSQL_CLIENT", self.install_mysql_client) + self._req_arg("INSTALL_PACKAGES_FROM_CONTEXT", self.install_packages_from_context) + self._req_arg("INSTALL_POSTGRES_CLIENT", self.install_postgres_client) + self._req_arg("INSTALL_PROVIDERS_FROM_SOURCES", self.install_providers_from_sources) + self._req_arg("PYTHON_BASE_IMAGE", self.python_base_image) + # optional build args + self._opt_arg("AIRFLOW_CONSTRAINTS_LOCATION", self.airflow_constraints_location) + self._opt_arg("ADDITIONAL_AIRFLOW_EXTRAS", self.additional_airflow_extras) + self._opt_arg("ADDITIONAL_DEV_APT_COMMAND", self.additional_dev_apt_command) + self._opt_arg("ADDITIONAL_DEV_APT_DEPS", self.additional_dev_apt_deps) + self._opt_arg("ADDITIONAL_DEV_APT_ENV", self.additional_dev_apt_env) + self._opt_arg("ADDITIONAL_PIP_INSTALL_FLAGS", self.additional_pip_install_flags) + self._opt_arg("ADDITIONAL_PYTHON_DEPS", self.additional_python_deps) + self._opt_arg("ADDITIONAL_RUNTIME_APT_COMMAND", self.additional_runtime_apt_command) + self._opt_arg("ADDITIONAL_RUNTIME_APT_DEPS", self.additional_runtime_apt_deps) + self._opt_arg("ADDITIONAL_RUNTIME_APT_ENV", self.additional_runtime_apt_env) + self._opt_arg("DEV_APT_COMMAND", self.dev_apt_command) + self._opt_arg("DEV_APT_DEPS", self.dev_apt_deps) + self._opt_arg("RUNTIME_APT_COMMAND", self.runtime_apt_command) + self._opt_arg("RUNTIME_APT_DEPS", self.runtime_apt_deps) + self._opt_arg("VERSION_SUFFIX_FOR_PYPI", self.version_suffix_for_pypi) + self._opt_arg("COMMIT_SHA", self.commit_sha) + self._opt_arg("BUILD_PROGRESS", self.build_progress) + self._opt_arg("USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES", self.use_constraints_for_context_packages) + build_args = self._to_build_args() + build_args.extend(self._extra_prod_docker_build_flags()) + return build_args diff --git a/dev/breeze/src/airflow_breeze/params/common_build_params.py b/dev/breeze/src/airflow_breeze/params/common_build_params.py index 782d01bca44b1..29e748f8c8171 100644 --- a/dev/breeze/src/airflow_breeze/params/common_build_params.py +++ b/dev/breeze/src/airflow_breeze/params/common_build_params.py @@ -18,8 +18,9 @@ import os import sys -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime +from typing import Any from airflow_breeze.branch_defaults import AIRFLOW_BRANCH, DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH from airflow_breeze.global_constants import ( @@ -37,24 +38,23 @@ class CommonBuildParams: Common build parameters. Those parameters are common parameters for CI And PROD build. """ - additional_airflow_extras: str = "" - additional_dev_apt_command: str = "" - additional_dev_apt_deps: str = "" - additional_dev_apt_env: str = "" - additional_python_deps: str = "" - additional_pip_install_flags: str = "" + additional_airflow_extras: str | None = None + additional_dev_apt_command: str | None = None + additional_dev_apt_deps: str | None = None + additional_dev_apt_env: str | None = None + additional_python_deps: str | None = None + additional_pip_install_flags: str | None = None airflow_branch: str = os.environ.get("DEFAULT_BRANCH", AIRFLOW_BRANCH) default_constraints_branch: str = os.environ.get( "DEFAULT_CONSTRAINTS_BRANCH", DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH ) - airflow_constraints_location: str = "" - build_id: int = 0 + airflow_constraints_location: str | None = None builder: str = "autodetect" build_progress: str = ALLOWED_BUILD_PROGRESS[0] constraints_github_repository: str = APACHE_AIRFLOW_GITHUB_REPOSITORY - commit_sha: str = "" - dev_apt_command: str = "" - dev_apt_deps: str = "" + commit_sha: str | None = None + dev_apt_command: str | None = None + dev_apt_deps: str | None = None docker_cache: str = "registry" github_actions: str = os.environ.get("GITHUB_ACTIONS", "false") github_repository: str = APACHE_AIRFLOW_GITHUB_REPOSITORY @@ -68,14 +68,19 @@ class CommonBuildParams: python: str = "3.8" tag_as_latest: bool = False dry_run: bool = False - version_suffix_for_pypi: str = "" + version_suffix_for_pypi: str | None = None verbose: bool = False debian_version: str = "bookworm" + build_arg_values: list[str] = field(default_factory=list) @property def airflow_version(self): raise NotImplementedError() + @property + def build_id(self) -> str: + return os.environ.get("CI_BUILD_ID", "0") + @property def image_type(self) -> str: raise NotImplementedError() @@ -99,23 +104,14 @@ def airflow_image_name(self): return image @property - def extra_docker_build_flags(self) -> list[str]: - extra_flass = [] - if self.build_progress: - extra_flass.append(f"--progress={self.build_progress}") - return extra_flass - - @property - def docker_cache_directive(self) -> list[str]: - docker_cache_directive = [] + def common_docker_build_flags(self) -> list[str]: + extra_flags = [] + extra_flags.append(f"--progress={self.build_progress}") if self.docker_cache == "registry": - for platform in self.platforms: - docker_cache_directive.append(f"--cache-from={self.get_cache(platform)}") + extra_flags.append(f"--cache-from={self.get_cache(self.platform)}") elif self.docker_cache == "disabled": - docker_cache_directive.append("--no-cache") - else: - docker_cache_directive = [] - return docker_cache_directive + extra_flags.append("--no-cache") + return extra_flags @property def python_base_image(self): @@ -169,13 +165,31 @@ def preparing_latest_image(self) -> bool: def platforms(self) -> list[str]: return self.platform.split(",") - @property - def required_image_args(self) -> list[str]: - raise NotImplementedError() + def _build_arg(self, name: str, value: Any, optional: bool): + if value is None or "": + if optional: + return + else: + raise ValueError(f"Value for {name} cannot be empty or None") + if value is True: + str_value = "true" + elif value is False: + str_value = "false" + else: + str_value = str(value) if value is not None else "" + self.build_arg_values.append(f"{name}={str_value}") - @property - def optional_image_args(self) -> list[str]: + def _req_arg(self, name: str, value: Any): + self._build_arg(name, value, False) + + def _opt_arg(self, name: str, value: Any): + self._build_arg(name, value, True) + + def prepare_arguments_for_docker_build_command(self) -> list[str]: raise NotImplementedError() - def __post_init__(self): - pass + def _to_build_args(self): + build_args = [] + for arg in self.build_arg_values: + build_args.extend(["--build-arg", arg]) + return build_args diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 2a001f2757720..249faaafd1513 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -293,7 +293,7 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, help="When set, attempt to run upgrade to newer dependencies when regular build fails.", envvar="UPGRADE_ON_FAILURE", ) -option_additional_extras = click.option( +option_additional_airflow_extras = click.option( "--additional-airflow-extras", help="Additional extra package while installing Airflow in the image.", envvar="ADDITIONAL_AIRFLOW_EXTRAS", @@ -413,7 +413,13 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, option_version_suffix_for_pypi = click.option( "--version-suffix-for-pypi", help="Version suffix used for PyPI packages (alpha, beta, rc1, etc.).", - default="", + envvar="VERSION_SUFFIX_FOR_PYPI", +) +option_version_suffix_for_pypi_ci = click.option( + "--version-suffix-for-pypi", + help="Version suffix used for PyPI packages (alpha, beta, rc1, etc.).", + default="dev0", + show_default=True, envvar="VERSION_SUFFIX_FOR_PYPI", ) option_package_format = click.option( @@ -478,7 +484,6 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, option_airflow_constraints_location = click.option( "--airflow-constraints-location", type=str, - default="", help="If specified, it is used instead of calculating reference to the constraint file. " "It could be full remote URL to the location file, or local file placed in `docker-context-files` " "(in this case it has to start with /opt/airflow/docker-context-files).", @@ -615,7 +620,6 @@ def _set_default_from_parent(ctx: click.core.Context, option: click.core.Option, ) option_commit_sha = click.option( "--commit-sha", - default=None, show_default=True, envvar="COMMIT_SHA", help="Commit SHA that is used to build the images.", diff --git a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py index 36c976b2fed4a..78a8d69754ba0 100644 --- a/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/docker_command_utils.py @@ -388,35 +388,6 @@ def get_env_variable_value(arg_name: str, params: CommonBuildParams | ShellParam return value -def prepare_arguments_for_docker_build_command(image_params: CommonBuildParams) -> list[str]: - """ - Constructs docker compose command arguments list based on parameters passed. Maps arguments to - argument values. - - It maps: - * all the truthy/falsy values are converted to "true" / "false" respectively - * if upgrade_to_newer_dependencies is set to True, it is replaced by a random string to account - for the need of always triggering upgrade for docker build. - - :param image_params: parameters of the image - :return: list of `--build-arg` commands to use for the parameters passed - """ - - args_command = [] - for required_arg in image_params.required_image_args: - args_command.append("--build-arg") - args_command.append( - required_arg.upper() + "=" + get_env_variable_value(arg_name=required_arg, params=image_params) - ) - for optional_arg in image_params.optional_image_args: - param_value = get_env_variable_value(optional_arg, params=image_params) - if param_value: - args_command.append("--build-arg") - args_command.append(optional_arg.upper() + "=" + param_value) - args_command.extend(image_params.docker_cache_directive) - return args_command - - def prepare_docker_build_cache_command( image_params: CommonBuildParams, ) -> list[str]: @@ -427,16 +398,14 @@ def prepare_docker_build_cache_command( :return: Command to run as list of string """ - arguments = prepare_arguments_for_docker_build_command(image_params) - build_flags = image_params.extra_docker_build_flags final_command = [] final_command.extend(["docker"]) final_command.extend( ["buildx", "build", "--builder", get_and_use_docker_context(image_params.builder), "--progress=auto"] ) - final_command.extend(build_flags) + final_command.extend(image_params.common_docker_build_flags) final_command.extend(["--pull"]) - final_command.extend(arguments) + final_command.extend(image_params.prepare_arguments_for_docker_build_command()) final_command.extend(["--target", "main", "."]) final_command.extend( ["-f", "Dockerfile" if isinstance(image_params, BuildProdParams) else "Dockerfile.ci"] @@ -470,7 +439,6 @@ def prepare_base_build_command(image_params: CommonBuildParams) -> list[str]: "build", "--builder", get_and_use_docker_context(image_params.builder), - "--progress=auto", "--push" if image_params.push else "--load", ] ) @@ -488,17 +456,15 @@ def prepare_docker_build_command( :return: Command to run as list of string """ - arguments = prepare_arguments_for_docker_build_command(image_params) build_command = prepare_base_build_command( image_params=image_params, ) - build_flags = image_params.extra_docker_build_flags final_command = [] final_command.extend(["docker"]) final_command.extend(build_command) - final_command.extend(build_flags) + final_command.extend(image_params.common_docker_build_flags) final_command.extend(["--pull"]) - final_command.extend(arguments) + final_command.extend(image_params.prepare_arguments_for_docker_build_command()) final_command.extend(["-t", image_params.airflow_image_name_with_tag, "--target", "main", "."]) final_command.extend( ["-f", "Dockerfile" if isinstance(image_params, BuildProdParams) else "Dockerfile.ci"] diff --git a/images/breeze/output_ci-image_build.svg b/images/breeze/output_ci-image_build.svg index 2a7147c2fd763..57fb8ac402e76 100644 --- a/images/breeze/output_ci-image_build.svg +++ b/images/breeze/output_ci-image_build.svg @@ -1,4 +1,4 @@ - +