Move tests_common from "dev" to top-level. (apache#42985)

* Move tests common without changes

* Fix docstrings in tests_common

* Move tests_common from "dev" to top-level.

Follow-up after apache#42505 fixing teething problem with tests_common.

Originally in apache#42505 common test code was moved to "dev" folder, but
the "dev" folder is really dedicated to "build" scripts and the
problem with moving "tests_common" to the folder was that the
whole "dev" folder is replaced (for non-committer PRs) with
the content from the target branch.

This is done for security reasons, because we can accidentally
use any of the scripts from dev in the CI build scripts and we
might not notice, which will open us to a security issue where
a file in "dev" coming from PR could be accidentally executed
during the "pull_request_target" workflow - which would expose
our secrets and GitHub Package write permissions to a
contributor coming from a fork.

This change moves the files, fixes pre-commit specification and
docs, also fixes a number of "doc" issues detected by "ruff" in
the tests_common folder as they were detected after the move.
The tests_common folder is added to folders mounted when
breeze is executed with local folders mounted (in order to
avoid accidental mounting of randomly generated files to
inside the breeze container).

All imports for the common tests were updated to reflect this
move.

.dockerignore

@@ -42,6 +42,7 @@
 
 # Add tests and kubernetes_tests to context.
 !tests
+!tests_common
 !kubernetes_tests
 !helm_tests
 !docker_tests

.pre-commit-config.yaml

@@ -364,7 +364,7 @@ repos:
         args: []
         require_serial: true
         additional_dependencies: ["ruff==0.5.5"]
-        exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py|^airflow/contrib/
+        exclude: ^.*/.*_vendor/|^tests/dags/test_imports.py$
       - id: replace-bad-characters
         name: Replace bad characters
         entry: ./scripts/ci/pre_commit/replace_bad_characters.py
@@ -736,7 +736,7 @@ repos:
         name: Verify usage of Airflow deprecation classes in core
         entry: category=DeprecationWarning|category=PendingDeprecationWarning
         files: \.py$
-        exclude: ^airflow/configuration\.py$|^providers/src/airflow/providers/|^scripts/in_container/verify_providers\.py$|^(providers/)?tests/.*$|^dev/tests_common/
+        exclude: ^airflow/configuration\.py$|^providers/src/airflow/providers/|^scripts/in_container/verify_providers\.py$|^(providers/)?tests/.*$|^tests_common/
         pass_filenames: true
       - id: check-provide-create-sessions-imports
         language: pygrep
@@ -1184,7 +1184,9 @@ repos:
           ^(providers/)?tests/ |
           ^dev/.*\.py$ |
           ^scripts/.*\.py$ |
-          ^\w+_tests/ |
+          ^docker_tests/.*$ |
+          ^helm_tests/.*$ |
+          ^tests_common/.*$ |
           ^docs/.*\.py$ |
           ^hatch_build.py$
       - id: check-provider-docs-valid

Dockerfile.ci

@@ -1369,6 +1369,7 @@ COPY pyproject.toml ${AIRFLOW_SOURCES}/pyproject.toml
 COPY providers/pyproject.toml ${AIRFLOW_SOURCES}/providers/pyproject.toml
 COPY task_sdk/pyproject.toml ${AIRFLOW_SOURCES}/task_sdk/pyproject.toml
 COPY airflow/__init__.py ${AIRFLOW_SOURCES}/airflow/
+COPY tests_common/ ${AIRFLOW_SOURCES}/tests_common/
 COPY generated/* ${AIRFLOW_SOURCES}/generated/
 COPY constraints/* ${AIRFLOW_SOURCES}/constraints/
 COPY LICENSE ${AIRFLOW_SOURCES}/LICENSE

contributing-docs/testing/system_tests.rst

@@ -35,7 +35,7 @@ Airflow system tests are pretty special because they serve three purposes:
 Old System Tests
 ----------------
 
-The system tests derive from the ``dev.tests_common.test_utils.system_test_class.SystemTests`` class.
+The system tests derive from the ``tests_common.test_utils.system_test_class.SystemTests`` class.
 
 Old versions of System tests should also be marked with ``@pytest.marker.system(SYSTEM)`` where ``system``
 designates the system to be tested (for example, ``google.cloud``). These tests are skipped by default.

contributing-docs/testing/unit_tests.rst

@@ -1171,7 +1171,7 @@ are not part of the public API. We deal with it in one of the following ways:
 1) If the whole provider is supposed to only work for later airflow version, we remove the whole provider
    by excluding it from compatibility test configuration (see below)
 
-2) Some compatibility shims are defined in ``dev.tests_common.test_utils/compat.py`` - and they can be used to make the
+2) Some compatibility shims are defined in ``tests_common.test_utils/compat.py`` - and they can be used to make the
    tests compatible - for example importing ``ParseImportError`` after the exception has been renamed from
    ``ImportError`` and it would fail in Airflow 2.9, but we have a fallback import in ``compat.py`` that
    falls back to old import automatically, so all tests testing / expecting ``ParseImportError`` should import
@@ -1184,7 +1184,7 @@ are not part of the public API. We deal with it in one of the following ways:
 
 .. code-block:: python
 
-  from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS
+  from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS
 
 
   @pytest.mark.skipif(not AIRFLOW_V_2_8_PLUS, reason="The tests should be skipped for Airflow < 2.8")
@@ -1197,7 +1197,7 @@ are not part of the public API. We deal with it in one of the following ways:
 
 .. code-block:: python
 
-  from dev.tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES
+  from tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES
 
 
   @pytest.mark.skipif(

dev/breeze/src/airflow_breeze/utils/docker_command_utils.py

@@ -97,6 +97,7 @@
     ("pyproject.toml", "/opt/airflow/pyproject.toml"),
     ("scripts", "/opt/airflow/scripts"),
     ("scripts/docker/entrypoint_ci.sh", "/entrypoint"),
+    ("tests_common", "/opt/airflow/tests_common"),
     ("tests", "/opt/airflow/tests"),
     ("helm_tests", "/opt/airflow/helm_tests"),
     ("kubernetes_tests", "/opt/airflow/kubernetes_tests"),

dev/breeze/src/airflow_breeze/utils/selective_checks.py

@@ -245,7 +245,7 @@ def __hash__(self):
         ],
         FileGroupForCi.TESTS_UTILS_FILES: [
             r"^tests/utils/",
-            r"^dev/tests_common/.*\.py$",
+            r"^tests_common/.*\.py$",
         ],
         FileGroupForCi.TASK_SDK_FILES: [
             r"^task_sdk/src/airflow/sdk/.*\.py$",

dev/breeze/tests/test_selective_checks.py

@@ -778,7 +778,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
         ),
         (
             pytest.param(
-                ("dev/tests_common/__init__.py",),
+                ("tests_common/__init__.py",),
                 {
                     "affected-providers-list-as-string": ALL_PROVIDERS_AFFECTED,
                     "all-python-versions": "['3.9']",
@@ -799,7 +799,7 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str):
                     "needs-mypy": "true",
                     "mypy-folders": "['airflow', 'providers', 'docs', 'dev']",
                 },
-                id="All tests should be run when dev/tests_common/ change",
+                id="All tests should be run when tests_common/ change",
             )
         ),
     ],

dev/perf/scheduler_dag_execution_timing.py

@@ -107,7 +107,7 @@ def get_executor_under_test(dotted_path):
     from airflow.executors.executor_loader import ExecutorLoader
 
     if dotted_path == "MockExecutor":
-        from dev.tests_common.test_utils.mock_executor import MockExecutor as executor
+        from tests_common.test_utils.mock_executor import MockExecutor as executor
 
     else:
         executor = ExecutorLoader.load_executor(dotted_path)

helm_tests/airflow_aux/test_pod_template_file.py

@@ -24,6 +24,7 @@
 import pytest
 
 from helm_tests.airflow_aux.test_container_lifecycle import CONTAINER_LIFECYCLE_PARAMETERS
+
 from tests.charts.helm_template_generator import render_chart
 
 

providers/tests/alibaba/cloud/log/test_oss_task_handler.py

@@ -22,14 +22,13 @@
 from unittest.mock import PropertyMock
 
 import pytest
+from tests_common.test_utils.config import conf_vars
+from tests_common.test_utils.db import clear_db_dags, clear_db_runs
 
 from airflow.providers.alibaba.cloud.log.oss_task_handler import OSSTaskHandler
 from airflow.utils.state import TaskInstanceState
 from airflow.utils.timezone import datetime
 
-from dev.tests_common.test_utils.config import conf_vars
-from dev.tests_common.test_utils.db import clear_db_dags, clear_db_runs
-
 pytestmark = pytest.mark.db_test
 
 OSS_TASK_HANDLER_STRING = "airflow.providers.alibaba.cloud.log.oss_task_handler.{}"

providers/tests/amazon/aws/auth_manager/avp/test_facade.py

@@ -21,15 +21,14 @@
 from unittest.mock import Mock
 
 import pytest
+from tests_common.test_utils.config import conf_vars
 
 from airflow.exceptions import AirflowException
 from airflow.providers.amazon.aws.auth_manager.avp.entities import AvpEntities, get_action_id, get_entity_type
 from airflow.providers.amazon.aws.auth_manager.avp.facade import AwsAuthManagerAmazonVerifiedPermissionsFacade
 from airflow.providers.amazon.aws.auth_manager.user import AwsAuthManagerUser
 from airflow.utils.helpers import prune_dict
 
-from dev.tests_common.test_utils.config import conf_vars
-
 if TYPE_CHECKING:
     from airflow.auth.managers.base_auth_manager import ResourceMethod
 

providers/tests/amazon/aws/auth_manager/cli/test_avp_commands.py

@@ -20,13 +20,12 @@
 from unittest.mock import ANY, Mock, patch
 
 import pytest
+from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS
+from tests_common.test_utils.config import conf_vars
 
 from airflow.cli import cli_parser
 from airflow.providers.amazon.aws.auth_manager.cli.avp_commands import init_avp, update_schema
 
-from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS
-from dev.tests_common.test_utils.config import conf_vars
-
 mock_boto3 = Mock()
 
 pytestmark = [

providers/tests/amazon/aws/auth_manager/security_manager/test_aws_security_manager_override.py

@@ -20,8 +20,7 @@
 
 import pytest
 from flask import Flask
-
-from dev.tests_common.test_utils.compat import ignore_provider_compatibility_error
+from tests_common.test_utils.compat import ignore_provider_compatibility_error
 
 python3_saml = pytest.importorskip("python3-saml")
 

providers/tests/amazon/aws/auth_manager/test_aws_auth_manager.py

@@ -22,6 +22,9 @@
 import pytest
 from flask import Flask, session
 from flask_appbuilder.menu import MenuItem
+from tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_2_9_PLUS
+from tests_common.test_utils.config import conf_vars
+from tests_common.test_utils.www import check_content_in_response
 
 from airflow.providers.amazon.aws.auth_manager.avp.entities import AvpEntities
 from airflow.providers.amazon.aws.auth_manager.avp.facade import AwsAuthManagerAmazonVerifiedPermissionsFacade
@@ -39,10 +42,6 @@
 from airflow.www import app as application
 from airflow.www.extensions.init_appbuilder import init_appbuilder
 
-from dev.tests_common.test_utils.compat import AIRFLOW_V_2_8_PLUS, AIRFLOW_V_2_9_PLUS
-from dev.tests_common.test_utils.config import conf_vars
-from dev.tests_common.test_utils.www import check_content_in_response
-
 try:
     from airflow.auth.managers.models.resource_details import (
         AccessView,

providers/tests/amazon/aws/auth_manager/views/test_auth.py

@@ -20,13 +20,12 @@
 
 import pytest
 from flask import session, url_for
+from tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS
+from tests_common.test_utils.config import conf_vars
 
 from airflow.exceptions import AirflowException
 from airflow.www import app as application
 
-from dev.tests_common.test_utils.compat import AIRFLOW_V_2_9_PLUS
-from dev.tests_common.test_utils.config import conf_vars
-
 pytestmark = [
     pytest.mark.skipif(not AIRFLOW_V_2_9_PLUS, reason="Test requires Airflow 2.9+"),
     pytest.mark.skip_if_database_isolation_mode,

providers/tests/amazon/aws/executors/batch/test_batch_executor.py

@@ -26,6 +26,8 @@
 import yaml
 from botocore.exceptions import ClientError, NoCredentialsError
 from semver import VersionInfo
+from tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES
+from tests_common.test_utils.config import conf_vars
 
 from airflow.exceptions import AirflowException
 from airflow.executors.base_executor import BaseExecutor
@@ -46,9 +48,6 @@
 from airflow.utils.state import State
 from airflow.version import version as airflow_version_str
 
-from dev.tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES
-from dev.tests_common.test_utils.config import conf_vars
-
 airflow_version = VersionInfo(*map(int, airflow_version_str.split(".")[:3]))
 ARN1 = "arn1"
 

providers/tests/amazon/aws/executors/ecs/test_ecs_executor.py

@@ -32,6 +32,9 @@
 from botocore.exceptions import ClientError
 from inflection import camelize
 from semver import VersionInfo
+from tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES
+from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS
+from tests_common.test_utils.config import conf_vars
 
 from airflow.exceptions import AirflowException
 from airflow.executors.base_executor import BaseExecutor
@@ -57,10 +60,6 @@
 from airflow.utils.timezone import utcnow
 from airflow.version import version as airflow_version_str
 
-from dev.tests_common import RUNNING_TESTS_AGAINST_AIRFLOW_PACKAGES
-from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS
-from dev.tests_common.test_utils.config import conf_vars
-
 pytestmark = pytest.mark.db_test
 
 airflow_version = VersionInfo(*map(int, airflow_version_str.split(".")[:3]))

providers/tests/amazon/aws/hooks/test_base_aws.py

@@ -38,6 +38,7 @@
 from botocore.utils import FileWebIdentityTokenLoader
 from moto import mock_aws
 from moto.core import DEFAULT_ACCOUNT_ID
+from tests_common.test_utils.config import conf_vars
 
 from airflow.exceptions import AirflowException
 from airflow.models.connection import Connection
@@ -50,8 +51,6 @@
 )
 from airflow.providers.amazon.aws.utils.connection_wrapper import AwsConnectionWrapper
 
-from dev.tests_common.test_utils.config import conf_vars
-
 pytest.importorskip("aiobotocore")
 
 MOCK_AWS_CONN_ID = "mock-conn-id"

providers/tests/amazon/aws/hooks/test_s3.py

@@ -30,6 +30,7 @@
 import pytest
 from botocore.exceptions import ClientError
 from moto import mock_aws
+from tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS
 
 from airflow.exceptions import AirflowException
 from airflow.models import Connection
@@ -43,8 +44,6 @@
 )
 from airflow.utils.timezone import datetime
 
-from dev.tests_common.test_utils.compat import AIRFLOW_V_2_10_PLUS
-
 
 @pytest.fixture
 def mocked_s3_res():

providers/tests/amazon/aws/links/test_base_aws.py

@@ -21,14 +21,13 @@
 from unittest import mock
 
 import pytest
+from tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS
+from tests_common.test_utils.mock_operators import MockOperator
 
 from airflow.models.xcom import XCom
 from airflow.providers.amazon.aws.links.base_aws import BaseAwsLink
 from airflow.serialization.serialized_objects import SerializedDAG
 
-from dev.tests_common.test_utils.compat import AIRFLOW_V_3_0_PLUS
-from dev.tests_common.test_utils.mock_operators import MockOperator
-
 if TYPE_CHECKING:
     from airflow.models.taskinstance import TaskInstance
 

providers/tests/amazon/aws/log/test_cloudwatch_task_handler.py

@@ -26,6 +26,7 @@
 import boto3
 import pytest
 from moto import mock_aws
+from tests_common.test_utils.config import conf_vars
 from watchtower import CloudWatchLogHandler
 
 from airflow.models import DAG, DagRun, TaskInstance
@@ -36,8 +37,6 @@
 from airflow.utils.state import State
 from airflow.utils.timezone import datetime
 
-from dev.tests_common.test_utils.config import conf_vars
-
 
 def get_time_str(time_in_milliseconds):
     dt_time = dt.fromtimestamp(time_in_milliseconds / 1000.0, tz=timezone.utc)

providers/tests/amazon/aws/log/test_s3_task_handler.py

@@ -26,6 +26,7 @@
 import pytest
 from botocore.exceptions import ClientError
 from moto import mock_aws
+from tests_common.test_utils.config import conf_vars
 
 from airflow.models import DAG, DagRun, TaskInstance
 from airflow.operators.empty import EmptyOperator
@@ -34,8 +35,6 @@
 from airflow.utils.state import State, TaskInstanceState
 from airflow.utils.timezone import datetime
 
-from dev.tests_common.test_utils.config import conf_vars
-
 
 @pytest.fixture(autouse=True)
 def s3mock():

providers/tests/amazon/aws/secrets/test_systems_manager.py

@@ -21,12 +21,11 @@
 
 import pytest
 from moto import mock_aws
+from tests_common.test_utils.config import conf_vars
 
 from airflow.configuration import initialize_secrets_backends
 from airflow.providers.amazon.aws.secrets.systems_manager import SystemsManagerParameterStoreBackend
 
-from dev.tests_common.test_utils.config import conf_vars
-
 URI_CONNECTION = pytest.param(
     "postgres://my-login:my-pass@my-host:5432/my-schema?param1=val1&param2=val2", id="uri-connection"
 )

providers/tests/amazon/aws/transfers/test_redshift_to_s3.py

@@ -22,14 +22,13 @@
 
 import pytest
 from boto3.session import Session
+from tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces
 
 from airflow.exceptions import AirflowException
 from airflow.models.connection import Connection
 from airflow.providers.amazon.aws.transfers.redshift_to_s3 import RedshiftToS3Operator
 from airflow.providers.amazon.aws.utils.redshift import build_credentials_block
 
-from dev.tests_common.test_utils.asserts import assert_equal_ignore_multiple_spaces
-
 
 class TestRedshiftToS3Transfer:
     @pytest.mark.parametrize("table_as_file_name, expected_s3_key", [[True, "key/table_"], [False, "key"]])