Remove from __future__ from airflow/providers __init__.py (ap…

…ache#43173)

Cleans-up airflow and providers `__init__.py" files in order to
get providers import work again.

This is done by excluding the two `__init__.py` files from
automated ruff isort rules adding `from __future__ import annotations`.

That should finally get rid of the Intellij teething import
problem that has been introduced in apache#42505.

There were earlier - unsuccessful - attempts to fix it in
the apache#43116 and apache#43081 that followed apache#42951 - but the key is that Pycharm
requires the namespace's extend_path to be first "real" line
of code in the `__init__.py` to understand that the package
is an "explicit" namespace package - and it conflicts with
the requirement of "from __future__ import annotations" to be
the first line of Python code.

Also this PR fixes following problem:

* pytest_plugin expecting .asf.yml in "airflow" sources - even during
  compatibility tests with older version of airflow (where the
  .asf.yml is not present)

---------

Co-authored-by: Kaxil Naik <[email protected]>

airflow/__init__.py

@@ -15,7 +15,10 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from __future__ import annotations
+
+# We do not use "from __future__ import annotations" here because it is not supported
+# by Pycharm when we want to make sure all imports in airflow work from namespace packages
+# Adding it automatically is excluded in pyproject.toml via I002 ruff rule exclusion
 
 # Make `airflow` a namespace package, supporting installing
 # airflow.providers.* in different locations (i.e. one in site, and one in user

providers/src/airflow/providers/__init__.py

@@ -16,7 +16,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
-from __future__ import annotations
+# We do not use "from __future__ import annotations" here because it is not supported
+# by Pycharm when we want to make sure all imports in airflow work from namespace packages
+# Adding it automatically is excluded in pyproject.toml via I002 ruff rule exclusion
 
 # Make `airflow` a namespace package, supporting installing
 # airflow.providers.* in different locations (i.e. one in site, and one in user

pyproject.toml

@@ -346,9 +346,10 @@ section-order = [
 testing = ["dev", "providers.tests", "task_sdk.tests", "tests_common", "tests"]
 
 [tool.ruff.lint.extend-per-file-ignores]
-"airflow/__init__.py" = ["F401", "TCH004"]
+"airflow/__init__.py" = ["F401", "TCH004", "I002"]
 "airflow/models/__init__.py" = ["F401", "TCH004"]
 "airflow/models/sqla_models.py" = ["F401"]
+"providers/src/airflow/providers/__init__.py" = ["I002"]
 
 # The test_python.py is needed because adding __future__.annotations breaks runtime checks that are
 # needed for the test to work

tests_common/pytest_plugin.py

@@ -371,21 +371,24 @@ def initialize_airflow_tests(request):
 def pytest_configure(config: pytest.Config) -> None:
     # Ensure that the airflow sources dir is at the end of the sys path if it's not already there. Needed to
     # run import from `providers/tests/`
-    desired = AIRFLOW_SOURCES_ROOT_DIR.as_posix()
-    for path in sys.path:
-        if path == desired:
-            break
-    else:
-        # This "desired" path should be the Airflow source directory (repo root)
-        assert (AIRFLOW_SOURCES_ROOT_DIR / ".asf.yaml").exists(), f"Path {desired} is not Airflow root"
-        sys.path.append(desired)
-
-    if (backend := config.getoption("backend", default=None)) and backend not in SUPPORTED_DB_BACKENDS:
-        msg = (
-            f"Provided DB backend {backend!r} not supported, "
-            f"expected one of: {', '.join(map(repr, SUPPORTED_DB_BACKENDS))}"
-        )
-        pytest.exit(msg, returncode=6)
+    if os.environ.get("USE_AIRFLOW_VERSION") == "":
+        # if USE_AIRFLOW_VERSION is not empty, we are running tests against the installed version of Airflow
+        # and providers so there is no need to add the sources directory to the path
+        desired = AIRFLOW_SOURCES_ROOT_DIR.as_posix()
+        for path in sys.path:
+            if path == desired:
+                break
+        else:
+            # This "desired" path should be the Airflow source directory (repo root)
+            assert (AIRFLOW_SOURCES_ROOT_DIR / ".asf.yaml").exists(), f"Path {desired} is not Airflow root"
+            sys.path.append(desired)
+
+        if (backend := config.getoption("backend", default=None)) and backend not in SUPPORTED_DB_BACKENDS:
+            msg = (
+                f"Provided DB backend {backend!r} not supported, "
+                f"expected one of: {', '.join(map(repr, SUPPORTED_DB_BACKENDS))}"
+            )
+            pytest.exit(msg, returncode=6)
 
     config.addinivalue_line("markers", "integration(name): mark test to run with named integration")
     config.addinivalue_line("markers", "backend(name): mark test to run with named backend")