From 9eb7249e56971827092e67103dfc601c1861214b Mon Sep 17 00:00:00 2001 From: Shubham Raj <48172486+shubhamraj-git@users.noreply.github.com> Date: Sun, 22 Dec 2024 22:54:14 +0530 Subject: [PATCH 1/6] [providers-fab/v1-5] Invalidate user session on password reset (#45139) * session expire on pass change * fix statis checks * add tests (cherry picked from commit cf401c48bb6d06f1b30fef59d2a07afab22118bc) Co-authored-by: Shubham Raj <48172486+shubhamraj-git@users.noreply.github.com> --- .../core_api/openapi/v1-generated.yaml | 2 +- .../ui/openapi-gen/requests/schemas.gen.ts | 2 +- .../fab/auth_manager/cli_commands/utils.py | 12 +++++ .../auth_manager/security_manager/override.py | 1 + .../auth_manager/cli_commands/test_utils.py | 54 ++++++++++++++++++- 5 files changed, 67 insertions(+), 4 deletions(-) diff --git a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml index 890fb6b6c8d0a..0e44f22c8876b 100644 --- a/airflow/api_fastapi/core_api/openapi/v1-generated.yaml +++ b/airflow/api_fastapi/core_api/openapi/v1-generated.yaml @@ -5949,7 +5949,7 @@ components: properties: __type: type: string - title: ' Type' + title: Type default: TimeDelta days: type: integer diff --git a/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow/ui/openapi-gen/requests/schemas.gen.ts index 8a8a50bb7437a..bdf55b7241c1e 100644 --- a/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -3649,7 +3649,7 @@ export const $TimeDelta = { properties: { __type: { type: "string", - title: " Type", + title: "Type", default: "TimeDelta", }, days: { diff --git a/providers/src/airflow/providers/fab/auth_manager/cli_commands/utils.py b/providers/src/airflow/providers/fab/auth_manager/cli_commands/utils.py index 78403e24079f1..30963f5cf1fdd 100644 --- a/providers/src/airflow/providers/fab/auth_manager/cli_commands/utils.py +++ b/providers/src/airflow/providers/fab/auth_manager/cli_commands/utils.py @@ -26,7 +26,10 @@ import airflow from airflow.configuration import conf +from airflow.exceptions import AirflowConfigException +from airflow.www.app import isabs, make_url from airflow.www.extensions.init_appbuilder import init_appbuilder +from airflow.www.extensions.init_session import init_airflow_session_interface from airflow.www.extensions.init_views import init_plugins if TYPE_CHECKING: @@ -38,6 +41,7 @@ def _return_appbuilder(app: Flask) -> AirflowAppBuilder: """Return an appbuilder instance for the given app.""" init_appbuilder(app) init_plugins(app) + init_airflow_session_interface(app) return app.appbuilder # type: ignore[attr-defined] @@ -49,4 +53,12 @@ def get_application_builder() -> Generator[AirflowAppBuilder, None, None]: with flask_app.app_context(): # Enable customizations in webserver_config.py to be applied via Flask.current_app. flask_app.config.from_pyfile(webserver_config, silent=True) + flask_app.config["SQLALCHEMY_DATABASE_URI"] = conf.get("database", "SQL_ALCHEMY_CONN") + url = make_url(flask_app.config["SQLALCHEMY_DATABASE_URI"]) + if url.drivername == "sqlite" and url.database and not isabs(url.database): + raise AirflowConfigException( + f'Cannot use relative path: `{conf.get("database", "SQL_ALCHEMY_CONN")}` to connect to sqlite. ' + "Please use absolute path such as `sqlite:////tmp/airflow.db`." + ) + flask_app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = False yield _return_appbuilder(flask_app) diff --git a/providers/src/airflow/providers/fab/auth_manager/security_manager/override.py b/providers/src/airflow/providers/fab/auth_manager/security_manager/override.py index 9c49845b42066..1592bfb2cb381 100644 --- a/providers/src/airflow/providers/fab/auth_manager/security_manager/override.py +++ b/providers/src/airflow/providers/fab/auth_manager/security_manager/override.py @@ -576,6 +576,7 @@ def reset_user_sessions(self, user: User) -> None: session_details = interface.serializer.loads(want_bytes(s.data)) if session_details.get("_user_id") == user.id: session.delete(s) + session.commit() else: self._cli_safe_flash( "Since you are using `securecookie` session backend mechanism, we cannot prevent " diff --git a/providers/tests/fab/auth_manager/cli_commands/test_utils.py b/providers/tests/fab/auth_manager/cli_commands/test_utils.py index cacf614a82fd0..e7f25185b5c98 100644 --- a/providers/tests/fab/auth_manager/cli_commands/test_utils.py +++ b/providers/tests/fab/auth_manager/cli_commands/test_utils.py @@ -16,19 +16,69 @@ # under the License. from __future__ import annotations +import os + import pytest +import airflow +from airflow.configuration import conf +from airflow.exceptions import AirflowConfigException +from airflow.www.extensions.init_appbuilder import AirflowAppBuilder +from airflow.www.session import AirflowDatabaseSessionInterface + from tests_common.test_utils.compat import ignore_provider_compatibility_error +from tests_common.test_utils.config import conf_vars with ignore_provider_compatibility_error("2.9.0+", __file__): from airflow.providers.fab.auth_manager.cli_commands.utils import get_application_builder -from airflow.www.extensions.init_appbuilder import AirflowAppBuilder - pytestmark = pytest.mark.db_test +@pytest.fixture +def flask_app(): + """Fixture to set up the Flask app with the necessary configuration.""" + # Get the webserver config file path + webserver_config = conf.get_mandatory_value("webserver", "config_file") + + with get_application_builder() as appbuilder: + flask_app = appbuilder.app + + # Load webserver configuration + flask_app.config.from_pyfile(webserver_config, silent=True) + + yield flask_app + + class TestCliUtils: def test_get_application_builder(self): + """Test that get_application_builder returns an AirflowAppBuilder instance.""" with get_application_builder() as appbuilder: assert isinstance(appbuilder, AirflowAppBuilder) + + def test_sqlalchemy_uri_configured(self, flask_app): + """Test that the SQLALCHEMY_DATABASE_URI is correctly set in the Flask app.""" + sqlalchemy_uri = conf.get("database", "SQL_ALCHEMY_CONN") + + # Assert that the SQLAlchemy URI is correctly set + assert sqlalchemy_uri == flask_app.config["SQLALCHEMY_DATABASE_URI"] + + def test_relative_path_sqlite_raises_exception(self): + """Test that a relative SQLite path raises an AirflowConfigException.""" + # Directly simulate the configuration for relative SQLite path + with conf_vars({("database", "SQL_ALCHEMY_CONN"): "sqlite://relative/path"}): + with pytest.raises(AirflowConfigException, match="Cannot use relative path"): + with get_application_builder(): + pass + + def test_static_folder_exists(self, flask_app): + """Test that the static folder is correctly configured in the Flask app.""" + static_folder = os.path.join(os.path.dirname(airflow.__file__), "www", "static") + assert flask_app.static_folder == static_folder + + def test_database_auth_backend_in_session(self, flask_app): + """Test that the database is used for session management when AUTH_BACKEND is set to 'database'.""" + with get_application_builder() as appbuilder: + flask_app = appbuilder.app + # Ensure that the correct session interface is set (for 'database' auth backend) + assert isinstance(flask_app.session_interface, AirflowDatabaseSessionInterface) From ccfc52e6b4e1f7bb7be7569aafa99deb04517466 Mon Sep 17 00:00:00 2001 From: Amogh Desai Date: Fri, 6 Dec 2024 10:14:09 +0530 Subject: [PATCH 2/6] Fixing cli test failure in CI (#44679) * Fixing cli test failure in CI * review comments (cherry picked from commit 98e0977a53ea3dc55987f5a2c512fb3b590d3d1c) --- tests/cli/test_cli_parser.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/cli/test_cli_parser.py b/tests/cli/test_cli_parser.py index 0336dcb1eade3..1d0bd04d4bcac 100644 --- a/tests/cli/test_cli_parser.py +++ b/tests/cli/test_cli_parser.py @@ -418,10 +418,8 @@ def test_invalid_choice_raises_for_export_format_in_db_export_archived_command( ["db", "export-archived", "--export-format", export_format, "--output-path", "mydir"] ) error_msg = stderr.getvalue() - assert error_msg == ( - "\nairflow db export-archived command error: argument " - f"--export-format: invalid choice: '{export_format}' " - "(choose from 'csv'), see help above.\n" + assert ( + "airflow db export-archived command error: argument --export-format: invalid choice" in error_msg ) @pytest.mark.parametrize( From 8cf73ed2579ceb4ca9eef0dec3a5daafd3e707f0 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 18 Dec 2024 17:36:44 +0100 Subject: [PATCH 3/6] Avoid 1.1.8 version of msgraph-core (#45044) The 1.1.8 version of msgraph-core is buggy - importing some basic classes causes import error "ABCMeta" is not subscriptable. We are removing the version from azure provider dependencies hoping that it will be fixed in the next version. https://github.com/microsoftgraph/msgraph-sdk-python-core/issues/781 (cherry picked from commit 3310b8628e34578d16d2afbea3809ae422cd6d76) --- generated/provider_dependencies.json | 2 +- providers/src/airflow/providers/microsoft/azure/provider.yaml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index 31a8beb99c79a..c5e413e1e7679 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -838,7 +838,7 @@ "microsoft-kiota-http>=1.3.0,!=1.3.4", "microsoft-kiota-serialization-json==1.0.0", "microsoft-kiota-serialization-text==1.0.0", - "msgraph-core>=1.0.0" + "msgraph-core>=1.0.0,!=1.1.8" ], "devel-deps": [ "pywinrm>=0.4" diff --git a/providers/src/airflow/providers/microsoft/azure/provider.yaml b/providers/src/airflow/providers/microsoft/azure/provider.yaml index 257d50a6f8fb2..6fb098acd784a 100644 --- a/providers/src/airflow/providers/microsoft/azure/provider.yaml +++ b/providers/src/airflow/providers/microsoft/azure/provider.yaml @@ -109,7 +109,9 @@ dependencies: - azure-mgmt-datafactory>=2.0.0 - azure-mgmt-containerregistry>=8.0.0 - azure-mgmt-containerinstance>=10.1.0 - - msgraph-core>=1.0.0 + # msgraph-core 1.1.8 has a bug which causes ABCMeta object is not subscriptable error + # See https://github.com/microsoftgraph/msgraph-sdk-python-core/issues/781 + - msgraph-core>=1.0.0,!=1.1.8 # msgraph-core has transient import failures with microsoft-kiota-http==1.3.4 # See https://github.com/microsoftgraph/msgraph-sdk-python-core/issues/706 - microsoft-kiota-http>=1.3.0,!=1.3.4 From 1c8e0eba2f9d19bce6e2ae50e4e9ec0f0af3545c Mon Sep 17 00:00:00 2001 From: Kaxil Naik Date: Thu, 14 Nov 2024 19:34:32 +0000 Subject: [PATCH 4/6] Fix Google Cloud Datacatalog test (#44037) This field (`dataplex_transfer_status`) got added in https://github.com/googleapis/google-cloud-python/pull/13277 as part of `google-cloud-datacatalog==3.22.0` (cherry picked from commit 060f75f2b80ede71d2911a0cb3956c45b12222e0) --- providers/tests/google/cloud/operators/test_datacatalog.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/providers/tests/google/cloud/operators/test_datacatalog.py b/providers/tests/google/cloud/operators/test_datacatalog.py index 4247831933337..577852baf6b61 100644 --- a/providers/tests/google/cloud/operators/test_datacatalog.py +++ b/providers/tests/google/cloud/operators/test_datacatalog.py @@ -370,7 +370,7 @@ def test_assert_valid_hook_call(self, mock_xcom, mock_hook) -> None: "project_id": TEST_PROJECT_ID, }, ) - assert TEST_TAG_TEMPLATE_DICT == result + assert {**result, **TEST_TAG_TEMPLATE_DICT} == result class TestCloudDataCatalogCreateTagTemplateFieldOperator: @@ -418,7 +418,7 @@ def test_assert_valid_hook_call(self, mock_xcom, mock_hook) -> None: "project_id": TEST_PROJECT_ID, }, ) - assert TEST_TAG_TEMPLATE_FIELD_DICT == result + assert {**result, **TEST_TAG_TEMPLATE_FIELD_DICT} == result class TestCloudDataCatalogDeleteEntryOperator: From 32e0e2ba7ae7002b645f9a36e7038adc85cacf2a Mon Sep 17 00:00:00 2001 From: GPK Date: Fri, 22 Nov 2024 09:58:23 +0000 Subject: [PATCH 5/6] fix google datacatalog operator tests (#44281) * fix google datacatalog operator tests * bump google-cloud-datacatalog (cherry picked from commit f6bbc633cb4fd05ae26137e5f5ca99b5f6d64abb) --- generated/provider_dependencies.json | 2 +- .../src/airflow/providers/google/provider.yaml | 2 +- .../google/cloud/operators/test_datacatalog.py | 16 ++++++++++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/generated/provider_dependencies.json b/generated/provider_dependencies.json index c5e413e1e7679..e3ecbed29d979 100644 --- a/generated/provider_dependencies.json +++ b/generated/provider_dependencies.json @@ -655,7 +655,7 @@ "google-cloud-build>=3.22.0", "google-cloud-compute>=1.10.0", "google-cloud-container>=2.17.4", - "google-cloud-datacatalog>=3.11.1", + "google-cloud-datacatalog>=3.23.0", "google-cloud-dataflow-client>=0.8.6", "google-cloud-dataform>=0.5.0", "google-cloud-dataplex>=1.10.0", diff --git a/providers/src/airflow/providers/google/provider.yaml b/providers/src/airflow/providers/google/provider.yaml index 604eff4190fc5..44ab9a52154cf 100644 --- a/providers/src/airflow/providers/google/provider.yaml +++ b/providers/src/airflow/providers/google/provider.yaml @@ -125,7 +125,7 @@ dependencies: - google-cloud-build>=3.22.0 - google-cloud-compute>=1.10.0 - google-cloud-container>=2.17.4 - - google-cloud-datacatalog>=3.11.1 + - google-cloud-datacatalog>=3.23.0 - google-cloud-dataflow-client>=0.8.6 - google-cloud-dataform>=0.5.0 - google-cloud-dataplex>=1.10.0 diff --git a/providers/tests/google/cloud/operators/test_datacatalog.py b/providers/tests/google/cloud/operators/test_datacatalog.py index 577852baf6b61..1d0d0c4992898 100644 --- a/providers/tests/google/cloud/operators/test_datacatalog.py +++ b/providers/tests/google/cloud/operators/test_datacatalog.py @@ -102,9 +102,20 @@ "name": TEST_ENTRY_PATH, } TEST_ENTRY_GROUP: EntryGroup = EntryGroup(name=TEST_ENTRY_GROUP_PATH) -TEST_ENTRY_GROUP_DICT: dict = {"description": "", "display_name": "", "name": TEST_ENTRY_GROUP_PATH} +TEST_ENTRY_GROUP_DICT: dict = { + "description": "", + "display_name": "", + "name": TEST_ENTRY_GROUP_PATH, + "transferred_to_dataplex": False, +} TEST_TAG: Tag = Tag(name=TEST_TAG_PATH) -TEST_TAG_DICT: dict = {"fields": {}, "name": TEST_TAG_PATH, "template": "", "template_display_name": ""} +TEST_TAG_DICT: dict = { + "fields": {}, + "name": TEST_TAG_PATH, + "template": "", + "template_display_name": "", + "dataplex_transfer_status": 0, +} TEST_TAG_TEMPLATE: TagTemplate = TagTemplate(name=TEST_TAG_TEMPLATE_PATH) TEST_TAG_TEMPLATE_DICT: dict = { "display_name": "", @@ -172,6 +183,7 @@ def test_assert_valid_hook_call(self, mock_xcom, mock_hook) -> None: "project_id": TEST_PROJECT_ID, }, ) + assert TEST_ENTRY_DICT == result @mock.patch("airflow.providers.google.cloud.operators.datacatalog.CloudDataCatalogHook") From b165db9368887d7ddebd458e75b90ae0220c9d53 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 23 Dec 2024 15:24:39 +0100 Subject: [PATCH 6/6] Correctly import isabs from os.path (#45178) The #45139 imported isabs from "airflow.www.app" - but isabs has been added there fairly recently and it is anyhow stdlib's os.path isabs - so it should be imported from there. This breaks fab 1.5.2 backport compatibility tests, so we need to cherry-pick it there alongside #45139 (cherry picked from commit 7002966ccf02fca59c3ac4a604a6ee34cc2cca26) --- .../airflow/providers/fab/auth_manager/cli_commands/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/providers/src/airflow/providers/fab/auth_manager/cli_commands/utils.py b/providers/src/airflow/providers/fab/auth_manager/cli_commands/utils.py index 30963f5cf1fdd..e848c2094ce5b 100644 --- a/providers/src/airflow/providers/fab/auth_manager/cli_commands/utils.py +++ b/providers/src/airflow/providers/fab/auth_manager/cli_commands/utils.py @@ -20,6 +20,7 @@ import os from contextlib import contextmanager from functools import lru_cache +from os.path import isabs from typing import TYPE_CHECKING, Generator from flask import Flask @@ -27,7 +28,7 @@ import airflow from airflow.configuration import conf from airflow.exceptions import AirflowConfigException -from airflow.www.app import isabs, make_url +from airflow.www.app import make_url from airflow.www.extensions.init_appbuilder import init_appbuilder from airflow.www.extensions.init_session import init_airflow_session_interface from airflow.www.extensions.init_views import init_plugins