From 217906fdc5e4e67b38c769f270b6a5e9a2987e45 Mon Sep 17 00:00:00 2001 From: Maxime Carbonneau-Leclerc <3360483+maxi297@users.noreply.github.com> Date: Mon, 9 Dec 2024 14:26:22 -0500 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Source=20Jira:=20fix=20timezone?= =?UTF-8?q?=20issue=20(#48838)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../connectors/source-jira/metadata.yaml | 2 +- .../connectors/source-jira/poetry.lock | 16 +++- .../connectors/source-jira/pyproject.toml | 3 +- .../source-jira/source_jira/manifest.yaml | 2 +- .../unit_tests/integration/__init__.py | 0 .../unit_tests/integration/config.py | 34 +++++++ .../unit_tests/integration/test_issues.py | 88 +++++++++++++++++++ docs/integrations/sources/jira.md | 1 + 8 files changed, 142 insertions(+), 4 deletions(-) create mode 100644 airbyte-integrations/connectors/source-jira/unit_tests/integration/__init__.py create mode 100644 airbyte-integrations/connectors/source-jira/unit_tests/integration/config.py create mode 100644 airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py diff --git a/airbyte-integrations/connectors/source-jira/metadata.yaml b/airbyte-integrations/connectors/source-jira/metadata.yaml index 4c5f5d331e1d..7cf62bb72715 100644 --- a/airbyte-integrations/connectors/source-jira/metadata.yaml +++ b/airbyte-integrations/connectors/source-jira/metadata.yaml @@ -10,7 +10,7 @@ data: connectorSubtype: api connectorType: source definitionId: 68e63de2-bb83-4c7e-93fa-a8a9051e3993 - dockerImageTag: 3.4.1 + dockerImageTag: 3.4.2 dockerRepository: airbyte/source-jira documentationUrl: https://docs.airbyte.com/integrations/sources/jira erdUrl: https://dbdocs.io/airbyteio/source-jira?view=relationships diff --git a/airbyte-integrations/connectors/source-jira/poetry.lock b/airbyte-integrations/connectors/source-jira/poetry.lock index adbd441fccdb..c9c3d3a5f3e3 100644 --- a/airbyte-integrations/connectors/source-jira/poetry.lock +++ b/airbyte-integrations/connectors/source-jira/poetry.lock @@ -509,6 +509,20 @@ files = [ [package.extras] test = ["pytest (>=6)"] +[[package]] +name = "freezegun" +version = "1.2.2" +description = "Let your Python tests travel through time" +optional = false +python-versions = ">=3.6" +files = [ + {file = "freezegun-1.2.2-py3-none-any.whl", hash = "sha256:ea1b963b993cb9ea195adbd893a48d573fda951b0da64f60883d7e988b606c9f"}, + {file = "freezegun-1.2.2.tar.gz", hash = "sha256:cd22d1ba06941384410cd967d8a99d5ae2442f57dfafeff2fda5de8dc5c05446"}, +] + +[package.dependencies] +python-dateutil = ">=2.7" + [[package]] name = "genson" version = "1.3.0" @@ -2043,4 +2057,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.10,<3.12" -content-hash = "471fabf3282fea3f534c909270bb02fcfebd773eb5e26d666f9194e174ded6d8" +content-hash = "72d248f774a68421f400671e384a73bbe6c228d20f78289d109573d06362fea7" diff --git a/airbyte-integrations/connectors/source-jira/pyproject.toml b/airbyte-integrations/connectors/source-jira/pyproject.toml index f1235e3c957c..21301cdc4855 100644 --- a/airbyte-integrations/connectors/source-jira/pyproject.toml +++ b/airbyte-integrations/connectors/source-jira/pyproject.toml @@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",] build-backend = "poetry.core.masonry.api" [tool.poetry] -version = "3.4.1" +version = "3.4.2" name = "source-jira" description = "Source implementation for Jira." authors = [ "Airbyte ",] @@ -23,6 +23,7 @@ airbyte-cdk = "^6" source-jira = "source_jira.run:run" [tool.poetry.group.dev.dependencies] +freezegun = "==1.2.2" pytest = "==6.2.5" requests-mock = "^1.9.3" pytest-mock = "^3.6.1" diff --git a/airbyte-integrations/connectors/source-jira/source_jira/manifest.yaml b/airbyte-integrations/connectors/source-jira/source_jira/manifest.yaml index 490ab3aee0a5..aadd4f0a17eb 100644 --- a/airbyte-integrations/connectors/source-jira/source_jira/manifest.yaml +++ b/airbyte-integrations/connectors/source-jira/source_jira/manifest.yaml @@ -1077,7 +1077,7 @@ definitions: type: DatetimeBasedCursor cursor_field: "updated" start_datetime: "{{ config.get('start_date', '1970-01-01T00:00:00Z') }}" - datetime_format: "%Y-%m-%dT%H:%M:%SZ" + datetime_format: "%Y-%m-%dT%H:%M:%S%z" cursor_datetime_formats: - "%Y-%m-%dT%H:%M:%S.%f%z" lookback_window: "PT{{ config.get('lookback_window_minutes', '0') }}M" diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/integration/__init__.py b/airbyte-integrations/connectors/source-jira/unit_tests/integration/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/integration/config.py b/airbyte-integrations/connectors/source-jira/unit_tests/integration/config.py new file mode 100644 index 000000000000..ad2f7bd7b5ec --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/unit_tests/integration/config.py @@ -0,0 +1,34 @@ +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. + +from datetime import datetime +from typing import Any, Dict, List + + +class ConfigBuilder: + def __init__(self) -> None: + self._config: Dict[str, Any] = { + "api_token": "any_api_token", + "domain": "airbyteio.atlassian.net", + "email": "integration-test@airbyte.io", + "start_date": "2021-01-01T00:00:00Z", + "projects": [], + } + + def with_api_token(self, api_token: str) -> "ConfigBuilder": + self._config["api_token"] = api_token + return self + + def with_domain(self, domain: str) -> "ConfigBuilder": + self._config["domain"] = domain + return self + + def with_start_date(self, start_datetime: datetime) -> "ConfigBuilder": + self._config["start_date"] = start_datetime.strftime("%Y-%m-%dT%H:%M:%SZ") + return self + + def with_projects(self, projects: List[str]) -> "ConfigBuilder": + self._config["projects"] = projects + return self + + def build(self) -> Dict[str, Any]: + return self._config diff --git a/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py b/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py new file mode 100644 index 000000000000..c0306ea87a4a --- /dev/null +++ b/airbyte-integrations/connectors/source-jira/unit_tests/integration/test_issues.py @@ -0,0 +1,88 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. +import json +import os +from datetime import datetime, timedelta, timezone +from typing import Any, Dict +from unittest import TestCase + +import freezegun +from airbyte_cdk.models import ConfiguredAirbyteCatalog, SyncMode +from airbyte_cdk.test.catalog_builder import CatalogBuilder +from airbyte_cdk.test.entrypoint_wrapper import read +from airbyte_cdk.test.mock_http import HttpMocker, HttpRequest +from airbyte_cdk.test.mock_http.response_builder import ( + FieldPath, + HttpResponseBuilder, + RecordBuilder, + create_record_builder, + create_response_builder, + find_template, +) +from airbyte_cdk.test.state_builder import StateBuilder +from integration.config import ConfigBuilder +from source_jira import SourceJira + +_STREAM_NAME = "issues" +_API_TOKEN = "api_token" +_DOMAIN = "airbyteio.atlassian.net" +_NOW = datetime(2024, 1, 1, tzinfo=timezone.utc) + + +def _create_config() -> ConfigBuilder: + return ConfigBuilder().with_api_token(_API_TOKEN).with_domain(_DOMAIN) + + +def _create_catalog(sync_mode: SyncMode = SyncMode.full_refresh) -> ConfiguredAirbyteCatalog: + return CatalogBuilder().with_stream(name="issues", sync_mode=sync_mode).build() + + +def _response_template() -> Dict[str, Any]: + with open(os.path.join(os.path.dirname(__file__), "..", "responses", "issues.json")) as response_file_handler: + return json.load(response_file_handler) + +def _create_response() -> HttpResponseBuilder: + return create_response_builder( + response_template=_response_template(), + records_path=FieldPath("issues"), + ) + + +def _create_record() -> RecordBuilder: + return create_record_builder( + _response_template(), FieldPath("issues"), record_id_path=FieldPath("id"), record_cursor_path=FieldPath("updated") + ) + + +@freezegun.freeze_time(_NOW.isoformat()) +class IssuesTest(TestCase): + @HttpMocker() + def test_given_timezone_in_state_when_read_consider_timezone(self, http_mocker: HttpMocker) -> None: + config = _create_config().build() + datetime_with_timezone = "2023-11-01T00:00:00.000-0800" + timestamp_with_timezone = 1698825600000 + state = StateBuilder().with_stream_state( + "issues", + { + "use_global_cursor":False, + "state": {"updated": datetime_with_timezone}, + "lookback_window": 2, + "states": [{"partition":{"parent_slice":{},"project_id":"10025"},"cursor":{"updated": datetime_with_timezone}}] + } + ).build() + http_mocker.get( + HttpRequest( + f"https://{_DOMAIN}/rest/api/3/search", + { + "fields": "*all", + "jql": f"updated >= {timestamp_with_timezone} ORDER BY updated asc", + "expand": "renderedFields,transitions,changelog", + "maxResults": "50", + } + ), + _create_response().with_record(_create_record()).with_record(_create_record()).build(), + ) + + source = SourceJira(config=config, catalog=_create_catalog(), state=state) + actual_messages = read(source, config=config, catalog=_create_catalog(), state=state) + + assert len(actual_messages.records) == 2 diff --git a/docs/integrations/sources/jira.md b/docs/integrations/sources/jira.md index cc0c23a74953..56390a666d5c 100644 --- a/docs/integrations/sources/jira.md +++ b/docs/integrations/sources/jira.md @@ -154,6 +154,7 @@ The Jira connector should not run into Jira API limitations under normal usage. | Version | Date | Pull Request | Subject | |:-----------|:-----------|:-----------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------| +| 3.4.2 | 2024-12-09 | [48838](https://github.com/airbytehq/airbyte/pull/48838) | Fixing timezone gaps with state | | 3.4.1 | 2024-12-09 | [48859](https://github.com/airbytehq/airbyte/pull/48859) | Add a couple of fixes regarding memory usage | | 3.4.0 | 2024-12-05 | [48738](https://github.com/airbytehq/airbyte/pull/48738) | Enable concurrency for substreams without cursor | | 3.3.1 | 2024-11-18 | [48539](https://github.com/airbytehq/airbyte/pull/48539) | Update dependencies |