diff --git a/octopoes/octopoes/connector/octopoes.py b/octopoes/octopoes/connector/octopoes.py index 7bd8c261949..c8f6ba76061 100644 --- a/octopoes/octopoes/connector/octopoes.py +++ b/octopoes/octopoes/connector/octopoes.py @@ -279,7 +279,7 @@ def list_findings( def list_reports( self, valid_time: datetime, offset: int = DEFAULT_OFFSET, limit: int = DEFAULT_LIMIT - ) -> Paginated[Report]: + ) -> Paginated[tuple[Report, list[Report | None]]]: params: dict[str, str | int | list[str]] = {"valid_time": str(valid_time), "offset": offset, "limit": limit} res = self.session.get(f"/{self.client}/reports", params=params) diff --git a/rocky/.ci/.env.test b/rocky/.ci/.env.test index a834fb38ef8..c7e4af41224 100644 --- a/rocky/.ci/.env.test +++ b/rocky/.ci/.env.test @@ -16,8 +16,6 @@ ROCKY_DB_HOST=ci_rocky-db ROCKY_DB_PORT=5432 ROCKY_DB_DSN=postgres://${ROCKY_DB_USER}:${ROCKY_DB_PASSWORD}@${ROCKY_DB_HOST}:${ROCKY_DB_PORT}/${ROCKY_DB} -LOG_LEVEL=debug - KEIKO_API=http://placeholder:8005 KATALOGUS_API=http://katalogus_mock:8000 OCTOPOES_API=http://ci_octopoes:80 diff --git a/rocky/Makefile b/rocky/Makefile index 615e9719125..bce14cf9615 100644 --- a/rocky/Makefile +++ b/rocky/Makefile @@ -35,6 +35,7 @@ test: python3 manage.py test testclean: + docker compose -f .ci/docker-compose.yml kill docker compose -f .ci/docker-compose.yml down --remove-orphans docker compose -f .ci/docker-compose.yml build diff --git a/rocky/katalogus/client.py b/rocky/katalogus/client.py index 02c4999409b..9edc2cd81b0 100644 --- a/rocky/katalogus/client.py +++ b/rocky/katalogus/client.py @@ -129,7 +129,7 @@ def __init__(self, error: httpx.HTTPStatusError): class KATalogusClientV1: - def __init__(self, base_uri: str, organization: str): + def __init__(self, base_uri: str, organization: str | None): self.session = httpx.Client(base_url=base_uri) self.organization = valid_organization_code(organization) if organization else organization self.organization_uri = f"/v1/organisations/{organization}" diff --git a/rocky/onboarding/views.py b/rocky/onboarding/views.py index b935915a0f5..37826f74514 100644 --- a/rocky/onboarding/views.py +++ b/rocky/onboarding/views.py @@ -358,11 +358,14 @@ def post(self, request, *args, **kwargs): report_ooi = self.save_report([("Onboarding Report", "Onboarding Report")]) - return redirect( - reverse("view_report", kwargs={"organization_code": self.organization.code}) - + "?" - + urlencode({"report_id": report_ooi.reference}) - ) + if report_ooi: + return redirect( + reverse("view_report", kwargs={"organization_code": self.organization.code}) + + "?" + + urlencode({"report_id": report_ooi.reference}) + ) + + return self.get(request, *args, **kwargs) def set_member_onboarded(self): member = OrganizationMember.objects.get(user=self.request.user, organization=self.organization) diff --git a/rocky/reports/forms.py b/rocky/reports/forms.py index 73765b0f38c..9388d4fe4f4 100644 --- a/rocky/reports/forms.py +++ b/rocky/reports/forms.py @@ -106,7 +106,7 @@ class CustomReportScheduleForm(BaseRockyForm): class ParentReportNameForm(BaseRockyForm): parent_report_name = forms.CharField( - label=_("Report name format"), required=False, initial="{report type} for {ooi}" + label=_("Report name format"), required=False, initial="{report type} for {oois_count} objects" ) diff --git a/rocky/reports/runner/local.py b/rocky/reports/runner/local.py deleted file mode 100644 index d95cf15bff8..00000000000 --- a/rocky/reports/runner/local.py +++ /dev/null @@ -1,45 +0,0 @@ -from datetime import datetime, timezone - -from django.conf import settings -from katalogus.client import KATalogusError, get_katalogus -from tools.models import Organization - -from octopoes.connector.octopoes import OctopoesAPIConnector -from octopoes.models import Reference -from octopoes.models.ooi.reports import ReportRecipe -from reports.report_types.helpers import get_report_by_id -from reports.runner.models import JobRuntimeError, ReportJobRunner -from reports.views.base import format_plugin_data, hydrate_plugins -from reports.views.mixins import collect_reports, save_report_data -from rocky.bytes_client import get_bytes_client -from rocky.scheduler import ReportTask - - -class LocalReportJobRunner(ReportJobRunner): - def run(self, report_task: ReportTask) -> None: - now = datetime.now(timezone.utc) - connector = OctopoesAPIConnector(settings.OCTOPOES_API, report_task.organisation_id) - recipe: ReportRecipe = connector.get( - Reference.from_str(f"ReportRecipe|{report_task.report_recipe_id}"), datetime.now(timezone.utc) - ) - parsed_report_types = [get_report_by_id(report_type_id) for report_type_id in recipe.report_types] - - error_reports, report_data = collect_reports( - now, connector, list(recipe.input_recipe["input_oois"]), parsed_report_types - ) - - try: - report_type_plugins = hydrate_plugins(parsed_report_types, get_katalogus(report_task.organisation_id)) - plugins = format_plugin_data(report_type_plugins) - except KATalogusError as e: - raise JobRuntimeError("Failed to hydrate plugins from KATalogus") from e - - save_report_data( - get_bytes_client(report_task.organisation_id), - now, - connector, - Organization.objects.get(code=report_task.organisation_id), - plugins, - report_data, - [(recipe.report_name_format, recipe.report_name_format)], - ) diff --git a/rocky/reports/runner/models.py b/rocky/reports/runner/models.py index 6ea2409cbd3..6551b3db619 100644 --- a/rocky/reports/runner/models.py +++ b/rocky/reports/runner/models.py @@ -1,7 +1,7 @@ from octopoes.models.ooi.reports import ReportRecipe -class ReportJobRunner: +class ReportRunner: def run(self, recipe: ReportRecipe) -> None: raise NotImplementedError() diff --git a/rocky/reports/runner/report_runner.py b/rocky/reports/runner/report_runner.py new file mode 100644 index 00000000000..f8a742e500e --- /dev/null +++ b/rocky/reports/runner/report_runner.py @@ -0,0 +1,63 @@ +from datetime import datetime, timezone + +from django.conf import settings +from tools.models import Organization + +from octopoes.connector.octopoes import OctopoesAPIConnector +from octopoes.models import Reference +from reports.report_types.definitions import report_plugins_union +from reports.report_types.helpers import get_report_by_id +from reports.runner.models import ReportRunner +from reports.views.mixins import collect_reports, save_report_data +from rocky.bytes_client import BytesClient +from rocky.scheduler import ReportTask + + +class LocalReportRunner(ReportRunner): + def __init__(self, bytes_client: BytesClient, valid_time: datetime | None = None): + self.bytes_client = bytes_client + self.valid_time = valid_time + + def run(self, report_task: ReportTask) -> None: + valid_time = self.valid_time or datetime.now(timezone.utc) + + connector = OctopoesAPIConnector(settings.OCTOPOES_API, report_task.organisation_id) + recipe = connector.get(Reference.from_str(f"ReportRecipe|{report_task.report_recipe_id}"), valid_time) + report_types = [get_report_by_id(report_type_id) for report_type_id in recipe.report_types] + + error_reports, report_data = collect_reports( + valid_time, connector, recipe.input_recipe["input_oois"], report_types + ) + + self.bytes_client.organization = report_task.organisation_id + report_names = [] + oois_count = 0 + + for report_type_id, data in report_data.items(): + oois_count += len(data) + report_type = get_report_by_id(report_type_id) + + for ooi in data: + report_name = recipe.subreport_name_format.replace("{ooi}", ooi).replace( + "{report type}", str(report_type.name) + ) + report_names.append((report_name, report_name)) + + save_report_data( + self.bytes_client, + valid_time, + connector, + Organization.objects.get(code=report_task.organisation_id), + { + "input_data": { + "input_oois": recipe.input_recipe["input_oois"], + "report_types": recipe.report_types, + "plugins": report_plugins_union(report_types), + } + }, + report_data, + report_names, + recipe.report_name_format.replace("{oois_count}", str(oois_count)), + ) + + self.bytes_client.organization = None diff --git a/rocky/reports/runner/worker.py b/rocky/reports/runner/worker.py index 34397c20120..350bdd8ed80 100644 --- a/rocky/reports/runner/worker.py +++ b/rocky/reports/runner/worker.py @@ -10,8 +10,9 @@ from httpx import HTTPError from pydantic import ValidationError -from reports.runner.local import LocalReportJobRunner -from reports.runner.models import ReportJobRunner, WorkerManager +from reports.runner.models import ReportRunner, WorkerManager +from reports.runner.report_runner import LocalReportRunner +from rocky.bytes_client import get_bytes_client from rocky.scheduler import SchedulerClient, Task, TaskStatus, scheduler_client logger = structlog.get_logger(__name__) @@ -20,7 +21,7 @@ class SchedulerWorkerManager(WorkerManager): def __init__( self, - runner: ReportJobRunner, + runner: ReportRunner, scheduler: SchedulerClient, pool_size: int, poll_interval: int, @@ -221,7 +222,7 @@ def _format_exit_code(exitcode: int | None) -> str: def _start_working( - task_queue: mp.Queue, runner: ReportJobRunner, scheduler: SchedulerClient, handling_tasks: dict[int, str] + task_queue: mp.Queue, runner: ReportRunner, scheduler: SchedulerClient, handling_tasks: dict[int, str] ): logger.info("Started listening for tasks from worker[pid=%s]", os.getpid()) @@ -251,7 +252,7 @@ def _start_working( def get_runtime_manager() -> WorkerManager: return SchedulerWorkerManager( - LocalReportJobRunner(), + LocalReportRunner(get_bytes_client("")), # These are set dynamically. Needs a refactor. scheduler_client(None), settings.POOL_SIZE, settings.POLL_INTERVAL, diff --git a/rocky/reports/templates/partials/report_names_header.html b/rocky/reports/templates/partials/report_names_header.html index b2b0a28854c..6e3a11906fd 100644 --- a/rocky/reports/templates/partials/report_names_header.html +++ b/rocky/reports/templates/partials/report_names_header.html @@ -12,9 +12,10 @@

{% translate "Report name" %}

{% blocktranslate trimmed %} To make the report names more descriptive, you can include placeholders for the - object name, the report type and/or the reference date. Use the placeholder "{ooi}" for the - object name, "{report type}" for the report type and use a Python - strftime code for the reference date. + object name, the report type and/or the reference date. For subreports and reports over a single object, + use the placeholder "{ooi}" for the object name, "{report type}" for the report type and use a + Python strftime code for the reference + date. For reports over multiple objects, use "{oois_count}" for the number of objects in the report. {% endblocktranslate %}

diff --git a/rocky/reports/views/base.py b/rocky/reports/views/base.py index 49507b6b1b2..f03947015a9 100644 --- a/rocky/reports/views/base.py +++ b/rocky/reports/views/base.py @@ -120,13 +120,9 @@ def get(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse: return redirect(reverse("report_history", kwargs=self.get_kwargs())) -def get_plugin_ids(report_types: list[type[BaseReport]]): - return report_plugins_union(report_types) - - def hydrate_plugins(report_types: list[type["BaseReport"]], katalogus: KATalogusClientV1) -> dict[str, list[Plugin]]: plugins: dict[str, list[Plugin]] = {"required": [], "optional": []} - merged_plugins = get_plugin_ids(report_types) + merged_plugins = report_plugins_union(report_types) required_plugins_ids = list(merged_plugins["required"]) optional_plugins_ids = list(merged_plugins["optional"]) @@ -255,14 +251,6 @@ def get_available_report_types(self) -> tuple[list[dict[str, str]] | dict[str, l report_types = self.get_report_types_for_generate_report() return report_types, len(report_types) - def get_plugin_data_for_saving(self) -> list[dict]: - try: - report_type_plugins = hydrate_plugins(self.get_report_types(), get_katalogus(self.organization.code)) - except KATalogusError as error: - return messages.error(self.request, error.message) - - return format_plugin_data(report_type_plugins) - def get_observed_at(self): return self.observed_at if self.observed_at < datetime.now(timezone.utc) else datetime.now(timezone.utc) @@ -296,7 +284,7 @@ def get_input_data(self) -> dict[str, Any]: "input_data": { "input_oois": self.get_ooi_pks(), "report_types": self.get_report_type_ids(), - "plugins": get_plugin_ids(self.get_report_types()), + "plugins": report_plugins_union(self.get_report_types()), } } diff --git a/rocky/reports/views/mixins.py b/rocky/reports/views/mixins.py index 62e1ecc5fa4..14ac15cbccd 100644 --- a/rocky/reports/views/mixins.py +++ b/rocky/reports/views/mixins.py @@ -14,7 +14,6 @@ from reports.report_types.concatenated_report.report import ConcatenatedReport from reports.report_types.helpers import REPORTS, get_report_by_id from reports.views.base import BaseReportView, ReportDataDict -from rocky.bytes_client import BytesClient def collect_reports(observed_at: datetime, octopoes_connector: OctopoesAPIConnector, ooi_pks: list[str], report_types): @@ -55,24 +54,28 @@ def collect_reports(observed_at: datetime, octopoes_connector: OctopoesAPIConnec return error_reports, report_data -def save_report_raw(bytes_client: BytesClient, data: dict) -> str: - report_data_raw_id = bytes_client.upload_raw( - raw=ReportDataDict(data).model_dump_json().encode(), manual_mime_types={"openkat/report"} - ) - - return report_data_raw_id - - def save_report_data( - bytes_client, observed_at, octopoes_api_connector, organization, input_data, report_data, report_names -): + bytes_client, + observed_at, + octopoes_api_connector, + organization, + input_data: dict, + report_data, + report_names, + parent_report_name, +) -> Report | None: + if len(report_data) == 0: + return None + now = datetime.now(timezone.utc) # if it's not a single report, we need a parent if len(report_data) > 1 or len(list(report_data.values())[0]) > 1: - raw_id = save_report_raw(bytes_client, data=input_data) - name = now.strftime(report_names[0][1]) + raw_id = bytes_client.upload_raw( + raw=ReportDataDict(input_data).model_dump_json().encode(), manual_mime_types={"openkat/report"} + ) + name = now.strftime(parent_report_name.replace("{report type}", str(ConcatenatedReport.name))) if not name or name.isspace(): name = ConcatenatedReport.name @@ -121,11 +124,13 @@ def save_report_data( ] child_input_data = { - "input_data": {"input_oois": [ooi], "report_types": [report_type_id], "plugins": child_plugins} + "input_data": {"input_oois": [ooi], "report_types": [report_type_id], "plugins": [child_plugins]} } - raw_id = save_report_raw(bytes_client, data={"report_data": data["data"]} | child_input_data) - + raw_id = bytes_client.upload_raw( + raw=ReportDataDict({"report_data": data["data"]} | child_input_data).model_dump_json().encode(), + manual_mime_types={"openkat/report"}, + ) name = now.strftime(name_to_save) if not name or name.isspace(): name = ConcatenatedReport.name @@ -153,10 +158,12 @@ def save_report_data( report_type_id = next(iter(report_data)) ooi = next(iter(report_data[report_type_id])) data = report_data[report_type_id][ooi] - - raw_id = save_report_raw(bytes_client, data={"report_data": data["data"]} | input_data) + raw_id = bytes_client.upload_raw( + raw=ReportDataDict({"report_data": data["data"]} | input_data).model_dump_json().encode(), + manual_mime_types={"openkat/report"}, + ) report_type = get_report_by_id(report_type_id) - name = now.strftime(report_names[0][1]) + name = now.strftime(parent_report_name.replace("{report type}", str(report_type.name))) if not name or name.isspace(): name = ConcatenatedReport.name @@ -182,7 +189,7 @@ def save_report_data( class SaveGenerateReportMixin(BaseReportView): - def save_report(self, report_names: list) -> Report: + def save_report(self, report_names: list) -> Report | None: error_reports, report_data = collect_reports( self.observed_at, self.octopoes_api_connector, @@ -198,6 +205,7 @@ def save_report(self, report_names: list) -> Report: self.get_input_data(), report_data, report_names, + report_names[0][0], ) # If OOI could not be found or the date is incorrect, it will be shown to the user as a message error @@ -239,10 +247,11 @@ def save_report(self, report_names: list) -> Report: now = datetime.utcnow() bytes_client = self.bytes_client - # Save report data into bytes - - report_data_raw_id = save_report_raw(bytes_client, data=post_processed_data | self.get_input_data()) - + # Create the report + report_data_raw_id = bytes_client.upload_raw( + raw=ReportDataDict(post_processed_data | self.get_input_data()).model_dump_json().encode(), + manual_mime_types={"openkat/report"}, + ) report_type = type(aggregate_report) name = now.strftime(report_names[0][1]) if not name or name.isspace(): @@ -268,6 +277,9 @@ def save_report(self, report_names: list) -> Report: # Save the child reports to bytes for ooi, types in report_data.items(): for report_type, data in types.items(): - save_report_raw(bytes_client, data=data | self.get_input_data()) + bytes_client.upload_raw( + raw=ReportDataDict(data | self.get_input_data()).model_dump_json().encode(), + manual_mime_types={"openkat/report"}, + ) return report_ooi diff --git a/rocky/rocky/bytes_client.py b/rocky/rocky/bytes_client.py index f874daab3c5..791ee8b5a5d 100644 --- a/rocky/rocky/bytes_client.py +++ b/rocky/rocky/bytes_client.py @@ -16,7 +16,7 @@ class BytesClient: - def __init__(self, base_url: str, username: str, password: str, organization: str): + def __init__(self, base_url: str, username: str, password: str, organization: str | None): self.credentials = {"username": username, "password": password} self.session = httpx.Client(base_url=base_url) self.organization = organization diff --git a/rocky/rocky/locale/django.pot b/rocky/rocky/locale/django.pot index 4af966792b8..9dc9f1faa12 100644 --- a/rocky/rocky/locale/django.pot +++ b/rocky/rocky/locale/django.pot @@ -9,7 +9,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2024-10-09 14:20+0000\n" +"POT-Creation-Date: 2024-10-16 14:17+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -3793,10 +3793,12 @@ msgstr "" #: reports/templates/partials/report_names_header.html msgid "" "To make the report names more descriptive, you can include placeholders for " -"the object name, the report type and/or the reference date. Use the " -"placeholder \"{ooi}\" for the object name, \"{report type}\" for the report " -"type and use a Python strftime code for the reference date." +"the object name, the report type and/or the reference date. For subreports " +"and reports over a single object, use the placeholder \"{ooi}\" for the " +"object name, \"{report type}\" for the report type and use a Python " +"strftime code for the reference date. For reports over multiple objects, " +"use \"{oois_count}\" for the number of objects in the report." msgstr "" #: reports/templates/partials/report_names_header.html diff --git a/rocky/rocky/views/mixins.py b/rocky/rocky/views/mixins.py index f8f1be7bd2f..fda5d4eb250 100644 --- a/rocky/rocky/views/mixins.py +++ b/rocky/rocky/views/mixins.py @@ -391,26 +391,27 @@ def get_subreports(self, report_id: str) -> list[tuple[str, Report]]: return subreports - def hydrate_report_list(self, reports: list[Report]) -> list[HydratedReport]: + def hydrate_report_list(self, reports: list[tuple[Report, list[Report | None]]]) -> list[HydratedReport]: hydrated_reports: list[HydratedReport] = [] for report in reports: hydrated_report: HydratedReport = HydratedReport() parent_report, children_reports = report + filtered_children_reports = list(filter(None, children_reports)) - hydrated_report.total_children_reports = len(children_reports) + hydrated_report.total_children_reports = len(filtered_children_reports) if len(parent_report.input_oois) > 0: hydrated_report.total_objects = len(parent_report.input_oois) else: - hydrated_report.total_objects = len(self.get_children_input_oois(children_reports)) + hydrated_report.total_objects = len(self.get_children_input_oois(filtered_children_reports)) - hydrated_report.report_type_summary = self.report_type_summary(children_reports) + hydrated_report.report_type_summary = self.report_type_summary(filtered_children_reports) if not parent_report.has_parent: hydrated_children_reports: list[Report] = [] - for child_report in children_reports: + for child_report in filtered_children_reports: if str(child_report.parent_report) == str(parent_report): hydrated_children_reports.append(child_report) if len(hydrated_children_reports) >= 5: # We want to show only 5 children reports diff --git a/rocky/tests/integration/conftest.py b/rocky/tests/integration/conftest.py index fb80952ba6e..63ed1abd011 100644 --- a/rocky/tests/integration/conftest.py +++ b/rocky/tests/integration/conftest.py @@ -4,6 +4,8 @@ import pytest from django.conf import settings +from reports.runner.report_runner import LocalReportRunner +from tools.models import Organization from octopoes.api.models import Declaration, Observation from octopoes.connector.octopoes import OctopoesAPIConnector @@ -15,6 +17,7 @@ from octopoes.models.ooi.service import IPService, Service from octopoes.models.ooi.software import Software, SoftwareInstance from octopoes.models.ooi.web import URL, HostnameHTTPURL, HTTPHeader, HTTPResource, SecurityTXT, Website +from rocky.health import ServiceHealth @pytest.fixture @@ -23,10 +26,30 @@ def valid_time(): @pytest.fixture -def octopoes_api_connector(request) -> OctopoesAPIConnector: +def katalogus_mock(mocker): + katalogus = mocker.patch("katalogus.client.KATalogusClientV1") + katalogus().health.return_value = ServiceHealth(service="katalogus", healthy=True) + + return katalogus + + +@pytest.fixture +def integration_organization(katalogus_mock, request) -> Organization: test_node = f"test-{request.node.originalname}" - connector = OctopoesAPIConnector(settings.OCTOPOES_API, test_node) + return Organization.objects.create(name="Test", code=test_node) + + +@pytest.fixture +def integration_organization_2(request) -> Organization: + test_node = f"test-{request.node.originalname}-2" + + return Organization.objects.create(name="Test 2", code=test_node) + + +@pytest.fixture +def octopoes_api_connector(integration_organization) -> OctopoesAPIConnector: + connector = OctopoesAPIConnector(settings.OCTOPOES_API, integration_organization.code) connector.create_node() yield connector @@ -34,16 +57,19 @@ def octopoes_api_connector(request) -> OctopoesAPIConnector: @pytest.fixture -def octopoes_api_connector_2(request) -> OctopoesAPIConnector: - test_node = f"test-{request.node.originalname}-2" - - connector = OctopoesAPIConnector(settings.OCTOPOES_API, test_node) +def octopoes_api_connector_2(integration_organization_2) -> OctopoesAPIConnector: + connector = OctopoesAPIConnector(settings.OCTOPOES_API, integration_organization_2.code) connector.create_node() yield connector connector.delete_node() +@pytest.fixture +def report_runner(valid_time, mocker) -> LocalReportRunner: + return LocalReportRunner(mocker.MagicMock(), valid_time) + + def seed_system( octopoes_api_connector: OctopoesAPIConnector, valid_time: datetime, diff --git a/rocky/tests/integration/test_report_runner.py b/rocky/tests/integration/test_report_runner.py new file mode 100644 index 00000000000..75f60a3a7b7 --- /dev/null +++ b/rocky/tests/integration/test_report_runner.py @@ -0,0 +1,101 @@ +import json + +from reports.runner.report_runner import LocalReportRunner + +from octopoes.api.models import Declaration +from octopoes.connector.octopoes import OctopoesAPIConnector +from octopoes.models.ooi.reports import ReportRecipe +from rocky.health import ServiceHealth +from rocky.scheduler import ReportTask +from tests.integration.conftest import seed_system + + +def test_run_report_task(octopoes_api_connector: OctopoesAPIConnector, report_runner: LocalReportRunner, valid_time): + oois = seed_system(octopoes_api_connector, valid_time) + report_runner.bytes_client.health.return_value = ServiceHealth(service="bytes", healthy=True) + report_runner.bytes_client.upload_raw.return_value = "abcdabcd-f8ab-4bdf-9b1b-58cd98ef6342" + + recipe = ReportRecipe( + recipe_id="abc4e52b-812c-4cc2-8196-35fb8efc63ca", + report_name_format="Concatenated report for {oois_count} objects", + subreport_name_format="{report type} for {ooi} in %Y", + input_recipe={"input_oois": [oois["hostnames"][0].reference, oois["hostnames"][1].reference]}, + report_types=["dns-report"], + cron_expression="* * * * *", + ) + octopoes_api_connector.save_declaration(Declaration(ooi=recipe, valid_time=valid_time)) + + task = ReportTask(organisation_id=octopoes_api_connector.client, report_recipe_id=str(recipe.recipe_id)) + report_runner.run(task) + + assert len(report_runner.bytes_client.upload_raw.mock_calls) == 3 + + assert report_runner.bytes_client.upload_raw.mock_calls[0].kwargs["manual_mime_types"] == {"openkat/report"} + assert report_runner.bytes_client.upload_raw.mock_calls[1].kwargs["manual_mime_types"] == {"openkat/report"} + assert report_runner.bytes_client.upload_raw.mock_calls[2].kwargs["manual_mime_types"] == {"openkat/report"} + + data = json.loads(report_runner.bytes_client.upload_raw.mock_calls[0].kwargs["raw"]) + data["input_data"]["plugins"]["required"] = set(data["input_data"]["plugins"]["required"]) # ordering issues + assert data == { + "input_data": { + "input_oois": ["Hostname|test|example.com", "Hostname|test|a.example.com"], + "report_types": ["dns-report"], + "plugins": {"required": {"dns-sec", "dns-records"}, "optional": ["dns-zone"]}, + } + } + + # The order of the OOIs being processed is not guaranteed, so this is a simple workaround + both_calls = [ + { + "report_data": { + "input_ooi": "Hostname|test|example.com", + "records": [], + "security": {"spf": True, "dkim": True, "dmarc": True, "dnssec": True, "caa": True}, + "finding_types": [], + }, + "input_data": { + "input_oois": ["Hostname|test|example.com"], + "report_types": ["dns-report"], + "plugins": [{"required": {"dns-sec", "dns-records"}, "optional": ["dns-zone"]}], + }, + }, + { + "report_data": { + "input_ooi": "Hostname|test|a.example.com", + "records": [], + "security": {"spf": True, "dkim": True, "dmarc": True, "dnssec": True, "caa": True}, + "finding_types": [], + }, + "input_data": { + "input_oois": ["Hostname|test|a.example.com"], + "report_types": ["dns-report"], + "plugins": [{"required": {"dns-sec", "dns-records"}, "optional": ["dns-zone"]}], + }, + }, + ] + + data_1 = json.loads(report_runner.bytes_client.upload_raw.mock_calls[1].kwargs["raw"]) + data_1["input_data"]["plugins"][0]["required"] = set( + data_1["input_data"]["plugins"][0]["required"] + ) # ordering issues + data_2 = json.loads(report_runner.bytes_client.upload_raw.mock_calls[2].kwargs["raw"]) + data_2["input_data"]["plugins"][0]["required"] = set( + data_2["input_data"]["plugins"][0]["required"] + ) # ordering issues + + assert data_1 in both_calls + assert data_2 in both_calls + + reports = octopoes_api_connector.list_reports(valid_time) + assert reports.count == 1 + report, subreports = reports.items[0] + assert len(subreports) == 2 + + assert report.name == "Concatenated report for 2 objects" + assert "DNS Report for Hostname|test|a.example.com in 2024" in {x.name for x in subreports} + + # FIXME: the naming logic in reports/views/mixins.py 107-112 is not right. We expect to find example.com in this + # set, but instead only find a.example.com because when ooi_name is 'example.com', the check: + # `ooi_name in default_name` also passes for 'DNS Report for Hostname|test|a.example.com in %Y'. + # We shouldn't have to guess the match in the report_names argument. The name should be overridden on an object + # in the report_data list probably. Note that sometimes this does work when the OOIs are ordered differently.