Fix/report naming

octopoes/octopoes/connector/octopoes.py

@@ -279,7 +279,7 @@ def list_findings(
 
     def list_reports(
         self, valid_time: datetime, offset: int = DEFAULT_OFFSET, limit: int = DEFAULT_LIMIT
-    ) -> Paginated[Report]:
+    ) -> Paginated[tuple[Report, list[Report | None]]]:
         params: dict[str, str | int | list[str]] = {"valid_time": str(valid_time), "offset": offset, "limit": limit}
         res = self.session.get(f"/{self.client}/reports", params=params)
 

rocky/.ci/.env.test

@@ -16,8 +16,6 @@ ROCKY_DB_HOST=ci_rocky-db
 ROCKY_DB_PORT=5432
 ROCKY_DB_DSN=postgres://${ROCKY_DB_USER}:${ROCKY_DB_PASSWORD}@${ROCKY_DB_HOST}:${ROCKY_DB_PORT}/${ROCKY_DB}
 
-LOG_LEVEL=debug
-
 KEIKO_API=http://placeholder:8005
 KATALOGUS_API=http://katalogus_mock:8000
 OCTOPOES_API=http://ci_octopoes:80

rocky/Makefile

@@ -35,6 +35,7 @@ test:
 	python3 manage.py test
 
 testclean:
+	docker compose -f .ci/docker-compose.yml kill
 	docker compose -f .ci/docker-compose.yml down --remove-orphans
 	docker compose -f .ci/docker-compose.yml build
 

rocky/katalogus/client.py

@@ -129,7 +129,7 @@ def __init__(self, error: httpx.HTTPStatusError):
 
 
 class KATalogusClientV1:
-    def __init__(self, base_uri: str, organization: str):
+    def __init__(self, base_uri: str, organization: str | None):
         self.session = httpx.Client(base_url=base_uri)
         self.organization = valid_organization_code(organization) if organization else organization
         self.organization_uri = f"/v1/organisations/{organization}"

rocky/onboarding/views.py

@@ -358,11 +358,12 @@ def post(self, request, *args, **kwargs):
 
         report_ooi = self.save_report([("Onboarding Report", "Onboarding Report")])
 
-        return redirect(
-            reverse("view_report", kwargs={"organization_code": self.organization.code})
-            + "?"
-            + urlencode({"report_id": report_ooi.reference})
-        )
+        if report_ooi:
+            return redirect(
+                reverse("view_report", kwargs={"organization_code": self.organization.code})
+                + "?"
+                + urlencode({"report_id": report_ooi.reference})
+            )
 
     def set_member_onboarded(self):
         member = OrganizationMember.objects.get(user=self.request.user, organization=self.organization)

rocky/reports/forms.py

@@ -106,7 +106,7 @@ class CustomReportScheduleForm(BaseRockyForm):
 
 class ParentReportNameForm(BaseRockyForm):
     parent_report_name = forms.CharField(
-        label=_("Report name format"), required=False, initial="{report type} for {ooi}"
+        label=_("Report name format"), required=False, initial="{report type} for {oois_count} objects"
     )
 
 

rocky/reports/runner/local.py

@@ -1,45 +1,71 @@
 from datetime import datetime, timezone
 
 from django.conf import settings
-from katalogus.client import KATalogusError, get_katalogus
+from katalogus.client import KATalogusClientV1, KATalogusError
 from tools.models import Organization
 
 from octopoes.connector.octopoes import OctopoesAPIConnector
 from octopoes.models import Reference
-from octopoes.models.ooi.reports import ReportRecipe
 from reports.report_types.helpers import get_report_by_id
 from reports.runner.models import JobRuntimeError, ReportJobRunner
 from reports.views.base import format_plugin_data, hydrate_plugins
 from reports.views.mixins import collect_reports, save_report_data
-from rocky.bytes_client import get_bytes_client
+from rocky.bytes_client import BytesClient
 from rocky.scheduler import ReportTask
 
 
 class LocalReportJobRunner(ReportJobRunner):
+    def __init__(
+        self, katalogus_client: KATalogusClientV1, bytes_client: BytesClient, valid_time: datetime | None = None
+    ):
+        self.katalogus_client = katalogus_client
+        self.bytes_client = bytes_client
+        self.valid_time = valid_time
+
     def run(self, report_task: ReportTask) -> None:
-        now = datetime.now(timezone.utc)
+        valid_time = self.valid_time or datetime.now(timezone.utc)
+
         connector = OctopoesAPIConnector(settings.OCTOPOES_API, report_task.organisation_id)
-        recipe: ReportRecipe = connector.get(
-            Reference.from_str(f"ReportRecipe|{report_task.report_recipe_id}"), datetime.now(timezone.utc)
-        )
-        parsed_report_types = [get_report_by_id(report_type_id) for report_type_id in recipe.report_types]
+        recipe = connector.get(Reference.from_str(f"ReportRecipe|{report_task.report_recipe_id}"), valid_time)
+        report_types = [get_report_by_id(report_type_id) for report_type_id in recipe.report_types]
 
         error_reports, report_data = collect_reports(
-            now, connector, list(recipe.input_recipe["input_oois"]), parsed_report_types
+            valid_time, connector, recipe.input_recipe["input_oois"], report_types
         )
 
+        self.katalogus_client.organization = report_task.organisation_id
+        self.katalogus_client.organization_uri = f"/v1/organisations/{report_task.organisation_id}"
+
         try:
-            report_type_plugins = hydrate_plugins(parsed_report_types, get_katalogus(report_task.organisation_id))
+            report_type_plugins = hydrate_plugins(report_types, self.katalogus_client)
             plugins = format_plugin_data(report_type_plugins)
         except KATalogusError as e:
             raise JobRuntimeError("Failed to hydrate plugins from KATalogus") from e
 
+        self.katalogus_client.organization = None
+        self.katalogus_client.organization_uri = ""
+
+        self.bytes_client.organization = report_task.organisation_id
+        report_names = []
+        oois_count = 0
+
+        for report_type_id, data in report_data.items():
+            oois_count += len(data)
+            report_type = get_report_by_id(report_type_id)
+
+            for ooi in data:
+                report_name = recipe.subreport_name_format.format(ooi=ooi, report_type=str(report_type.name))
+                report_names.append((report_name, report_name))
+
         save_report_data(
-            get_bytes_client(report_task.organisation_id),
-            now,
+            self.bytes_client,
+            valid_time,
             connector,
             Organization.objects.get(code=report_task.organisation_id),
             plugins,
             report_data,
-            [(recipe.report_name_format, recipe.report_name_format)],
+            report_names,
+            recipe.report_name_format.format(oois_count=oois_count),
         )
+
+        self.bytes_client.organization = None

rocky/reports/runner/worker.py

@@ -8,10 +8,12 @@
 import structlog
 from django.conf import settings
 from httpx import HTTPError
+from katalogus.client import get_katalogus
 from pydantic import ValidationError
 
 from reports.runner.local import LocalReportJobRunner
 from reports.runner.models import ReportJobRunner, WorkerManager
+from rocky.bytes_client import get_bytes_client
 from rocky.scheduler import SchedulerClient, Task, TaskStatus, scheduler_client
 
 logger = structlog.get_logger(__name__)
@@ -251,7 +253,7 @@ def _start_working(
 
 def get_runtime_manager() -> WorkerManager:
     return SchedulerWorkerManager(
-        LocalReportJobRunner(),
+        LocalReportJobRunner(get_katalogus(""), get_bytes_client("")),  # These are set dynamically. Needs a refactor.
         scheduler_client(None),
         settings.POOL_SIZE,
         settings.POLL_INTERVAL,

rocky/reports/views/mixins.py

@@ -14,7 +14,6 @@
 from reports.report_types.concatenated_report.report import ConcatenatedReport
 from reports.report_types.helpers import REPORTS, get_report_by_id
 from reports.views.base import BaseReportView, ReportDataDict
-from rocky.bytes_client import BytesClient
 
 
 def collect_reports(observed_at: datetime, octopoes_connector: OctopoesAPIConnector, ooi_pks: list[str], report_types):
@@ -55,25 +54,28 @@ def collect_reports(observed_at: datetime, octopoes_connector: OctopoesAPIConnec
     return error_reports, report_data
 
 
-def save_report_raw(bytes_client: BytesClient, data: dict) -> str:
-    report_data_raw_id = bytes_client.upload_raw(
-        raw=ReportDataDict(data).model_dump_json().encode(), manual_mime_types={"openkat/report"}
-    )
-
-    return report_data_raw_id
-
-
 def save_report_data(
-    bytes_client, observed_at, octopoes_api_connector, organization, input_data, report_data, report_names
-):
+    bytes_client,
+    observed_at,
+    octopoes_api_connector,
+    organization,
+    input_data,
+    report_data,
+    report_names,
+    parent_report_name,
+) -> Report | None:
+    if len(report_data) == 0:
+        return None
+
     now = datetime.now(timezone.utc)
 
     # if it's not a single report, we need a parent
 
     if len(report_data) > 1 or len(list(report_data.values())[0]) > 1:
-        raw_id = save_report_raw(bytes_client, data=input_data)
-        name = now.strftime(report_names[0][1])
-
+        raw_id = bytes_client.upload_raw(
+            raw=ReportDataDict(input_data).model_dump_json().encode(), manual_mime_types={"openkat/report"}
+        )
+        name = now.strftime(parent_report_name)
         if not name or name.isspace():
             name = ConcatenatedReport.name
 
@@ -124,8 +126,10 @@ def save_report_data(
                     "input_data": {"input_oois": [ooi], "report_types": [report_type_id], "plugins": child_plugins}
                 }
 
-                raw_id = save_report_raw(bytes_client, data={"report_data": data["data"]} | child_input_data)
-
+                raw_id = bytes_client.upload_raw(
+                    raw=ReportDataDict({"report_data": data["data"]} | child_input_data).model_dump_json().encode(),
+                    manual_mime_types={"openkat/report"},
+                )
                 name = now.strftime(name_to_save)
                 if not name or name.isspace():
                     name = ConcatenatedReport.name
@@ -153,8 +157,10 @@ def save_report_data(
         report_type_id = next(iter(report_data))
         ooi = next(iter(report_data[report_type_id]))
         data = report_data[report_type_id][ooi]
-
-        raw_id = save_report_raw(bytes_client, data={"report_data": data["data"]} | input_data)
+        raw_id = bytes_client.upload_raw(
+            raw=ReportDataDict({"report_data": data["data"]} | input_data).model_dump_json().encode(),
+            manual_mime_types={"openkat/report"},
+        )
         report_type = get_report_by_id(report_type_id)
         name = now.strftime(report_names[0][1])
 
@@ -182,7 +188,7 @@ def save_report_data(
 
 
 class SaveGenerateReportMixin(BaseReportView):
-    def save_report(self, report_names: list) -> Report:
+    def save_report(self, report_names: list) -> Report | None:
         error_reports, report_data = collect_reports(
             self.observed_at,
             self.octopoes_api_connector,
@@ -198,6 +204,7 @@ def save_report(self, report_names: list) -> Report:
             self.get_input_data(),
             report_data,
             report_names,
+            report_names[0][0],
         )
 
         # If OOI could not be found or the date is incorrect, it will be shown to the user as a message error
@@ -239,10 +246,11 @@ def save_report(self, report_names: list) -> Report:
         now = datetime.utcnow()
         bytes_client = self.bytes_client
 
-        # Save report data into bytes
-
-        report_data_raw_id = save_report_raw(bytes_client, data=post_processed_data | self.get_input_data())
-
+        # Create the report
+        report_data_raw_id = bytes_client.upload_raw(
+            raw=ReportDataDict(post_processed_data | self.get_input_data()).model_dump_json().encode(),
+            manual_mime_types={"openkat/report"},
+        )
         report_type = type(aggregate_report)
         name = now.strftime(report_names[0][1])
         if not name or name.isspace():
@@ -268,6 +276,9 @@ def save_report(self, report_names: list) -> Report:
         # Save the child reports to bytes
         for ooi, types in report_data.items():
             for report_type, data in types.items():
-                save_report_raw(bytes_client, data=data | self.get_input_data())
+                bytes_client.upload_raw(
+                    raw=ReportDataDict(data | self.get_input_data()).model_dump_json().encode(),
+                    manual_mime_types={"openkat/report"},
+                )
 
         return report_ooi

rocky/rocky/bytes_client.py

@@ -16,7 +16,7 @@
 
 
 class BytesClient:
-    def __init__(self, base_url: str, username: str, password: str, organization: str):
+    def __init__(self, base_url: str, username: str, password: str, organization: str | None):
         self.credentials = {"username": username, "password": password}
         self.session = httpx.Client(base_url=base_url)
         self.organization = organization

rocky/rocky/views/mixins.py

@@ -391,26 +391,27 @@ def get_subreports(self, report_id: str) -> list[tuple[str, Report]]:
 
         return subreports
 
-    def hydrate_report_list(self, reports: list[Report]) -> list[HydratedReport]:
+    def hydrate_report_list(self, reports: list[tuple[Report, list[Report | None]]]) -> list[HydratedReport]:
         hydrated_reports: list[HydratedReport] = []
 
         for report in reports:
             hydrated_report: HydratedReport = HydratedReport()
 
             parent_report, children_reports = report
+            filtered_children_reports = list(filter(None, children_reports))
 
-            hydrated_report.total_children_reports = len(children_reports)
+            hydrated_report.total_children_reports = len(filtered_children_reports)
 
             if len(parent_report.input_oois) > 0:
                 hydrated_report.total_objects = len(parent_report.input_oois)
             else:
-                hydrated_report.total_objects = len(self.get_children_input_oois(children_reports))
+                hydrated_report.total_objects = len(self.get_children_input_oois(filtered_children_reports))
 
-            hydrated_report.report_type_summary = self.report_type_summary(children_reports)
+            hydrated_report.report_type_summary = self.report_type_summary(filtered_children_reports)
 
             if not parent_report.has_parent:
                 hydrated_children_reports: list[Report] = []
-                for child_report in children_reports:
+                for child_report in filtered_children_reports:
                     if str(child_report.parent_report) == str(parent_report):
                         hydrated_children_reports.append(child_report)
                     if len(hydrated_children_reports) >= 5:  # We want to show only 5 children reports