Fix/report naming (#3666)

Co-authored-by: Rieven <[email protected]>
Co-authored-by: stephanie0x00 <[email protected]>

octopoes/octopoes/connector/octopoes.py

@@ -279,7 +279,7 @@ def list_findings(
 
     def list_reports(
         self, valid_time: datetime, offset: int = DEFAULT_OFFSET, limit: int = DEFAULT_LIMIT
-    ) -> Paginated[Report]:
+    ) -> Paginated[tuple[Report, list[Report | None]]]:
         params: dict[str, str | int | list[str]] = {"valid_time": str(valid_time), "offset": offset, "limit": limit}
         res = self.session.get(f"/{self.client}/reports", params=params)
 

rocky/.ci/.env.test

@@ -16,8 +16,6 @@ ROCKY_DB_HOST=ci_rocky-db
 ROCKY_DB_PORT=5432
 ROCKY_DB_DSN=postgres://${ROCKY_DB_USER}:${ROCKY_DB_PASSWORD}@${ROCKY_DB_HOST}:${ROCKY_DB_PORT}/${ROCKY_DB}
 
-LOG_LEVEL=debug
-
 KEIKO_API=http://placeholder:8005
 KATALOGUS_API=http://katalogus_mock:8000
 OCTOPOES_API=http://ci_octopoes:80

rocky/Makefile

@@ -35,6 +35,7 @@ test:
 	python3 manage.py test
 
 testclean:
+	docker compose -f .ci/docker-compose.yml kill
 	docker compose -f .ci/docker-compose.yml down --remove-orphans
 	docker compose -f .ci/docker-compose.yml build
 

rocky/katalogus/client.py

@@ -129,7 +129,7 @@ def __init__(self, error: httpx.HTTPStatusError):
 
 
 class KATalogusClientV1:
-    def __init__(self, base_uri: str, organization: str):
+    def __init__(self, base_uri: str, organization: str | None):
         self.session = httpx.Client(base_url=base_uri)
         self.organization = valid_organization_code(organization) if organization else organization
         self.organization_uri = f"/v1/organisations/{organization}"

rocky/onboarding/views.py

@@ -358,11 +358,14 @@ def post(self, request, *args, **kwargs):
 
         report_ooi = self.save_report([("Onboarding Report", "Onboarding Report")])
 
-        return redirect(
-            reverse("view_report", kwargs={"organization_code": self.organization.code})
-            + "?"
-            + urlencode({"report_id": report_ooi.reference})
-        )
+        if report_ooi:
+            return redirect(
+                reverse("view_report", kwargs={"organization_code": self.organization.code})
+                + "?"
+                + urlencode({"report_id": report_ooi.reference})
+            )
+
+        return self.get(request, *args, **kwargs)
 
     def set_member_onboarded(self):
         member = OrganizationMember.objects.get(user=self.request.user, organization=self.organization)

rocky/reports/forms.py

@@ -106,7 +106,7 @@ class CustomReportScheduleForm(BaseRockyForm):
 
 class ParentReportNameForm(BaseRockyForm):
     parent_report_name = forms.CharField(
-        label=_("Report name format"), required=False, initial="{report type} for {ooi}"
+        label=_("Report name format"), required=False, initial="{report type} for {oois_count} objects"
     )
 
 

rocky/reports/runner/models.py

@@ -1,7 +1,7 @@
 from octopoes.models.ooi.reports import ReportRecipe
 
 
-class ReportJobRunner:
+class ReportRunner:
     def run(self, recipe: ReportRecipe) -> None:
         raise NotImplementedError()
 

rocky/reports/runner/report_runner.py

@@ -0,0 +1,63 @@
+from datetime import datetime, timezone
+
+from django.conf import settings
+from tools.models import Organization
+
+from octopoes.connector.octopoes import OctopoesAPIConnector
+from octopoes.models import Reference
+from reports.report_types.definitions import report_plugins_union
+from reports.report_types.helpers import get_report_by_id
+from reports.runner.models import ReportRunner
+from reports.views.mixins import collect_reports, save_report_data
+from rocky.bytes_client import BytesClient
+from rocky.scheduler import ReportTask
+
+
+class LocalReportRunner(ReportRunner):
+    def __init__(self, bytes_client: BytesClient, valid_time: datetime | None = None):
+        self.bytes_client = bytes_client
+        self.valid_time = valid_time
+
+    def run(self, report_task: ReportTask) -> None:
+        valid_time = self.valid_time or datetime.now(timezone.utc)
+
+        connector = OctopoesAPIConnector(settings.OCTOPOES_API, report_task.organisation_id)
+        recipe = connector.get(Reference.from_str(f"ReportRecipe|{report_task.report_recipe_id}"), valid_time)
+        report_types = [get_report_by_id(report_type_id) for report_type_id in recipe.report_types]
+
+        error_reports, report_data = collect_reports(
+            valid_time, connector, recipe.input_recipe["input_oois"], report_types
+        )
+
+        self.bytes_client.organization = report_task.organisation_id
+        report_names = []
+        oois_count = 0
+
+        for report_type_id, data in report_data.items():
+            oois_count += len(data)
+            report_type = get_report_by_id(report_type_id)
+
+            for ooi in data:
+                report_name = recipe.subreport_name_format.replace("{ooi}", ooi).replace(
+                    "{report type}", str(report_type.name)
+                )
+                report_names.append((report_name, report_name))
+
+        save_report_data(
+            self.bytes_client,
+            valid_time,
+            connector,
+            Organization.objects.get(code=report_task.organisation_id),
+            {
+                "input_data": {
+                    "input_oois": recipe.input_recipe["input_oois"],
+                    "report_types": recipe.report_types,
+                    "plugins": report_plugins_union(report_types),
+                }
+            },
+            report_data,
+            report_names,
+            recipe.report_name_format.replace("{oois_count}", str(oois_count)),
+        )
+
+        self.bytes_client.organization = None

rocky/reports/runner/worker.py

@@ -10,8 +10,9 @@
 from httpx import HTTPError
 from pydantic import ValidationError
 
-from reports.runner.local import LocalReportJobRunner
-from reports.runner.models import ReportJobRunner, WorkerManager
+from reports.runner.models import ReportRunner, WorkerManager
+from reports.runner.report_runner import LocalReportRunner
+from rocky.bytes_client import get_bytes_client
 from rocky.scheduler import SchedulerClient, Task, TaskStatus, scheduler_client
 
 logger = structlog.get_logger(__name__)
@@ -20,7 +21,7 @@
 class SchedulerWorkerManager(WorkerManager):
     def __init__(
         self,
-        runner: ReportJobRunner,
+        runner: ReportRunner,
         scheduler: SchedulerClient,
         pool_size: int,
         poll_interval: int,
@@ -221,7 +222,7 @@ def _format_exit_code(exitcode: int | None) -> str:
 
 
 def _start_working(
-    task_queue: mp.Queue, runner: ReportJobRunner, scheduler: SchedulerClient, handling_tasks: dict[int, str]
+    task_queue: mp.Queue, runner: ReportRunner, scheduler: SchedulerClient, handling_tasks: dict[int, str]
 ):
     logger.info("Started listening for tasks from worker[pid=%s]", os.getpid())
 
@@ -251,7 +252,7 @@ def _start_working(
 
 def get_runtime_manager() -> WorkerManager:
     return SchedulerWorkerManager(
-        LocalReportJobRunner(),
+        LocalReportRunner(get_bytes_client("")),  # These are set dynamically. Needs a refactor.
         scheduler_client(None),
         settings.POOL_SIZE,
         settings.POLL_INTERVAL,

rocky/reports/templates/partials/report_names_header.html

@@ -12,9 +12,10 @@ <h2>{% translate "Report name" %}</h2>
     <p>
         {% blocktranslate trimmed %}
             To make the report names more descriptive, you can include placeholders for the
-            object name, the report type and/or the reference date. Use the placeholder "{ooi}" for the
-            object name, "{report type}" for the report type and use a <a href="https://strftime.org/" target="_blank" rel="noopener">Python
-            strftime code</a> for the reference date.
+            object name, the report type and/or the reference date. For subreports and reports over a single object,
+            use the placeholder "{ooi}" for the object name, "{report type}" for the report type and use a
+            <a href="https://strftime.org/" target="_blank" rel="noopener">Python strftime code</a> for the reference
+            date. For reports over multiple objects, use "{oois_count}" for the number of objects in the report.
         {% endblocktranslate %}
     </p>
     <p>

rocky/reports/views/base.py

@@ -120,13 +120,9 @@ def get(self, request: HttpRequest, *args: Any, **kwargs: Any) -> HttpResponse:
         return redirect(reverse("report_history", kwargs=self.get_kwargs()))
 
 
-def get_plugin_ids(report_types: list[type[BaseReport]]):
-    return report_plugins_union(report_types)
-
-
 def hydrate_plugins(report_types: list[type["BaseReport"]], katalogus: KATalogusClientV1) -> dict[str, list[Plugin]]:
     plugins: dict[str, list[Plugin]] = {"required": [], "optional": []}
-    merged_plugins = get_plugin_ids(report_types)
+    merged_plugins = report_plugins_union(report_types)
 
     required_plugins_ids = list(merged_plugins["required"])
     optional_plugins_ids = list(merged_plugins["optional"])
@@ -255,14 +251,6 @@ def get_available_report_types(self) -> tuple[list[dict[str, str]] | dict[str, l
         report_types = self.get_report_types_for_generate_report()
         return report_types, len(report_types)
 
-    def get_plugin_data_for_saving(self) -> list[dict]:
-        try:
-            report_type_plugins = hydrate_plugins(self.get_report_types(), get_katalogus(self.organization.code))
-        except KATalogusError as error:
-            return messages.error(self.request, error.message)
-
-        return format_plugin_data(report_type_plugins)
-
     def get_observed_at(self):
         return self.observed_at if self.observed_at < datetime.now(timezone.utc) else datetime.now(timezone.utc)
 
@@ -296,7 +284,7 @@ def get_input_data(self) -> dict[str, Any]:
             "input_data": {
                 "input_oois": self.get_ooi_pks(),
                 "report_types": self.get_report_type_ids(),
-                "plugins": get_plugin_ids(self.get_report_types()),
+                "plugins": report_plugins_union(self.get_report_types()),
             }
         }