From f2feca17202c4e42011be2bdf7b20f7a0e3f2219 Mon Sep 17 00:00:00 2001 From: Rieven Date: Mon, 18 Nov 2024 17:35:15 +0100 Subject: [PATCH 1/6] subreports for aggregate reports --- rocky/reports/views/mixins.py | 67 ++++++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 21 deletions(-) diff --git a/rocky/reports/views/mixins.py b/rocky/reports/views/mixins.py index 0ed2d5cb872..04e5c7c16ff 100644 --- a/rocky/reports/views/mixins.py +++ b/rocky/reports/views/mixins.py @@ -13,6 +13,7 @@ from octopoes.models.ooi.reports import Report from reports.report_types.aggregate_organisation_report.report import aggregate_reports from reports.report_types.concatenated_report.report import ConcatenatedReport +from reports.report_types.definitions import BaseReport from reports.report_types.helpers import REPORTS, get_report_by_id from reports.report_types.multi_organization_report.report import MultiOrganizationReport, collect_report_data from reports.views.base import BaseReportView, ReportDataDict @@ -56,6 +57,22 @@ def collect_reports(observed_at: datetime, octopoes_connector: OctopoesAPIConnec return error_reports, report_data +def get_child_input_data(input_data: dict[str, Any], ooi: str, report_type: type[BaseReport]): + required_plugins = list(input_data["input_data"]["plugins"]["required"]) + optional_plugins = list(input_data["input_data"]["plugins"]["optional"]) + + child_plugins: dict[str, list[str]] = {"required": [], "optional": []} + + child_plugins["required"] = [ + plugin_id for plugin_id in required_plugins if plugin_id in report_type.plugins["required"] + ] + child_plugins["optional"] = [ + plugin_id for plugin_id in optional_plugins if plugin_id in report_type.plugins["optional"] + ] + + return {"input_data": {"input_oois": [ooi], "report_types": [report_type.id], "plugins": child_plugins}} + + def save_report_data( bytes_client, observed_at, @@ -114,21 +131,7 @@ def save_report_data( name_to_save = updated_name break - required_plugins = list(input_data["input_data"]["plugins"]["required"]) - optional_plugins = list(input_data["input_data"]["plugins"]["optional"]) - - child_plugins: dict[str, list[str]] = {"required": [], "optional": []} - - child_plugins["required"] = [ - plugin_id for plugin_id in required_plugins if plugin_id in report_type.plugins["required"] - ] - child_plugins["optional"] = [ - plugin_id for plugin_id in optional_plugins if plugin_id in report_type.plugins["optional"] - ] - - child_input_data = { - "input_data": {"input_oois": [ooi], "report_types": [report_type_id], "plugins": [child_plugins]} - } + child_input_data = get_child_input_data(input_data, ooi, report_type) raw_id = bytes_client.upload_raw( raw=ReportDataDict({"report_data": data["data"]} | child_input_data).model_dump_json().encode(), @@ -204,8 +207,7 @@ def save_aggregate_report_data( aggregate_report, ) -> Report: observed_at = get_observed_at - - now = datetime.utcnow() + now = datetime.now(timezone.utc) # Create the report report_data_raw_id = bytes_client.upload_raw( @@ -226,7 +228,7 @@ def save_aggregate_report_data( organization_name=organization.name, organization_tags=[tag.name for tag in organization.tags.all()], data_raw_id=report_data_raw_id, - date_generated=datetime.now(timezone.utc), + date_generated=now, input_oois=ooi_pks, observed_at=observed_at, parent_report=None, @@ -235,12 +237,35 @@ def save_aggregate_report_data( create_ooi(octopoes_api_connector, bytes_client, report_ooi, observed_at) # Save the child reports to bytes + for ooi, types in report_data.items(): - for report_type, data in types.items(): - bytes_client.upload_raw( - raw=ReportDataDict(data | input_data).model_dump_json().encode(), manual_mime_types={"openkat/report"} + for report_type_id, data in types.items(): + report_type = get_report_by_id(report_type_id) + child_input_data = get_child_input_data(input_data, ooi, report_type) + + raw_id = bytes_client.upload_raw( + raw=ReportDataDict({"report_data": data["data"]} | child_input_data).model_dump_json().encode(), + manual_mime_types={"openkat/report"}, ) + aggregate_sub_report_ooi = Report( + name=str(report_type.name), + report_type=report_type_id, + template=report_type.template_path, + report_id=uuid4(), + organization_code=organization.code, + organization_name=organization.name, + organization_tags=[tag.name for tag in organization.tags.all()], + data_raw_id=raw_id, + date_generated=now, + input_oois=[ooi], + observed_at=observed_at, + parent_report=report_ooi.reference, + has_parent=True, + ) + + create_ooi(octopoes_api_connector, bytes_client, aggregate_sub_report_ooi, observed_at) + return report_ooi From 213704b5192a198f06923aca32d9c3e199745387 Mon Sep 17 00:00:00 2001 From: Rieven Date: Mon, 18 Nov 2024 17:40:17 +0100 Subject: [PATCH 2/6] fixes for subreports --- rocky/reports/views/mixins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocky/reports/views/mixins.py b/rocky/reports/views/mixins.py index 226b295e50a..d42984322f7 100644 --- a/rocky/reports/views/mixins.py +++ b/rocky/reports/views/mixins.py @@ -249,7 +249,7 @@ def save_aggregate_report_data( child_input_data = get_child_input_data(input_data, ooi, report_type) raw_id = bytes_client.upload_raw( - raw=ReportDataDict({"report_data": data["data"]} | child_input_data).model_dump_json().encode(), + raw=ReportDataDict({"report_data": data} | child_input_data).model_dump_json().encode(), manual_mime_types={"openkat/report"}, ) From 096f6d4863699321da67fcf72a7e48d0e0a8109f Mon Sep 17 00:00:00 2001 From: Rieven Date: Tue, 19 Nov 2024 15:20:24 +0100 Subject: [PATCH 3/6] Update rocky/reports/views/mixins.py Co-authored-by: ammar92 --- rocky/reports/views/mixins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rocky/reports/views/mixins.py b/rocky/reports/views/mixins.py index d42984322f7..912716bebfa 100644 --- a/rocky/reports/views/mixins.py +++ b/rocky/reports/views/mixins.py @@ -61,7 +61,7 @@ def get_child_input_data(input_data: dict[str, Any], ooi: str, report_type: type required_plugins = list(input_data["input_data"]["plugins"]["required"]) optional_plugins = list(input_data["input_data"]["plugins"]["optional"]) - child_plugins: dict[str, list[str]] = {"required": [], "optional": []} + child_plugins: AggregateReportSubReports = {"required": [], "optional": []} child_plugins["required"] = [ plugin_id for plugin_id in required_plugins if plugin_id in report_type.plugins["required"] From 33f5d8ccbbaa8b7e196607d621c06ba37148f438 Mon Sep 17 00:00:00 2001 From: Rieven Date: Tue, 19 Nov 2024 15:25:01 +0100 Subject: [PATCH 4/6] set type dict for child plugibns --- rocky/reports/report_types/definitions.py | 5 +++++ rocky/reports/views/mixins.py | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/rocky/reports/report_types/definitions.py b/rocky/reports/report_types/definitions.py index 112d6807c80..cdd19d25e58 100644 --- a/rocky/reports/report_types/definitions.py +++ b/rocky/reports/report_types/definitions.py @@ -168,6 +168,11 @@ class AggregateReportSubReports(TypedDict): optional: list[type[Report]] +class SubReportPlugins(TypedDict): + required: list[type[Report]] + optional: list[type[Report]] + + class AggregateReport(BaseReport): reports: AggregateReportSubReports diff --git a/rocky/reports/views/mixins.py b/rocky/reports/views/mixins.py index 912716bebfa..48bfbeb2d30 100644 --- a/rocky/reports/views/mixins.py +++ b/rocky/reports/views/mixins.py @@ -13,7 +13,7 @@ from octopoes.models.ooi.reports import Report from reports.report_types.aggregate_organisation_report.report import aggregate_reports from reports.report_types.concatenated_report.report import ConcatenatedReport -from reports.report_types.definitions import BaseReport +from reports.report_types.definitions import BaseReport, SubReportPlugins from reports.report_types.helpers import REPORTS, get_report_by_id from reports.report_types.multi_organization_report.report import MultiOrganizationReport, collect_report_data from reports.views.base import BaseReportView, ReportDataDict @@ -61,7 +61,7 @@ def get_child_input_data(input_data: dict[str, Any], ooi: str, report_type: type required_plugins = list(input_data["input_data"]["plugins"]["required"]) optional_plugins = list(input_data["input_data"]["plugins"]["optional"]) - child_plugins: AggregateReportSubReports = {"required": [], "optional": []} + child_plugins: SubReportPlugins = {"required": [], "optional": []} child_plugins["required"] = [ plugin_id for plugin_id in required_plugins if plugin_id in report_type.plugins["required"] From 938ffcd42fe12c4d33d7513e21a83a511f557873 Mon Sep 17 00:00:00 2001 From: Rieven Date: Wed, 20 Nov 2024 16:11:41 +0100 Subject: [PATCH 5/6] change subreport typedict --- rocky/reports/report_types/definitions.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/rocky/reports/report_types/definitions.py b/rocky/reports/report_types/definitions.py index cdd19d25e58..ec7c4be9ed6 100644 --- a/rocky/reports/report_types/definitions.py +++ b/rocky/reports/report_types/definitions.py @@ -17,6 +17,11 @@ class ReportPlugins(TypedDict): optional: set[str] +class SubReportPlugins(TypedDict): + required: list[str] + optional: list[str] + + def report_plugins_union(report_types: list[type["BaseReport"]]) -> ReportPlugins: """Take the union of the required and optional plugin sets and remove optional plugins that are required""" @@ -168,11 +173,6 @@ class AggregateReportSubReports(TypedDict): optional: list[type[Report]] -class SubReportPlugins(TypedDict): - required: list[type[Report]] - optional: list[type[Report]] - - class AggregateReport(BaseReport): reports: AggregateReportSubReports From 08dcdc3c00675e2166d44d5fba4a022cbe84e6ba Mon Sep 17 00:00:00 2001 From: Rieven Date: Thu, 21 Nov 2024 11:59:54 +0100 Subject: [PATCH 6/6] Add test --- rocky/tests/conftest.py | 214 ++++++++++++++++++++ rocky/tests/reports/test_report_overview.py | 32 +++ 2 files changed, 246 insertions(+) diff --git a/rocky/tests/conftest.py b/rocky/tests/conftest.py index f73c57d106b..cde70e1abbf 100644 --- a/rocky/tests/conftest.py +++ b/rocky/tests/conftest.py @@ -2340,3 +2340,217 @@ def get_report_input_data_from_bytes(): } } return json.dumps(input_data).encode("utf-8") + + +@pytest.fixture +def aggregate_report_with_sub_reports(): + aggregate_report: Paginated[tuple[Report, list[Report | None]]] = Paginated( + count=1, + items=[ + ( + Report( + object_type="Report", + scan_profile=None, + user_id=None, + primary_key="Report|23820a64-db8f-41b7-b045-031338fbb91d", + name="Aggregate Report", + report_type="aggregate-organisation-report", + template="aggregate_organisation_report/report.html", + date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137), + input_oois=["Hostname|internet|mispo.es"], + report_id=UUID("23820a64-db8f-41b7-b045-031338fbb91d"), + organization_code="_rieven", + organization_name="Rieven", + organization_tags=[], + data_raw_id="3a362cd7-6348-4e91-8a6f-4cd83f9f6a83", + observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043), + parent_report=None, + report_recipe=None, + has_parent=False, + ), + [ + Report( + object_type="Report", + scan_profile=None, + user_id=None, + primary_key="Report|28c7b15e-6dda-49e8-a101-41df3124287e", + name="Mail Report", + report_type="mail-report", + template="mail_report/report.html", + date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137), + input_oois=["Hostname|internet|mispo.es"], + report_id=UUID("28c7b15e-6dda-49e8-a101-41df3124287e"), + organization_code="_rieven", + organization_name="Rieven", + organization_tags=[], + data_raw_id="a534b4d5-5dba-4ddc-9b77-970675ae4b1c", + observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043), + parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"), + report_recipe=None, + has_parent=True, + ), + Report( + object_type="Report", + scan_profile=None, + user_id=None, + primary_key="Report|2a56737f-492f-424b-88cc-0029ce2a444b", + name="IPv6 Report", + report_type="ipv6-report", + template="ipv6_report/report.html", + date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137), + input_oois=["Hostname|internet|mispo.es"], + report_id=UUID("2a56737f-492f-424b-88cc-0029ce2a444b"), + organization_code="_rieven", + organization_name="Rieven", + organization_tags=[], + data_raw_id="0bdea8eb-7ac0-46ef-ad14-ea3b0bfe1030", + observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043), + parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"), + report_recipe=None, + has_parent=True, + ), + Report( + object_type="Report", + scan_profile=None, + user_id=None, + primary_key="Report|4ec12350-7552-40de-8c9f-f75ac04b99cb", + name="RPKI Report", + report_type="rpki-report", + template="rpki_report/report.html", + date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137), + input_oois=["Hostname|internet|mispo.es"], + report_id=UUID("4ec12350-7552-40de-8c9f-f75ac04b99cb"), + organization_code="_rieven", + organization_name="Rieven", + organization_tags=[], + data_raw_id="53d5452c-9e67-42d2-9cb0-3b684d8967a2", + observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043), + parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"), + report_recipe=None, + has_parent=True, + ), + Report( + object_type="Report", + scan_profile=None, + user_id=None, + primary_key="Report|8137a050-f897-45ce-a695-fd21c63e2e5c", + name="Safe Connections Report", + report_type="safe-connections-report", + template="safe_connections_report/report.html", + date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137), + input_oois=["Hostname|internet|mispo.es"], + report_id=UUID("8137a050-f897-45ce-a695-fd21c63e2e5c"), + organization_code="_rieven", + organization_name="Rieven", + organization_tags=[], + data_raw_id="a218ca79-47de-4473-a93d-54d14baadd98", + observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043), + parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"), + report_recipe=None, + has_parent=True, + ), + Report( + object_type="Report", + scan_profile=None, + user_id=None, + primary_key="Report|9ca7ad01-e19e-42c9-9361-751db4399b94", + name="Web System Report", + report_type="web-system-report", + template="web_system_report/report.html", + date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137), + input_oois=["Hostname|internet|mispo.es"], + report_id=UUID("9ca7ad01-e19e-42c9-9361-751db4399b94"), + organization_code="_rieven", + organization_name="Rieven", + organization_tags=[], + data_raw_id="3779f5b0-3adf-41c8-9630-8eed8a857ae6", + observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043), + parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"), + report_recipe=None, + has_parent=True, + ), + Report( + object_type="Report", + scan_profile=None, + user_id=None, + primary_key="Report|a76878ba-55e0-4971-b645-63cfdfd34e78", + name="Open Ports Report", + report_type="open-ports-report", + template="open_ports_report/report.html", + date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137), + input_oois=["Hostname|internet|mispo.es"], + report_id=UUID("a76878ba-55e0-4971-b645-63cfdfd34e78"), + organization_code="_rieven", + organization_name="Rieven", + organization_tags=[], + data_raw_id="851feeab-7036-48f6-81ef-599467c52457", + observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043), + parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"), + report_recipe=None, + has_parent=True, + ), + Report( + object_type="Report", + scan_profile=None, + user_id=None, + primary_key="Report|ad33bbf1-bd35-4cb4-a61d-ebe1409e2f67", + name="Vulnerability Report", + report_type="vulnerability-report", + template="vulnerability_report/report.html", + date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137), + input_oois=["Hostname|internet|mispo.es"], + report_id=UUID("ad33bbf1-bd35-4cb4-a61d-ebe1409e2f67"), + organization_code="_rieven", + organization_name="Rieven", + organization_tags=[], + data_raw_id="1e259fce-3cd7-436f-b233-b4ae24a8f11b", + observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043), + parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"), + report_recipe=None, + has_parent=True, + ), + Report( + object_type="Report", + scan_profile=None, + user_id=None, + primary_key="Report|bd26a0c0-92c2-4323-977d-a10bd90619e7", + name="System Report", + report_type="systems-report", + template="systems_report/report.html", + date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137), + input_oois=["Hostname|internet|mispo.es"], + report_id=UUID("bd26a0c0-92c2-4323-977d-a10bd90619e7"), + organization_code="_rieven", + organization_name="Rieven", + organization_tags=[], + data_raw_id="50a9e4df-3b69-4ad8-b798-df626162db5a", + observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043), + parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"), + report_recipe=None, + has_parent=True, + ), + Report( + object_type="Report", + scan_profile=None, + user_id=None, + primary_key="Report|d8fcaa8f-65ca-4304-a18c-078767b37bcb", + name="Name Server Report", + report_type="name-server-report", + template="name_server_report/report.html", + date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137), + input_oois=["Hostname|internet|mispo.es"], + report_id=UUID("d8fcaa8f-65ca-4304-a18c-078767b37bcb"), + organization_code="_rieven", + organization_name="Rieven", + organization_tags=[], + data_raw_id="5faa3364-c8b2-4b9c-8cc8-99d8f19ccf8a", + observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043), + parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"), + report_recipe=None, + has_parent=True, + ), + ], + ) + ], + ) + return aggregate_report diff --git a/rocky/tests/reports/test_report_overview.py b/rocky/tests/reports/test_report_overview.py index c4fc478c15e..ba737aaa017 100644 --- a/rocky/tests/reports/test_report_overview.py +++ b/rocky/tests/reports/test_report_overview.py @@ -134,3 +134,35 @@ def test_report_overview_rerun_reports( assert list(request._messages)[0].message == "Rerun successful" assertContains(response, concatenated_report.name) + + +def test_aggregate_report_has_sub_reports( + rf, client_member, mock_organization_view_octopoes, mock_bytes_client, aggregate_report_with_sub_reports +): + mock_organization_view_octopoes().list_reports.return_value = aggregate_report_with_sub_reports + + aggregate_report, subreports = aggregate_report_with_sub_reports.items[0] + + response = ReportHistoryView.as_view()( + setup_request(rf.get("report_history"), client_member.user), organization_code=client_member.organization.code + ) + + assert response.status_code == 200 + + assertContains(response, "Nov. 21, 2024") + assertContains(response, "Nov. 21, 2024, 10:07 a.m.") + + assertContains(response, "expando-button icon ti-chevron-down") + + assertContains( + response, f"This report consist of {len(subreports)} subreports with the following report types and objects." + ) + + assertContains(response, f"Subreports (5/{len(subreports)})", html=True) + + assertContains(response, aggregate_report.name) + + for subreport in subreports: + assertContains(response, subreport.name) + + assertContains(response, "View all subreports")