Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sub reports for Aggregate Report #3852

Merged
merged 13 commits into from
Nov 21, 2024
Merged
5 changes: 5 additions & 0 deletions rocky/reports/report_types/definitions.py
Original file line number Diff line number Diff line change
@@ -17,6 +17,11 @@ class ReportPlugins(TypedDict):
optional: set[str]


class SubReportPlugins(TypedDict):
required: list[str]
optional: list[str]
Rieven marked this conversation as resolved.
Show resolved Hide resolved


def report_plugins_union(report_types: list[type["BaseReport"]]) -> ReportPlugins:
"""Take the union of the required and optional plugin sets and remove optional plugins that are required"""

67 changes: 46 additions & 21 deletions rocky/reports/views/mixins.py
Original file line number Diff line number Diff line change
@@ -13,6 +13,7 @@
from octopoes.models.ooi.reports import Report
from reports.report_types.aggregate_organisation_report.report import aggregate_reports
from reports.report_types.concatenated_report.report import ConcatenatedReport
from reports.report_types.definitions import BaseReport, SubReportPlugins
from reports.report_types.helpers import REPORTS, get_report_by_id
from reports.report_types.multi_organization_report.report import MultiOrganizationReport, collect_report_data
from reports.views.base import BaseReportView, ReportDataDict
@@ -56,6 +57,22 @@ def collect_reports(observed_at: datetime, octopoes_connector: OctopoesAPIConnec
return error_reports, report_data


def get_child_input_data(input_data: dict[str, Any], ooi: str, report_type: type[BaseReport]):
required_plugins = list(input_data["input_data"]["plugins"]["required"])
optional_plugins = list(input_data["input_data"]["plugins"]["optional"])

child_plugins: SubReportPlugins = {"required": [], "optional": []}

child_plugins["required"] = [
plugin_id for plugin_id in required_plugins if plugin_id in report_type.plugins["required"]
]
child_plugins["optional"] = [
plugin_id for plugin_id in optional_plugins if plugin_id in report_type.plugins["optional"]
]

return {"input_data": {"input_oois": [ooi], "report_types": [report_type.id], "plugins": child_plugins}}


def save_report_data(
bytes_client,
observed_at,
@@ -116,21 +133,7 @@ def save_report_data(
name_to_save = updated_name
break

required_plugins = list(input_data["input_data"]["plugins"]["required"])
optional_plugins = list(input_data["input_data"]["plugins"]["optional"])

child_plugins: dict[str, list[str]] = {"required": [], "optional": []}

child_plugins["required"] = [
plugin_id for plugin_id in required_plugins if plugin_id in report_type.plugins["required"]
]
child_plugins["optional"] = [
plugin_id for plugin_id in optional_plugins if plugin_id in report_type.plugins["optional"]
]

child_input_data = {
"input_data": {"input_oois": [ooi], "report_types": [report_type_id], "plugins": child_plugins}
}
child_input_data = get_child_input_data(input_data, ooi, report_type)

raw_id = bytes_client.upload_raw(
raw=ReportDataDict({"report_data": data["data"]} | child_input_data).model_dump_json().encode(),
@@ -208,8 +211,7 @@ def save_aggregate_report_data(
report_recipe: Reference | None = None,
) -> Report:
observed_at = get_observed_at

now = datetime.utcnow()
Rieven marked this conversation as resolved.
Show resolved Hide resolved
now = datetime.now(timezone.utc)

# Create the report
report_data_raw_id = bytes_client.upload_raw(
@@ -230,7 +232,7 @@ def save_aggregate_report_data(
organization_name=organization.name,
organization_tags=[tag.name for tag in organization.tags.all()],
data_raw_id=report_data_raw_id,
date_generated=datetime.now(timezone.utc),
date_generated=now,
input_oois=ooi_pks,
observed_at=observed_at,
parent_report=None,
@@ -240,12 +242,35 @@ def save_aggregate_report_data(
create_ooi(octopoes_api_connector, bytes_client, report_ooi, observed_at)

# Save the child reports to bytes

for ooi, types in report_data.items():
for report_type, data in types.items():
bytes_client.upload_raw(
raw=ReportDataDict(data | input_data).model_dump_json().encode(), manual_mime_types={"openkat/report"}
for report_type_id, data in types.items():
report_type = get_report_by_id(report_type_id)
child_input_data = get_child_input_data(input_data, ooi, report_type)

raw_id = bytes_client.upload_raw(
raw=ReportDataDict({"report_data": data} | child_input_data).model_dump_json().encode(),
manual_mime_types={"openkat/report"},
)

aggregate_sub_report_ooi = Report(
name=str(report_type.name),
report_type=report_type_id,
template=report_type.template_path,
report_id=uuid4(),
organization_code=organization.code,
organization_name=organization.name,
organization_tags=[tag.name for tag in organization.tags.all()],
data_raw_id=raw_id,
date_generated=now,
input_oois=[ooi],
observed_at=observed_at,
parent_report=report_ooi.reference,
has_parent=True,
)

create_ooi(octopoes_api_connector, bytes_client, aggregate_sub_report_ooi, observed_at)

return report_ooi


214 changes: 214 additions & 0 deletions rocky/tests/conftest.py
Original file line number Diff line number Diff line change
@@ -2340,3 +2340,217 @@ def get_report_input_data_from_bytes():
}
}
return json.dumps(input_data).encode("utf-8")


@pytest.fixture
def aggregate_report_with_sub_reports():
aggregate_report: Paginated[tuple[Report, list[Report | None]]] = Paginated(
count=1,
items=[
(
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|23820a64-db8f-41b7-b045-031338fbb91d",
name="Aggregate Report",
report_type="aggregate-organisation-report",
template="aggregate_organisation_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("23820a64-db8f-41b7-b045-031338fbb91d"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="3a362cd7-6348-4e91-8a6f-4cd83f9f6a83",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=None,
report_recipe=None,
has_parent=False,
),
[
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|28c7b15e-6dda-49e8-a101-41df3124287e",
name="Mail Report",
report_type="mail-report",
template="mail_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("28c7b15e-6dda-49e8-a101-41df3124287e"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="a534b4d5-5dba-4ddc-9b77-970675ae4b1c",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|2a56737f-492f-424b-88cc-0029ce2a444b",
name="IPv6 Report",
report_type="ipv6-report",
template="ipv6_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("2a56737f-492f-424b-88cc-0029ce2a444b"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="0bdea8eb-7ac0-46ef-ad14-ea3b0bfe1030",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|4ec12350-7552-40de-8c9f-f75ac04b99cb",
name="RPKI Report",
report_type="rpki-report",
template="rpki_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("4ec12350-7552-40de-8c9f-f75ac04b99cb"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="53d5452c-9e67-42d2-9cb0-3b684d8967a2",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|8137a050-f897-45ce-a695-fd21c63e2e5c",
name="Safe Connections Report",
report_type="safe-connections-report",
template="safe_connections_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("8137a050-f897-45ce-a695-fd21c63e2e5c"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="a218ca79-47de-4473-a93d-54d14baadd98",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|9ca7ad01-e19e-42c9-9361-751db4399b94",
name="Web System Report",
report_type="web-system-report",
template="web_system_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("9ca7ad01-e19e-42c9-9361-751db4399b94"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="3779f5b0-3adf-41c8-9630-8eed8a857ae6",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|a76878ba-55e0-4971-b645-63cfdfd34e78",
name="Open Ports Report",
report_type="open-ports-report",
template="open_ports_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("a76878ba-55e0-4971-b645-63cfdfd34e78"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="851feeab-7036-48f6-81ef-599467c52457",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|ad33bbf1-bd35-4cb4-a61d-ebe1409e2f67",
name="Vulnerability Report",
report_type="vulnerability-report",
template="vulnerability_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("ad33bbf1-bd35-4cb4-a61d-ebe1409e2f67"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="1e259fce-3cd7-436f-b233-b4ae24a8f11b",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|bd26a0c0-92c2-4323-977d-a10bd90619e7",
name="System Report",
report_type="systems-report",
template="systems_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("bd26a0c0-92c2-4323-977d-a10bd90619e7"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="50a9e4df-3b69-4ad8-b798-df626162db5a",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
Report(
object_type="Report",
scan_profile=None,
user_id=None,
primary_key="Report|d8fcaa8f-65ca-4304-a18c-078767b37bcb",
name="Name Server Report",
report_type="name-server-report",
template="name_server_report/report.html",
date_generated=datetime(2024, 11, 21, 10, 7, 7, 441137),
input_oois=["Hostname|internet|mispo.es"],
report_id=UUID("d8fcaa8f-65ca-4304-a18c-078767b37bcb"),
organization_code="_rieven",
organization_name="Rieven",
organization_tags=[],
data_raw_id="5faa3364-c8b2-4b9c-8cc8-99d8f19ccf8a",
observed_at=datetime(2024, 11, 21, 10, 7, 7, 441043),
parent_report=Reference("Report|23820a64-db8f-41b7-b045-031338fbb91d"),
report_recipe=None,
has_parent=True,
),
],
)
],
)
return aggregate_report
32 changes: 32 additions & 0 deletions rocky/tests/reports/test_report_overview.py
Original file line number Diff line number Diff line change
@@ -134,3 +134,35 @@ def test_report_overview_rerun_reports(
assert list(request._messages)[0].message == "Rerun successful"

assertContains(response, concatenated_report.name)


def test_aggregate_report_has_sub_reports(
rf, client_member, mock_organization_view_octopoes, mock_bytes_client, aggregate_report_with_sub_reports
):
mock_organization_view_octopoes().list_reports.return_value = aggregate_report_with_sub_reports

aggregate_report, subreports = aggregate_report_with_sub_reports.items[0]

response = ReportHistoryView.as_view()(
setup_request(rf.get("report_history"), client_member.user), organization_code=client_member.organization.code
)

assert response.status_code == 200

assertContains(response, "Nov. 21, 2024")
assertContains(response, "Nov. 21, 2024, 10:07 a.m.")

assertContains(response, "expando-button icon ti-chevron-down")

assertContains(
response, f"This report consist of {len(subreports)} subreports with the following report types and objects."
)

assertContains(response, f"Subreports (5/{len(subreports)})", html=True)

assertContains(response, aggregate_report.name)

for subreport in subreports:
assertContains(response, subreport.name)

assertContains(response, "View all subreports")