diff --git a/boefjes/boefjes/api.py b/boefjes/boefjes/api.py index 6919b331027..15c928e5d6e 100644 --- a/boefjes/boefjes/api.py +++ b/boefjes/boefjes/api.py @@ -59,7 +59,7 @@ class StatusEnum(str, Enum): class File(BaseModel): name: str | None = None - content: str = Field(..., contentEncoding="base64") + content: str = Field(json_schema_extra={"contentEncoding": "base64"}) tags: list[str] | None = None @@ -171,6 +171,6 @@ def create_boefje_meta(task, plugin: PluginType) -> BoefjeMeta: input_ooi=input_ooi, arguments=arguments, organization=organization, - environment=get_environment_settings(task.data, plugin.schema), + environment=get_environment_settings(task.data, plugin.boefje_schema), ) return boefje_meta diff --git a/boefjes/boefjes/clients/bytes_client.py b/boefjes/boefjes/clients/bytes_client.py index b7b66bbc272..b96369065c1 100644 --- a/boefjes/boefjes/clients/bytes_client.py +++ b/boefjes/boefjes/clients/bytes_client.py @@ -74,7 +74,7 @@ def _get_token(self) -> str: @retry_with_login def save_boefje_meta(self, boefje_meta: BoefjeMeta) -> None: - response = self._session.post("/bytes/boefje_meta", content=boefje_meta.json(), headers=self.headers) + response = self._session.post("/bytes/boefje_meta", content=boefje_meta.model_dump_json(), headers=self.headers) self._verify_response(response) @@ -87,7 +87,9 @@ def get_boefje_meta(self, boefje_meta_id: str) -> BoefjeMeta: @retry_with_login def save_normalizer_meta(self, normalizer_meta: NormalizerMeta) -> None: - response = self._session.post("/bytes/normalizer_meta", content=normalizer_meta.json(), headers=self.headers) + response = self._session.post( + "/bytes/normalizer_meta", content=normalizer_meta.model_dump_json(), headers=self.headers + ) self._verify_response(response) diff --git a/boefjes/boefjes/clients/scheduler_client.py b/boefjes/boefjes/clients/scheduler_client.py index 5e07d83d0be..bdcf38f392f 100644 --- a/boefjes/boefjes/clients/scheduler_client.py +++ b/boefjes/boefjes/clients/scheduler_client.py @@ -76,7 +76,7 @@ def pop_item(self, queue: str) -> Task | None: return TypeAdapter(Task | None).validate_json(response.content) def push_item(self, queue_id: str, p_item: Task) -> None: - response = self._session.post(f"/queues/{queue_id}/push", content=p_item.json()) + response = self._session.post(f"/queues/{queue_id}/push", content=p_item.model_dump_json()) self._verify_response(response) def patch_task(self, task_id: uuid.UUID, status: TaskStatus) -> None: diff --git a/boefjes/boefjes/dependencies/plugins.py b/boefjes/boefjes/dependencies/plugins.py index 33efce2d8ed..c7b6b61ffea 100644 --- a/boefjes/boefjes/dependencies/plugins.py +++ b/boefjes/boefjes/dependencies/plugins.py @@ -156,7 +156,7 @@ def schema(self, plugin_id: str) -> dict | None: try: boefje = self.plugin_storage.boefje_by_id(plugin_id) - return boefje.schema + return boefje.boefje_schema except PluginNotFound: return self.local_repo.schema(plugin_id) diff --git a/boefjes/boefjes/job_handler.py b/boefjes/boefjes/job_handler.py index 51cdc12a09f..06f8b0eaa39 100644 --- a/boefjes/boefjes/job_handler.py +++ b/boefjes/boefjes/job_handler.py @@ -117,7 +117,7 @@ def handle(self, boefje_meta: BoefjeMeta) -> None: boefje_meta.arguments["input"] = ooi.serialize() boefje_meta.runnable_hash = plugin.runnable_hash - boefje_meta.environment = get_environment_settings(boefje_meta, plugin.schema) + boefje_meta.environment = get_environment_settings(boefje_meta, plugin.boefje_schema) mime_types = _default_mime_types(boefje_meta.boefje).union(plugin.produces) diff --git a/boefjes/boefjes/katalogus/plugins.py b/boefjes/boefjes/katalogus/plugins.py index 68dbd87ba10..77c58c29460 100644 --- a/boefjes/boefjes/katalogus/plugins.py +++ b/boefjes/boefjes/katalogus/plugins.py @@ -130,11 +130,11 @@ class BoefjeIn(BaseModel): scan_level: int = 1 consumes: set[str] = Field(default_factory=set) produces: set[str] = Field(default_factory=set) - schema: dict | None = None + boefje_schema: dict | None = None oci_image: str | None = None oci_arguments: list[str] = Field(default_factory=list) - @field_validator("schema") + @field_validator("boefje_schema") @classmethod def json_schema_valid(cls, schema: dict | None) -> dict | None: if schema is not None: diff --git a/boefjes/boefjes/models.py b/boefjes/boefjes/models.py index 62855827d5b..56d502f60d9 100644 --- a/boefjes/boefjes/models.py +++ b/boefjes/boefjes/models.py @@ -29,12 +29,12 @@ class Boefje(Plugin): scan_level: int = 1 consumes: set[str] = Field(default_factory=set) produces: set[str] = Field(default_factory=set) - schema: dict | None = None + boefje_schema: dict | None = None runnable_hash: str | None = None oci_image: str | None = None oci_arguments: list[str] = Field(default_factory=list) - @field_validator("schema") + @field_validator("boefje_schema") @classmethod def json_schema_valid(cls, schema: dict) -> dict: if schema is not None: diff --git a/boefjes/boefjes/plugins/models.py b/boefjes/boefjes/plugins/models.py index 91af854f576..af08e385971 100644 --- a/boefjes/boefjes/plugins/models.py +++ b/boefjes/boefjes/plugins/models.py @@ -59,7 +59,7 @@ class BoefjeResource: def __init__(self, path: Path, package: str): self.path = path - self.boefje: Boefje = Boefje.parse_file(path / BOEFJE_DEFINITION_FILE) + self.boefje: Boefje = Boefje.model_validate_json(path.joinpath(BOEFJE_DEFINITION_FILE).read_text()) self.boefje.runnable_hash = get_runnable_hash(self.path) self.boefje.produces = self.boefje.produces.union(set(_default_mime_types(self.boefje))) self.module: Runnable | None = None @@ -69,7 +69,7 @@ def __init__(self, path: Path, package: str): if (path / SCHEMA_FILE).exists(): try: - self.boefje.schema = json.load((path / SCHEMA_FILE).open()) + self.boefje.boefje_schema = json.load((path / SCHEMA_FILE).open()) except JSONDecodeError as e: raise ModuleException("Invalid schema file") from e except SchemaError as e: @@ -81,7 +81,7 @@ class NormalizerResource: def __init__(self, path: Path, package: str): self.path = path - self.normalizer = Normalizer.parse_file(path / NORMALIZER_DEFINITION_FILE) + self.normalizer = Normalizer.model_validate_json(path.joinpath(NORMALIZER_DEFINITION_FILE).read_text()) self.normalizer.consumes.append(f"normalizer/{self.normalizer.id}") self.module = get_runnable_module_from_package(package, ENTRYPOINT_NORMALIZERS, parameter_count=2) diff --git a/boefjes/boefjes/sql/organisation_storage.py b/boefjes/boefjes/sql/organisation_storage.py index 4591ffc574f..fd1c748feef 100644 --- a/boefjes/boefjes/sql/organisation_storage.py +++ b/boefjes/boefjes/sql/organisation_storage.py @@ -30,7 +30,7 @@ def get_all(self) -> dict[str, Organisation]: return {organisation.id: self.to_organisation(organisation) for organisation in query.all()} def create(self, organisation: Organisation) -> None: - logger.info("Saving organisation: %s", organisation.json()) + logger.info("Saving organisation: %s", organisation.model_dump_json()) organisation_in_db = self.to_organisation_in_db(organisation) self.session.add(organisation_in_db) diff --git a/boefjes/boefjes/sql/plugin_storage.py b/boefjes/boefjes/sql/plugin_storage.py index 263d035ab76..72bb444dd2e 100644 --- a/boefjes/boefjes/sql/plugin_storage.py +++ b/boefjes/boefjes/sql/plugin_storage.py @@ -35,7 +35,7 @@ def normalizer_by_id(self, normalizer_id: str) -> Normalizer: return self.to_normalizer(instance) def create_boefje(self, boefje: Boefje) -> None: - logger.info("Saving plugin: %s", boefje.json()) + logger.info("Saving plugin: %s", boefje.model_dump_json()) boefje_in_db = self.to_boefje_in_db(boefje) self.session.add(boefje_in_db) @@ -49,13 +49,14 @@ def update_boefje(self, boefje_id: str, data: dict) -> None: if instance.static: raise NotAllowed(f"Plugin with id '{boefje_id}' is static, so updating it is not allowed") + field_mapping = {"boefje_schema": "schema"} # since Boefje.boefje_schema is the same as BoefjeInDB.schema for key, value in data.items(): - setattr(instance, key, value) + setattr(instance, field_mapping.get(key, key), value) self.session.add(instance) def create_normalizer(self, normalizer: Normalizer) -> None: - logger.info("Saving plugin: %s", normalizer.json()) + logger.info("Saving plugin: %s", normalizer.model_dump_json()) normalizer_in_db = self.to_normalizer_in_db(normalizer) self.session.add(normalizer_in_db) @@ -110,7 +111,7 @@ def to_boefje_in_db(boefje: Boefje) -> BoefjeInDB: scan_level=str(boefje.scan_level), consumes=boefje.consumes, produces=boefje.produces, - schema=boefje.schema, + schema=boefje.boefje_schema, oci_image=boefje.oci_image, oci_arguments=boefje.oci_arguments, version=boefje.version, @@ -141,7 +142,7 @@ def to_boefje(boefje_in_db: BoefjeInDB) -> Boefje: scan_level=int(boefje_in_db.scan_level), consumes=boefje_in_db.consumes, produces=boefje_in_db.produces, - schema=boefje_in_db.schema, + boefje_schema=boefje_in_db.schema, oci_image=boefje_in_db.oci_image, oci_arguments=boefje_in_db.oci_arguments, version=boefje_in_db.version, diff --git a/boefjes/tests/integration/test_api.py b/boefjes/tests/integration/test_api.py index 0bda784598b..9cb36dae370 100644 --- a/boefjes/tests/integration/test_api.py +++ b/boefjes/tests/integration/test_api.py @@ -32,7 +32,7 @@ def test_filter_plugins(test_client, organisation): boefje = Boefje( id="test_plugin", name="My test boefje", static=False, oci_image="ghcr.io/minvws/openkat/nmap:latest" ) - response = test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=boefje.json()) + response = test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=boefje.model_dump_json()) assert response.status_code == 201 response = test_client.get( @@ -43,19 +43,19 @@ def test_filter_plugins(test_client, organisation): def test_cannot_add_plugin_reserved_id(test_client, organisation): boefje = Boefje(id="dns-records", name="My test boefje", static=False) - response = test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=boefje.json()) + response = test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=boefje.model_dump_json()) assert response.status_code == 400 assert response.json() == {"message": "Plugin id 'dns-records' is already used"} normalizer = Normalizer(id="kat_nmap_normalize", name="My test normalizer") - response = test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=normalizer.json()) + response = test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=normalizer.model_dump_json()) assert response.status_code == 400 assert response.json() == {"message": "Plugin id 'kat_nmap_normalize' is already used"} def test_add_boefje(test_client, organisation): boefje = Boefje(id="test_plugin", name="My test boefje", static=False) - response = test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=boefje.json()) + response = test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=boefje.model_dump_json()) assert response.status_code == 201 response = test_client.post(f"/v1/organisations/{organisation.id}/plugins", json={"a": "b"}) @@ -64,7 +64,7 @@ def test_add_boefje(test_client, organisation): response = test_client.get(f"/v1/organisations/{organisation.id}/plugins/?plugin_type=boefje") assert len(response.json()) == 45 - boefje_dict = boefje.dict() + boefje_dict = boefje.model_dump() boefje_dict["consumes"] = list(boefje_dict["consumes"]) boefje_dict["produces"] = list(boefje_dict["produces"]) @@ -74,7 +74,7 @@ def test_add_boefje(test_client, organisation): def test_delete_boefje(test_client, organisation): boefje = Boefje(id="test_plugin", name="My test boefje", static=False) - response = test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=boefje.json()) + response = test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=boefje.model_dump_json()) assert response.status_code == 201 response = test_client.delete(f"/v1/organisations/{organisation.id}/boefjes/test_plugin") @@ -85,19 +85,19 @@ def test_delete_boefje(test_client, organisation): def test_add_normalizer(test_client, organisation): normalizer = Normalizer(id="test_normalizer", name="My test normalizer", static=False) - response = test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=normalizer.json()) + response = test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=normalizer.model_dump_json()) assert response.status_code == 201 response = test_client.get(f"/v1/organisations/{organisation.id}/plugins/?plugin_type=normalizer") assert len(response.json()) == 56 response = test_client.get(f"/v1/organisations/{organisation.id}/plugins/test_normalizer") - assert response.json() == normalizer.dict() + assert response.json() == normalizer.model_dump() def test_delete_normalizer(test_client, organisation): normalizer = Normalizer(id="test_normalizer", name="My test normalizer", static=False) - response = test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=normalizer.json()) + response = test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=normalizer.model_dump_json()) assert response.status_code == 201 response = test_client.delete(f"/v1/organisations/{organisation.id}/normalizers/test_normalizer") @@ -110,7 +110,7 @@ def test_update_plugins(test_client, organisation): normalizer = Normalizer(id="norm_id", name="My test normalizer") boefje = Boefje(id="test_plugin", name="My test boefje", description="123") - test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=boefje.json()) + test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=boefje.model_dump_json()) test_client.patch(f"/v1/organisations/{organisation.id}/boefjes/{boefje.id}", json={"description": "4"}) test_client.patch(f"/v1/organisations/{organisation.id}/plugins/{boefje.id}", json={"enabled": True}) @@ -118,7 +118,7 @@ def test_update_plugins(test_client, organisation): assert response.json()["description"] == "4" assert response.json()["enabled"] is True - test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=normalizer.json()) + test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=normalizer.model_dump_json()) test_client.patch(f"/v1/organisations/{organisation.id}/normalizers/{normalizer.id}", json={"version": "v1.2"}) response = test_client.get(f"/v1/organisations/{organisation.id}/plugins/{normalizer.id}") @@ -127,7 +127,7 @@ def test_update_plugins(test_client, organisation): def test_cannot_create_boefje_with_invalid_schema(test_client, organisation): boefje = Boefje(id="test_plugin", name="My test boefje", description="123").model_dump(mode="json") - boefje["schema"] = {"$schema": 3} + boefje["boefje_schema"] = {"$schema": 3} r = test_client.post(f"/v1/organisations/{organisation.id}/plugins", json=boefje) assert r.status_code == 400 @@ -135,9 +135,11 @@ def test_cannot_create_boefje_with_invalid_schema(test_client, organisation): def test_update_boefje_schema(test_client, organisation): boefje = Boefje(id="test_plugin", name="My test boefje", description="123") - test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=boefje.json()) + test_client.post(f"/v1/organisations/{organisation.id}/plugins", content=boefje.model_dump_json()) - r = test_client.patch(f"/v1/organisations/{organisation.id}/boefjes/{boefje.id}", json={"schema": {"$schema": 3}}) + r = test_client.patch( + f"/v1/organisations/{organisation.id}/boefjes/{boefje.id}", json={"boefje_schema": {"$schema": 3}} + ) assert r.status_code == 400 valid_schema = { @@ -151,16 +153,20 @@ def test_update_boefje_schema(test_client, organisation): }, "required": [], } - r = test_client.patch(f"/v1/organisations/{organisation.id}/boefjes/{boefje.id}", json={"schema": valid_schema}) + r = test_client.patch( + f"/v1/organisations/{organisation.id}/boefjes/{boefje.id}", json={"boefje_schema": valid_schema} + ) assert r.status_code == 204 schema = test_client.get(f"/v1/organisations/{organisation.id}/plugins/{boefje.id}/schema.json").json() assert schema == valid_schema api_boefje = test_client.get(f"/v1/organisations/{organisation.id}/plugins/{boefje.id}").json() - assert api_boefje["schema"] == valid_schema + assert api_boefje["boefje_schema"] == valid_schema - r = test_client.patch(f"/v1/organisations/{organisation.id}/boefjes/dns-records", json={"schema": valid_schema}) + r = test_client.patch( + f"/v1/organisations/{organisation.id}/boefjes/dns-records", json={"boefje_schema": valid_schema} + ) assert r.status_code == 404 @@ -221,7 +227,7 @@ def test_clone_settings(test_client, organisation): # Add the second organisation new_org_id = "org2" org2 = Organisation(id=new_org_id, name="Second test Organisation") - test_client.post("/v1/organisations/", content=org2.json()) + test_client.post("/v1/organisations/", content=org2.model_dump_json()) test_client.put(f"/v1/organisations/{new_org_id}/{plug}/settings", json={"test_key": "second value"}) # Show that the second organisation has no settings and dns-records is not enabled diff --git a/boefjes/tests/plugins/test_leakix.py b/boefjes/tests/plugins/test_leakix.py index 2d964069bbf..371df6d42c8 100644 --- a/boefjes/tests/plugins/test_leakix.py +++ b/boefjes/tests/plugins/test_leakix.py @@ -1,4 +1,4 @@ -from pydantic import parse_obj_as +from pydantic import TypeAdapter from boefjes.plugins.kat_leakix.normalize import run from octopoes.models.types import OOIType @@ -6,8 +6,7 @@ def test_output(): - input_ooi = parse_obj_as( - OOIType, + input_ooi = TypeAdapter(OOIType).validate_python( { "object_type": "HostnameHTTPURL", "network": "Network|internet", diff --git a/bytes/bytes/api/models.py b/bytes/bytes/api/models.py index fddfc0dc4a3..8aa23017734 100644 --- a/bytes/bytes/api/models.py +++ b/bytes/bytes/api/models.py @@ -9,7 +9,7 @@ class RawResponse(BaseModel): class File(BaseModel): name: str - content: str = Field(..., contentEncoding="base64") + content: str = Field(json_schema_extra={"contentEncoding": "base64"}) tags: list[str] = Field(default_factory=list) diff --git a/bytes/bytes/api/router.py b/bytes/bytes/api/router.py index 394f0e959b4..09f0c8e6e76 100644 --- a/bytes/bytes/api/router.py +++ b/bytes/bytes/api/router.py @@ -285,7 +285,7 @@ def get_raw_count_per_mime_type( def ignore_arguments_key(meta_repository: MetaDataRepository, query_filter: RawDataFilter): """Helper to not cache based on the stateful meta_repository, but only use the query parameters as a key.""" - return query_filter.json() + return query_filter.model_dump_json() @cached( diff --git a/bytes/bytes/database/sql_meta_repository.py b/bytes/bytes/database/sql_meta_repository.py index 2bdbacf2c37..fa124898eed 100644 --- a/bytes/bytes/database/sql_meta_repository.py +++ b/bytes/bytes/database/sql_meta_repository.py @@ -56,7 +56,7 @@ def get_boefje_meta_by_id(self, boefje_meta_id: uuid.UUID) -> BoefjeMeta: return to_boefje_meta(boefje_meta_in_db) def get_boefje_meta(self, query_filter: BoefjeMetaFilter) -> list[BoefjeMeta]: - logger.debug("Querying boefje meta: %s", query_filter.json()) + logger.debug("Querying boefje meta: %s", query_filter.model_dump_json()) query = self.session.query(BoefjeMetaInDB).filter(BoefjeMetaInDB.organization == query_filter.organization) @@ -86,7 +86,7 @@ def get_normalizer_meta_by_id(self, normalizer_meta_id: uuid.UUID) -> Normalizer return to_normalizer_meta(normalizer_meta_in_db) def get_normalizer_meta(self, query_filter: NormalizerMetaFilter) -> list[NormalizerMeta]: - logger.debug("Querying normalizer meta: %s", query_filter.json()) + logger.debug("Querying normalizer meta: %s", query_filter.model_dump_json()) query = self.session.query(NormalizerMetaInDB) @@ -134,7 +134,7 @@ def save_raw(self, raw: RawData) -> uuid.UUID: return raw_file_in_db.id def get_raw(self, query_filter: RawDataFilter) -> list[RawDataMeta]: - logger.debug("Querying raw data: %s", query_filter.json()) + logger.debug("Querying raw data: %s", query_filter.model_dump_json()) query = self.session.query(RawFileInDB) query = query_filter.apply(query) @@ -177,7 +177,7 @@ def get_raw_file_count_per_organization(self) -> dict[str, int]: return {organization_id: count for organization_id, count in query} def get_raw_file_count_per_mime_type(self, query_filter: RawDataFilter) -> dict[str, int]: - logger.debug("Querying count raw data per mime type: %s", query_filter.json()) + logger.debug("Querying count raw data per mime type: %s", query_filter.model_dump_json()) query = self.session.query(func.unnest(RawFileInDB.mime_types), func.count()).group_by( func.unnest(RawFileInDB.mime_types) ) diff --git a/bytes/tests/client.py b/bytes/tests/client.py index 0fcdb95b8b5..83c327d0d27 100644 --- a/bytes/tests/client.py +++ b/bytes/tests/client.py @@ -91,7 +91,7 @@ def get_boefje_meta_by_id(self, boefje_meta_id: UUID) -> BoefjeMeta: self._verify_response(response) boefje_meta_json = response.json() - return BoefjeMeta.parse_obj(boefje_meta_json) + return BoefjeMeta.model_validate(boefje_meta_json) @retry_with_login def get_boefje_meta(self, query_filter: BoefjeMetaFilter) -> list[BoefjeMeta]: @@ -99,7 +99,7 @@ def get_boefje_meta(self, query_filter: BoefjeMetaFilter) -> list[BoefjeMeta]: self._verify_response(response) boefje_meta_json = response.json() - return [BoefjeMeta.parse_obj(boefje_meta) for boefje_meta in boefje_meta_json] + return [BoefjeMeta.model_validate(boefje_meta) for boefje_meta in boefje_meta_json] @retry_with_login def save_normalizer_meta(self, normalizer_meta: NormalizerMeta) -> None: @@ -113,7 +113,7 @@ def get_normalizer_meta_by_id(self, normalizer_meta_id: UUID) -> NormalizerMeta: self._verify_response(response) normalizer_meta_json = response.json() - return NormalizerMeta.parse_obj(normalizer_meta_json) + return NormalizerMeta.model_validate(normalizer_meta_json) @retry_with_login def get_normalizer_meta(self, query_filter: NormalizerMetaFilter) -> list[NormalizerMeta]: @@ -121,7 +121,7 @@ def get_normalizer_meta(self, query_filter: NormalizerMetaFilter) -> list[Normal self._verify_response(response) normalizer_meta_json = response.json() - return [NormalizerMeta.parse_obj(normalizer_meta) for normalizer_meta in normalizer_meta_json] + return [NormalizerMeta.model_validate(normalizer_meta) for normalizer_meta in normalizer_meta_json] @retry_with_login def save_raw(self, boefje_meta_id: UUID, raw: bytes, mime_types: list[str] | None = None) -> str: diff --git a/keiko/keiko/keiko.py b/keiko/keiko/keiko.py index 8526113a34b..e4efe4c9422 100644 --- a/keiko/keiko/keiko.py +++ b/keiko/keiko/keiko.py @@ -92,7 +92,7 @@ def generate_report( # load data shape and validate data_shape_class = get_data_shape(template_name, settings) - data = data_shape_class.parse_obj(report_data.dict()) + data = data_shape_class.model_validate(report_data.model_dump()) current_span.add_event("Data shape validation successful") logger.info( "Data shape validation successful. [report_id=%s] [template=%s]", @@ -136,7 +136,7 @@ def generate_report( if bare_word in glossary_entries: found_entries.add(bare_word) - context = data.dict() + context = data.model_dump() # build and merge glossary glossary_items = [] diff --git a/mula/scheduler/schedulers/boefje.py b/mula/scheduler/schedulers/boefje.py index 16aa5022832..6238e009d8c 100644 --- a/mula/scheduler/schedulers/boefje.py +++ b/mula/scheduler/schedulers/boefje.py @@ -139,7 +139,7 @@ def push_tasks_for_scan_profile_mutations(self, body: bytes) -> None: mutation: The mutation that was received. """ # Convert body into a ScanProfileMutation - mutation = ScanProfileMutation.parse_raw(body) + mutation = ScanProfileMutation.model_validate_json(body) self.logger.debug( "Received scan level mutation %s for: %s", @@ -215,7 +215,7 @@ def push_tasks_for_scan_profile_mutations(self, body: bytes) -> None: continue boefje_task = BoefjeTask( - boefje=Boefje.parse_obj(boefje.dict()), + boefje=Boefje.model_validate(boefje.model_dump()), input_ooi=ooi.primary_key if ooi else None, organization=self.organisation.id, ) @@ -301,7 +301,7 @@ def push_tasks_for_new_boefjes(self) -> None: continue boefje_task = BoefjeTask( - boefje=Boefje.parse_obj(boefje.dict()), + boefje=Boefje.model_validate(boefje.dict()), input_ooi=ooi.primary_key, organization=self.organisation.id, ) @@ -368,7 +368,7 @@ def push_tasks_for_rescheduling(self): thread_name_prefix=f"BoefjeScheduler-TPE-{self.scheduler_id}-rescheduling" ) as executor: for schedule in schedules: - boefje_task = BoefjeTask.parse_obj(schedule.data) + boefje_task = BoefjeTask.model_validate(schedule.data) # Plugin still exists? try: @@ -486,7 +486,7 @@ def push_tasks_for_rescheduling(self): continue new_boefje_task = BoefjeTask( - boefje=Boefje.parse_obj(plugin.dict()), + boefje=Boefje.model_validate(plugin.dict()), input_ooi=ooi.primary_key if ooi else None, organization=self.organisation.id, ) @@ -659,7 +659,7 @@ def push_boefje_task( def push_item_to_queue(self, item: Task) -> Task: """Some boefje scheduler specific logic before pushing the item to the queue.""" - boefje_task = BoefjeTask.parse_obj(item.data) + boefje_task = BoefjeTask.model_validate(item.data) # Check if id's are unique and correctly set. Same id's are necessary # for the task runner. diff --git a/mula/scheduler/schedulers/normalizer.py b/mula/scheduler/schedulers/normalizer.py index 9c9465c08c3..3e25c85ca4f 100644 --- a/mula/scheduler/schedulers/normalizer.py +++ b/mula/scheduler/schedulers/normalizer.py @@ -101,7 +101,7 @@ def push_tasks_for_received_raw_data(self, body: bytes) -> None: message queue. """ # Convert body into a RawDataReceivedEvent - latest_raw_data = RawDataReceivedEvent.parse_raw(body) + latest_raw_data = RawDataReceivedEvent.model_validate_json(body) self.logger.debug( "Received raw data %s", @@ -158,7 +158,7 @@ def push_tasks_for_received_raw_data(self, body: bytes) -> None: continue normalizer_task = NormalizerTask( - normalizer=Normalizer.parse_obj(normalizer.dict()), + normalizer=Normalizer.model_validate(normalizer.model_dump()), raw_data=latest_raw_data.raw_data, ) @@ -292,7 +292,7 @@ def push_normalizer_task(self, normalizer_task: models.NormalizerTask, caller: s def push_item_to_queue(self, item: Task) -> Task: """Some normalizer scheduler specific logic before pushing the item to the queue.""" - normalizer_task = NormalizerTask.parse_obj(item.data) + normalizer_task = NormalizerTask.model_validate(item.data) # Check if id's are unique and correctly set. Same id's are necessary # for the task runner. diff --git a/mula/scheduler/server/handlers/schedules.py b/mula/scheduler/server/handlers/schedules.py index 23e44c638b3..29e50b30312 100644 --- a/mula/scheduler/server/handlers/schedules.py +++ b/mula/scheduler/server/handlers/schedules.py @@ -118,7 +118,7 @@ def list( def create(self, schedule: serializers.ScheduleCreate) -> Any: try: - new_schedule = models.Schedule(**schedule.dict()) + new_schedule = models.Schedule(**schedule.model_dump()) except pydantic.ValidationError as exc: raise fastapi.HTTPException( status_code=fastapi.status.HTTP_400_BAD_REQUEST, diff --git a/mula/tests/integration/test_boefje_scheduler.py b/mula/tests/integration/test_boefje_scheduler.py index 06589ac749c..4c0e28cbde6 100644 --- a/mula/tests/integration/test_boefje_scheduler.py +++ b/mula/tests/integration/test_boefje_scheduler.py @@ -619,7 +619,7 @@ def test_push_task(self): boefje = BoefjeFactory() boefje_task = models.BoefjeTask( - boefje=models.Boefje.parse_obj(boefje.dict()), + boefje=models.Boefje.model_validate(boefje.dict()), input_ooi=ooi.primary_key, organization=self.organisation.id, ) @@ -639,7 +639,7 @@ def test_push_task_no_ooi(self): boefje = BoefjeFactory() boefje_task = models.BoefjeTask( - boefje=models.Boefje.parse_obj(boefje.dict()), + boefje=models.Boefje.model_validate(boefje.dict()), input_ooi=None, organization=self.organisation.id, ) @@ -674,7 +674,7 @@ def test_push_task_queue_full( plugin = PluginFactory(scan_level=0, consumes=[ooi.object_type]) boefje_task = models.BoefjeTask( - boefje=models.Boefje.parse_obj(plugin.dict()), + boefje=models.Boefje.model_validate(plugin.dict()), input_ooi=ooi.primary_key, organization=self.organisation.id, ) @@ -1524,7 +1524,7 @@ def test_push_tasks_for_rescheduling(self): plugin = PluginFactory(scan_level=0, consumes=[ooi.object_type]) boefje_task = models.BoefjeTask( - boefje=models.Boefje.parse_obj(plugin.model_dump()), + boefje=models.Boefje.model_validate(plugin.model_dump()), input_ooi=ooi.primary_key, organization=self.organisation.id, ) @@ -1567,7 +1567,7 @@ def test_push_tasks_for_rescheduling_no_ooi(self): plugin = PluginFactory(scan_level=0, consumes=[ooi.object_type]) boefje_task = models.BoefjeTask( - boefje=models.Boefje.parse_obj(plugin.model_dump()), + boefje=models.Boefje.model_validate(plugin.model_dump()), input_ooi=ooi.primary_key, organization=self.organisation.id, ) @@ -1608,7 +1608,7 @@ def test_push_tasks_for_rescheduling_ooi_not_found(self): plugin = PluginFactory(scan_level=0, consumes=[ooi.object_type]) boefje_task = models.BoefjeTask( - boefje=models.Boefje.parse_obj(plugin.model_dump()), + boefje=models.Boefje.model_validate(plugin.model_dump()), input_ooi=ooi.primary_key, organization=self.organisation.id, ) @@ -1644,7 +1644,7 @@ def test_push_tasks_for_rescheduling_boefje_not_found(self): plugin = PluginFactory(scan_level=0, consumes=[ooi.object_type]) boefje_task = models.BoefjeTask( - boefje=models.Boefje.parse_obj(plugin.model_dump()), + boefje=models.Boefje.model_validate(plugin.model_dump()), input_ooi=ooi.primary_key, organization=self.organisation.id, ) @@ -1680,7 +1680,7 @@ def test_push_tasks_for_rescheduling_boefje_disabled(self): plugin = PluginFactory(scan_level=0, consumes=[ooi.object_type], enabled=False) boefje_task = models.BoefjeTask( - boefje=models.Boefje.parse_obj(plugin.model_dump()), + boefje=models.Boefje.model_validate(plugin.model_dump()), input_ooi=ooi.primary_key, organization=self.organisation.id, ) @@ -1716,7 +1716,7 @@ def test_push_tasks_for_rescheduling_boefje_doesnt_consume_ooi(self): plugin = PluginFactory(scan_level=0, consumes=[]) boefje_task = models.BoefjeTask( - boefje=models.Boefje.parse_obj(plugin.model_dump()), + boefje=models.Boefje.model_validate(plugin.model_dump()), input_ooi=ooi.primary_key, organization=self.organisation.id, ) @@ -1752,7 +1752,7 @@ def test_push_tasks_for_rescheduling_boefje_cannot_scan_ooi(self): plugin = PluginFactory(scan_level=1, consumes=[ooi.object_type]) boefje_task = models.BoefjeTask( - boefje=models.Boefje.parse_obj(plugin.model_dump()), + boefje=models.Boefje.model_validate(plugin.model_dump()), input_ooi=ooi.primary_key, organization=self.organisation.id, ) diff --git a/mula/tests/integration/test_report_scheduler.py b/mula/tests/integration/test_report_scheduler.py index f0d93232a47..ab5651895bf 100644 --- a/mula/tests/integration/test_report_scheduler.py +++ b/mula/tests/integration/test_report_scheduler.py @@ -98,7 +98,7 @@ def test_push_tasks_for_rescheduling(self): schedule = models.Schedule( scheduler_id=self.scheduler.scheduler_id, hash=report_task.hash, - data=report_task.dict(), + data=report_task.model_dump(), ) schedule_db = self.mock_ctx.datastores.schedule_store.create_schedule(schedule) @@ -132,7 +132,7 @@ def test_push_tasks_for_rescheduling_item_on_queue(self): schedule = models.Schedule( scheduler_id=self.scheduler.scheduler_id, hash=report_task.hash, - data=report_task.dict(), + data=report_task.model_dump(), ) schedule_db = self.mock_ctx.datastores.schedule_store.create_schedule(schedule) diff --git a/mula/tests/integration/test_scheduler.py b/mula/tests/integration/test_scheduler.py index 3f347045fe3..15a70b73e6c 100644 --- a/mula/tests/integration/test_scheduler.py +++ b/mula/tests/integration/test_scheduler.py @@ -288,7 +288,7 @@ def test_post_push_schedule_disabled(self): ) # Act - second_item = first_item_db.copy() + second_item = first_item_db.model_copy() second_item.id = uuid.uuid4() second_item_db = self.scheduler.push_item_to_queue(second_item) @@ -313,7 +313,7 @@ def test_post_push_schedule_update_schedule(self): self.scheduler.pop_item_from_queue() # Act - second_item = first_item_db.copy() + second_item = first_item_db.model_copy() second_item.id = uuid.uuid4() second_item_db = self.scheduler.push_item_to_queue(second_item) diff --git a/mula/tests/utils/functions.py b/mula/tests/utils/functions.py index 6fc7f13cf82..5d03b984fab 100644 --- a/mula/tests/utils/functions.py +++ b/mula/tests/utils/functions.py @@ -57,7 +57,7 @@ def create_item(scheduler_id: str, priority: int, task: models.Task | None = Non task = create_task(scheduler_id) item = models.Task( - **task.dict(), + **task.model_dump(), ) if priority is not None: diff --git a/octopoes/octopoes/events/manager.py b/octopoes/octopoes/events/manager.py index 3bd3cfa9102..7b86425de09 100644 --- a/octopoes/octopoes/events/manager.py +++ b/octopoes/octopoes/events/manager.py @@ -81,7 +81,7 @@ def _publish(self, event: DBEvent) -> None: # schedule celery event processor self.celery_app.send_task( "octopoes.tasks.tasks.handle_event", - (json.loads(event.json()),), + (json.loads(event.model_dump_json()),), queue=self.celery_queue_name, task_id=str(uuid.uuid4()), ) @@ -138,7 +138,7 @@ def _publish(self, event: DBEvent) -> None: self.channel.basic_publish( "", f"{event.client}__scan_profile_mutations", - mutation.json().encode(), + mutation.model_dump_json().encode(), properties=pika.BasicProperties(delivery_mode=pika.DeliveryMode.Persistent), ) diff --git a/octopoes/octopoes/models/__init__.py b/octopoes/octopoes/models/__init__.py index bb126e24f66..44153335a2c 100644 --- a/octopoes/octopoes/models/__init__.py +++ b/octopoes/octopoes/models/__init__.py @@ -203,7 +203,7 @@ def hydrate(node) -> dict | str: node[key] = natural_key_parts.pop(0) return node - return PrimaryKeyToken.parse_obj(hydrate(token_tree)) + return PrimaryKeyToken.model_validate(hydrate(token_tree)) @classmethod def format_reference_human_readable(cls, reference: Reference) -> str: diff --git a/octopoes/octopoes/models/persistence.py b/octopoes/octopoes/models/persistence.py index d10ff1bf03a..7c28805e807 100644 --- a/octopoes/octopoes/models/persistence.py +++ b/octopoes/octopoes/models/persistence.py @@ -15,11 +15,11 @@ def ReferenceField( ) -> FieldInfo: if not isinstance(object_type, str): object_type = object_type.get_object_type() - kwargs.update( - { - "object_type": object_type, - "max_issue_scan_level": max_issue_scan_level, - "max_inherit_scan_level": max_inherit_scan_level, - } - ) - return Field(**kwargs) + + json_schema_extra = { + "object_type": object_type, + "max_issue_scan_level": max_issue_scan_level, + "max_inherit_scan_level": max_inherit_scan_level, + } + + return Field(**kwargs, json_schema_extra=json_schema_extra) diff --git a/octopoes/octopoes/repositories/ooi_repository.py b/octopoes/octopoes/repositories/ooi_repository.py index 40f05bc4fba..67be24f1467 100644 --- a/octopoes/octopoes/repositories/ooi_repository.py +++ b/octopoes/octopoes/repositories/ooi_repository.py @@ -41,8 +41,8 @@ def merge_ooi(ooi_new: OOI, ooi_old: OOI) -> tuple[OOI, bool]: - data_old = ooi_old.dict() - data_new = ooi_new.dict() + data_old = ooi_old.model_dump() + data_new = ooi_new.model_dump() # Trim new None values clean_new = {key: val for key, val in data_new.items() if val is not None} @@ -54,7 +54,7 @@ def merge_ooi(ooi_new: OOI, ooi_old: OOI) -> tuple[OOI, bool]: break data_old.update(clean_new) - return ooi_new.__class__.parse_obj(data_old), changed + return ooi_new.__class__.model_validate(data_old), changed class OOIRepository(Repository): diff --git a/octopoes/octopoes/repositories/origin_parameter_repository.py b/octopoes/octopoes/repositories/origin_parameter_repository.py index 0ff6f98f888..e23af531104 100644 --- a/octopoes/octopoes/repositories/origin_parameter_repository.py +++ b/octopoes/octopoes/repositories/origin_parameter_repository.py @@ -48,14 +48,14 @@ def commit(self): @classmethod def serialize(cls, origin_parameter: OriginParameter) -> dict[str, Any]: - data = origin_parameter.dict() + data = origin_parameter.model_dump() data[cls.pk_prefix] = origin_parameter.id data["type"] = origin_parameter.__class__.__name__ return data @classmethod def deserialize(cls, data: dict[str, Any]) -> OriginParameter: - return OriginParameter.parse_obj(data) + return OriginParameter.model_validate(data) def get(self, origin_parameter_id: str, valid_time: datetime) -> OriginParameter: try: diff --git a/octopoes/tests/test_reference.py b/octopoes/tests/test_reference.py index fccab06298a..775cc713b36 100644 --- a/octopoes/tests/test_reference.py +++ b/octopoes/tests/test_reference.py @@ -26,5 +26,5 @@ def test_ref_equality(self): self.assertEqual(a, b) def test_parse_obj(self): - ip = MockIPAddressV4.parse_obj({"address": "1.1.1.1", "network": "MockNetwork|internet"}) + ip = MockIPAddressV4.model_validate({"address": "1.1.1.1", "network": "MockNetwork|internet"}) self.assertEqual(Reference("MockNetwork|internet"), ip.network) diff --git a/octopoes/tests/test_reference_node.py b/octopoes/tests/test_reference_node.py index c3d616c108e..5dd3a2c2588 100644 --- a/octopoes/tests/test_reference_node.py +++ b/octopoes/tests/test_reference_node.py @@ -104,7 +104,7 @@ def test_filter_children(self): self.assertDictEqual({}, root_node.children["ip_addresses"][0].children) def test_xtdb_reference_node_to_reference_node(self): - root = XTDBReferenceNode.parse_obj(xtdb_sample) + root = XTDBReferenceNode.model_validate(xtdb_sample) reference_node = root.to_reference_node("xt/id") self.assertEqual( "IPAddressV6|internet|2001:1c00:2303:8f00:21c7:4dc2:5738:28af", @@ -124,13 +124,13 @@ def test_collect_references(self): Reference.from_str("IPPort|internet|1.1.1.2|tcp|80"), } - root = XTDBReferenceNode.parse_obj(xtdb_sample) + root = XTDBReferenceNode.model_validate(xtdb_sample) reference_node = root.to_reference_node("xt/id") self.assertEqual(refs, reference_node.collect_references()) def test_xtdb_data_to_reference_node_complext(self): - root = XTDBReferenceNode.parse_obj(xtdb_sample_2) + root = XTDBReferenceNode.model_validate(xtdb_sample_2) reference_node = root.to_reference_node("xt/id") self.assertEqual("DNSZone|internet|nl", str(reference_node.reference)) diff --git a/rocky/katalogus/client.py b/rocky/katalogus/client.py index df073b57282..e98f053df68 100644 --- a/rocky/katalogus/client.py +++ b/rocky/katalogus/client.py @@ -43,7 +43,7 @@ class Boefje(Plugin): produces: set[str] = Field(default_factory=set) options: list[str] | None = None runnable_hash: str | None = None - schema: dict | None = None + boefje_schema: dict | None = None oci_image: str | None = None oci_arguments: list[str] = Field(default_factory=list) @@ -243,7 +243,7 @@ def parse_boefje(boefje: dict) -> Boefje: scan_level=scan_level, consumes=consumes, produces=boefje["produces"], - schema=boefje.get("schema"), + boefje_schema=boefje.get("schema"), oci_image=boefje.get("oci_image"), oci_arguments=boefje.get("oci_arguments", []), ) diff --git a/rocky/katalogus/views/boefje_setup.py b/rocky/katalogus/views/boefje_setup.py index d20738b2721..268ad10197e 100644 --- a/rocky/katalogus/views/boefje_setup.py +++ b/rocky/katalogus/views/boefje_setup.py @@ -45,7 +45,7 @@ def form_valid(self, form): scan_level=form_data["scan_level"], consumes=input_objects, produces=produces, - schema=form_data["schema"], + boefje_schema=form_data["schema"], oci_image=form_data.get("oci_image"), oci_arguments=arguments, ) diff --git a/rocky/reports/report_types/multi_organization_report/report.py b/rocky/reports/report_types/multi_organization_report/report.py index b32146f3f55..26f4af97278 100644 --- a/rocky/reports/report_types/multi_organization_report/report.py +++ b/rocky/reports/report_types/multi_organization_report/report.py @@ -257,6 +257,6 @@ def collect_report_data( ): report_data = {} for ooi in [x for x in input_ooi_references if Reference.from_str(x).class_type == ReportData]: - report_data[ooi] = connector.get(Reference.from_str(ooi), observed_at).dict() + report_data[ooi] = connector.get(Reference.from_str(ooi), observed_at).model_dump() return report_data diff --git a/rocky/rocky/bytes_client.py b/rocky/rocky/bytes_client.py index 9ddd97ff9d7..839e9444a6f 100644 --- a/rocky/rocky/bytes_client.py +++ b/rocky/rocky/bytes_client.py @@ -28,11 +28,11 @@ def health(self) -> ServiceHealth: response = self.session.get("/health") response.raise_for_status() - return ServiceHealth.parse_obj(response.json()) + return ServiceHealth.model_validate(response.json()) @staticmethod def raw_from_declarations(declarations: list[Declaration]): - json_string = f"[{','.join([declaration.json() for declaration in declarations])}]" + json_string = f"[{','.join([declaration.model_dump_json() for declaration in declarations])}]" return json_string.encode("utf-8") diff --git a/rocky/rocky/scheduler.py b/rocky/rocky/scheduler.py index 430bd135375..fc568af042f 100644 --- a/rocky/rocky/scheduler.py +++ b/rocky/rocky/scheduler.py @@ -255,7 +255,7 @@ def push_task(self, item: Task) -> None: queue_name = f"{item.data.type}-{self.organization_code}" res = self._client.post( f"/queues/{queue_name}/push", - content=item.json(exclude_none=True), + content=item.model_dump_json(exclude_none=True), headers={"Content-Type": "application/json"}, ) res.raise_for_status() diff --git a/rocky/rocky/views/task_detail.py b/rocky/rocky/views/task_detail.py index 82b462f67f1..0ea624ac266 100644 --- a/rocky/rocky/views/task_detail.py +++ b/rocky/rocky/views/task_detail.py @@ -15,7 +15,7 @@ def get(self, request, *args, **kwargs): filename = "task_" + task_id + ".json" task_details = self.get_task_details(task_id) if task_details is not None: - response = HttpResponse(FileResponse(task_details.json()), content_type="application/json") + response = HttpResponse(FileResponse(task_details.model_dump_json()), content_type="application/json") response["Content-Disposition"] = "attachment; filename=" + filename return response diff --git a/rocky/tests/conftest.py b/rocky/tests/conftest.py index 527407ad96f..fe82fda420d 100644 --- a/rocky/tests/conftest.py +++ b/rocky/tests/conftest.py @@ -1811,7 +1811,7 @@ def boefje_dns_records(): options=None, runnable_hash=None, produces={"boefje/dns-records"}, - schema={}, + boefje_schema={}, oci_image="ghcr.io/test/image:123", oci_arguments=["-test", "-arg"], ) @@ -1835,7 +1835,7 @@ def boefje_nmap_tcp(): options=None, runnable_hash=None, produces={"boefje/nmap"}, - schema={}, + boefje_schema={}, oci_image="ghcr.io/test/image:123", oci_arguments=["-test", "-arg"], ) diff --git a/rocky/tests/objects/test_objects_graph.py b/rocky/tests/objects/test_objects_graph.py index c214fd460d5..3df3738820f 100644 --- a/rocky/tests/objects/test_objects_graph.py +++ b/rocky/tests/objects/test_objects_graph.py @@ -27,7 +27,7 @@ def test_ooi_graph(rf, client_member, mock_organization_view_octopoes): - mock_organization_view_octopoes().get_tree.return_value = ReferenceTree.parse_obj(TREE_DATA) + mock_organization_view_octopoes().get_tree.return_value = ReferenceTree.model_validate(TREE_DATA) request = setup_request(rf.get("ooi_graph", {"ooi_id": "Network|testnetwork"}), client_member.user) request.resolver_match = resolve(