Merge branch 'main' into docs/update-userguide-objects-tasks-members-…

…settings

.github/workflows/pre_commit_checks.yml

@@ -24,7 +24,7 @@ jobs:
           cache: pip
 
       - name: Install pre-commit
-        run: pip install pre-commit==3.8.0
+        run: pip install pre-commit==4.0.1
 
       - uses: actions/cache@v4
         with:

.gitignore

@@ -449,6 +449,8 @@ docs/source/_static/mermaid.min.js
 # rpki cache
 /boefjes/boefjes/plugins/kat_rpki/rpki.json
 /boefjes/boefjes/plugins/kat_rpki/rpki-meta.json
+/boefjes/boefjes/plugins/kat_rpki/bgp.jsonl
+/boefjes/boefjes/plugins/kat_rpki/bgp-meta.json
 
 *.pstat
 **/.cache*

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: mixed-line-ending
@@ -29,13 +29,13 @@ repos:
         args: ["--autofix", "--no-ensure-ascii", "--no-sort-keys"]
 
   - repo: https://github.com/abravalheri/validate-pyproject
-    rev: v0.16
+    rev: v0.23
     hooks:
       - id: validate-pyproject
         files: pyproject.toml$
 
   - repo: https://github.com/rstcheck/rstcheck
-    rev: v6.2.1
+    rev: v6.2.4
     hooks:
       - id: rstcheck
         # https://github.com/rstcheck/rstcheck-core/issues/4
@@ -49,37 +49,37 @@ repos:
         additional_dependencies: ["rstcheck[sphinx]", "autodoc-pydantic==2.1.0"]
 
   - repo: https://github.com/MarketSquare/robotframework-tidy
-    rev: "4.11.0"
+    rev: "4.14.0"
     hooks:
       - id: robotidy
 
   - repo: https://github.com/jendrikseipp/vulture
-    rev: v2.11
+    rev: v2.13
     hooks:
       - id: vulture
         exclude: |
           /tests/
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: "v0.3.5"
+    rev: "v0.8.1"
     hooks:
       - id: ruff
       - id: ruff-format
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.2
+    rev: v3.19.0
     hooks:
       - id: pyupgrade
         args: [--py310-plus]
 
   - repo: https://github.com/adamchainz/django-upgrade
-    rev: 1.16.0
+    rev: 1.22.1
     hooks:
       - id: django-upgrade
         args: [--target-version, "5.0"]
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.9.0
+    rev: v1.13.0
     hooks:
       - id: mypy
         additional_dependencies:
@@ -92,6 +92,7 @@ repos:
           - types-python-dateutil
           - types-requests
           - types-croniter
+          - boto3-stubs[s3]
         exclude: |
           (?x)(
           ^boefjes/tools |
@@ -106,7 +107,7 @@ repos:
           )
 
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.3.0
     hooks:
       - id: codespell
         additional_dependencies: ["tomli"]
@@ -136,7 +137,7 @@ repos:
           )
 
   - repo: https://github.com/Riverside-Healthcare/djLint
-    rev: v1.34.1
+    rev: v1.36.3
     hooks:
       - id: djlint-reformat-django
         files: |
@@ -155,7 +156,7 @@ repos:
         exclude: '^rocky/rocky/templates/admin/.*\.html$'
 
   - repo: https://github.com/thibaudcolas/pre-commit-stylelint
-    rev: v16.3.1
+    rev: v16.10.0
     hooks:
       - id: stylelint
         args: [--fix]
@@ -171,13 +172,13 @@ repos:
         args: ["-e", "SC1091"]
 
   - repo: https://github.com/scop/pre-commit-shfmt
-    rev: v3.8.0-1
+    rev: v3.10.0-1
     hooks:
       - id: shfmt
         args: ["-w", "-s", "-i", "4", "-sr"]
 
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v3.1.0
+    rev: v4.0.0-alpha.8
     hooks:
       - id: prettier
         additional_dependencies:

boefjes/.ci/docker-compose.yml

@@ -25,7 +25,7 @@ services:
       dockerfile: boefjes/Dockerfile
       args:
         - ENVIRONMENT=dev
-    command: bash -c "python -m cProfile -o .ci/bench_$(date +%Y_%m_%d-%H:%M:%S).pstat -m pytest -v -m slow tests/integration"
+    command: bash -c "python -m cProfile -o .ci/bench_$(date +%Y_%m_%d-%H:%M:%S).pstat -m pytest -v -m slow tests/integration/test_bench.py::test_migration"
     depends_on:
       - ci_bytes
       - ci_octopoes

boefjes/Makefile

@@ -40,6 +40,7 @@ images:  # Build the images for the containerized boefjes
 	# docker build -f images/base.Dockerfile -t ghcr.io/minvws/openkat/dns-records --build-arg BOEFJE_PATH=./boefjes/plugins/kat_dns .
 	docker build -f ./boefjes/plugins/kat_dnssec/boefje.Dockerfile -t ghcr.io/minvws/openkat/dns-sec:latest .
 	docker build -f ./boefjes/plugins/kat_nmap_tcp/boefje.Dockerfile -t ghcr.io/minvws/openkat/nmap:latest  .
+	docker build -f ./boefjes/plugins/kat_export_http/boefje.Dockerfile -t ghcr.io/minvws/openkat/export-http:latest .
 
 
 ##
@@ -75,14 +76,21 @@ itest: ## Run the integration tests.
 	$(ci-docker-compose) build
 	$(ci-docker-compose) down --remove-orphans
 	$(ci-docker-compose) run --rm katalogus_integration
-	$(ci-docker-compose) down
+	$(ci-docker-compose) stop
 
-bench:  ## Run the report benchmark.
+migration_bench:  ## Run the migration benchmark.
 	$(ci-docker-compose) build
 	$(ci-docker-compose) down --remove-orphans
 	$(ci-docker-compose) run --rm migration_bench
 	$(ci-docker-compose) stop
 
+bench:  ## Run the other benchmarks
+	$(ci-docker-compose) build
+	$(ci-docker-compose) down --remove-orphans
+	$(ci-docker-compose) run --rm katalogus_integration \
+	python -m cProfile -o .ci/bench_$$(date +%Y_%m_%d-%H:%M:%S).pstat -m pytest -m slow --no-cov tests/integration
+	$(ci-docker-compose) stop
+
 debian12:
 	docker run --rm \
 	--env PKG_NAME=kat-boefjes \

boefjes/boefjes/__main__.py

@@ -37,7 +37,7 @@
 @click.command()
 @click.argument("worker_type", type=click.Choice([q.value for q in WorkerManager.Queue]))
 @click.option("--log-level", type=click.Choice(["DEBUG", "INFO", "WARNING", "ERROR"]), help="Log level", default="INFO")
-def cli(worker_type: str, log_level: str):
+def cli(worker_type: str, log_level: str) -> None:
     logger.setLevel(log_level)
     logger.info("Starting runtime for %s", worker_type)
 

boefjes/boefjes/api.py

@@ -33,7 +33,7 @@ def __init__(self, config: Config):
         self.server = Server(config=config)
         self.config = config
 
-    def stop(self):
+    def stop(self) -> None:
         self.terminate()
 
     def run(self, *args, **kwargs):
@@ -88,7 +88,7 @@ def boefje_input(
     task_id: UUID,
     scheduler_client: SchedulerAPIClient = Depends(get_scheduler_client),
     plugin_service: PluginService = Depends(get_plugin_service),
-):
+) -> BoefjeInput:
     task = get_task(task_id, scheduler_client)
 
     if task.status is not TaskStatus.RUNNING:
@@ -108,7 +108,7 @@ def boefje_output(
     scheduler_client: SchedulerAPIClient = Depends(get_scheduler_client),
     bytes_client: BytesAPIClient = Depends(get_bytes_client),
     plugin_service: PluginService = Depends(get_plugin_service),
-):
+) -> Response:
     task = get_task(task_id, scheduler_client)
 
     if task.status is not TaskStatus.RUNNING:
@@ -127,7 +127,7 @@ def boefje_output(
         for file in boefje_output.files:
             raw = base64.b64decode(file.content)
             # when supported, also save file.name to Bytes
-            bytes_client.save_raw(task_id, raw, mime_types.union(file.tags))
+            bytes_client.save_raw(task_id, raw, mime_types.union(file.tags) if file.tags else mime_types)
 
     if boefje_output.status == StatusEnum.COMPLETED:
         scheduler_client.patch_task(task_id, TaskStatus.COMPLETED)

boefjes/boefjes/app.py

@@ -80,7 +80,7 @@ def run(self, queue_type: WorkerManager.Queue) -> None:
 
                 raise
 
-    def _fill_queue(self, task_queue: Queue, queue_type: WorkerManager.Queue):
+    def _fill_queue(self, task_queue: Queue, queue_type: WorkerManager.Queue) -> None:
         if task_queue.qsize() > self.settings.pool_size:
             time.sleep(self.settings.worker_heartbeat)
             return
@@ -189,7 +189,7 @@ def _cleanup_pending_worker_task(self, worker: BaseProcess) -> None:
     def _worker_args(self) -> tuple:
         return self.task_queue, self.item_handler, self.scheduler_client, self.handling_tasks
 
-    def exit(self, signum: int | None = None):
+    def exit(self, signum: int | None = None) -> None:
         try:
             if signum:
                 logger.info("Received %s, exiting", signal.Signals(signum).name)
@@ -238,7 +238,7 @@ def _start_working(
     handler: Handler,
     scheduler_client: SchedulerClientInterface,
     handling_tasks: dict[int, str],
-):
+) -> None:
     logger.info("Started listening for tasks from worker[pid=%s]", os.getpid())
 
     while True:

boefjes/boefjes/dependencies/encryption.py

@@ -34,8 +34,8 @@ def __init__(self, private_key: str, public_key: str):
 
     def encode(self, contents: str) -> str:
         encrypted_contents = self.box.encrypt(contents.encode())
-        encrypted_contents = base64.b64encode(encrypted_contents)
-        return encrypted_contents.decode()
+        base64_encrypted_contents = base64.b64encode(encrypted_contents)
+        return base64_encrypted_contents.decode()
 
     def decode(self, contents: str) -> str:
         encrypted_binary = base64.b64decode(contents)

boefjes/boefjes/dependencies/plugins.py

@@ -1,4 +1,3 @@
-import contextlib
 from collections.abc import Iterator
 from pathlib import Path
 from typing import Literal
@@ -17,7 +16,6 @@
 from boefjes.storage.interfaces import (
     ConfigStorage,
     DuplicatePlugin,
-    NotFound,
     PluginNotFound,
     PluginStorage,
     SettingsNotConformingToSchema,
@@ -45,8 +43,15 @@ def __exit__(self, exc_type, exc_val, exc_tb):
 
     def get_all(self, organisation_id: str) -> list[PluginType]:
         all_plugins = self._get_all_without_enabled()
+        plugin_states = self.config_storage.get_states_for_organisation(organisation_id)
 
-        return [self._set_plugin_enabled(plugin, organisation_id) for plugin in all_plugins.values()]
+        for plugin in all_plugins.values():
+            if plugin.id not in plugin_states:
+                continue
+
+            all_plugins[plugin.id] = plugin.model_copy(update={"enabled": plugin_states[plugin.id]})
+
+        return list(all_plugins.values())
 
     def _get_all_without_enabled(self) -> dict[str, PluginType]:
         all_plugins = {plugin.id: plugin for plugin in self.plugin_storage.get_all()}
@@ -217,12 +222,6 @@ def _assert_settings_match_schema(self, all_settings: dict, plugin_id: str, orga
             except ValidationError as e:
                 raise SettingsNotConformingToSchema(plugin_id, e.message) from e
 
-    def _set_plugin_enabled(self, plugin: PluginType, organisation_id: str) -> PluginType:
-        with contextlib.suppress(KeyError, NotFound):
-            plugin.enabled = self.config_storage.is_enabled_by_id(plugin.id, organisation_id)
-
-        return plugin
-
 
 def get_plugin_service() -> Iterator[PluginService]:
     def closure(session: Session):

boefjes/boefjes/katalogus/organisations.py

@@ -3,25 +3,11 @@
 from boefjes.models import Organisation
 from boefjes.sql.db import ObjectNotFoundException
 from boefjes.sql.organisation_storage import get_organisations_store
-from boefjes.storage.interfaces import OrganisationNotFound, OrganisationStorage
+from boefjes.storage.interfaces import OrganisationStorage
 
 router = APIRouter(prefix="/organisations", tags=["organisations"])
 
 
-def check_organisation_exists(
-    organisation_id: str, storage: OrganisationStorage = Depends(get_organisations_store)
-) -> None:
-    """
-    Checks if an organisation exists, if not, creates it.
-    """
-    with storage as store:
-        try:
-            store.get_by_id(organisation_id)
-        except OrganisationNotFound:
-            add_organisation(Organisation(id=organisation_id, name=organisation_id), storage)
-            storage.get_by_id(organisation_id)
-
-
 @router.get("", response_model=dict[str, Organisation])
 def list_organisations(storage: OrganisationStorage = Depends(get_organisations_store)):
     return storage.get_all()