Merge branch 'main' into feature/mula/flexible-scheduling

* main: (85 commits)
  Fix wrong solving of merge conflict (#2585)
  Add metrics collection for scheduler using prometheus (#2468)
  Hotfix for where_in queries for abstract types (#2577)
  Update django (#2587)
  Fix octopoes typing (#2555)
  Create findings report (#2393)
  Raise exception if boefje input OOI has been deleted (#2573)
  Set a timeout on hanging test ssl container (#2560)
  Feature/efficient reporting (#2516)
  Updated findings database. Removed old findings, added Impact, Source… (#2569)
  add unit test for web report (#2528)
  Add pool size config and logs (#2541)
  Quick fix for PDF table overflow (#2562)
  Fix/2527 octopoes unicode (#2558)
  Add return typing to report test fixtures (#2557)
  Sort vulnerabilities in vulnerability report (#2378)
  Disable ruff split-on-trailing-comma and update ruff (#2544)
  Select all oois triggers toggle all (#2536)
  Remove unnecessary toplevel dependencies (#2554)
  Make valid time required parameter in the octopoes API (#2543)
  ...

.github/workflows/boefjes_tests.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: [ '3.8', '3.9', '3.10', '3.11' ]
+        version: [ '3.10', '3.11' ]
 
     runs-on: ubuntu-20.04
     env:

.github/workflows/bytes_tests.yml

@@ -19,7 +19,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: [ '3.8', '3.9', '3.10', '3.11' ]
+        version: [ '3.10', '3.11' ]
 
     runs-on: ubuntu-20.04
 

.github/workflows/debian_package.yml

@@ -15,7 +15,7 @@ jobs:
     outputs:
       packages: ${{ steps.filter.outputs.changes }}
     steps:
-    - uses: dorny/paths-filter@v2
+    - uses: dorny/paths-filter@v3
       if: github.event_name != 'push'
       id: filter
       with:

.github/workflows/keiko_itest.yml

@@ -20,10 +20,10 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v5
         with:
-          python-version: 3.8
+          python-version: '3.10'
 
       - name: Install requirements-dev.txt
         run: pip install -r requirements-dev.txt

.github/workflows/keiko_test.yml

@@ -19,7 +19,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: [ '3.8', '3.9', '3.10', '3.11' ]
+        version: [ '3.10', '3.11' ]
 
     runs-on: ubuntu-22.04
 

.github/workflows/mula_tests.yml

@@ -19,7 +19,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: ['3.8', '3.9', '3.10', '3.11']
+        version: ['3.10', '3.11']
 
     runs-on: ubuntu-20.04
     env:

.github/workflows/octopoes_rtest.yml

@@ -22,7 +22,7 @@ jobs:
 
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.10'
           cache: 'pip' # caching pip dependencies
 
       - name: Install requirements-dev.txt

.github/workflows/octopoes_tests.yml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: ['3.8', '3.9', '3.10', '3.11']
+        version: ['3.10', '3.11']
 
     runs-on: ubuntu-22.04
 

.github/workflows/pre_commit_checks.yml

@@ -25,7 +25,7 @@ jobs:
     - name: Install pre-commit
       run: pip install pre-commit==3.6.0
 
-    - uses: actions/cache@v3
+    - uses: actions/cache@v4
       with:
         path: ~/.cache/pre-commit
         key: pre-commit|${{ steps.py311.outputs.python-version }}|${{ hashFiles('.pre-commit-config.yaml') }}

.github/workflows/rocky_tests.yml

@@ -20,7 +20,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: ['3.8', '3.9', '3.10', '3.11']
+        version: ['3.10', '3.11']
 
     runs-on: ubuntu-22.04
 
@@ -29,10 +29,10 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Build image
-        run: DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f rocky/base.yml -f rocky/.ci/docker-compose.yml build --build-arg USER_UID="$(id -u)" --build-arg USER_GID="$(id -g)" --build-arg PYTHON_VERSION=${{ matrix.version }} rocky_tests rocky_integration_tests
+        run: DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f rocky/.ci/docker-compose.yml build --build-arg USER_UID="$(id -u)" --build-arg USER_GID="$(id -g)" --build-arg PYTHON_VERSION=${{ matrix.version }} rocky_tests rocky_integration
 
       - name: Run tests
-        run: DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f rocky/base.yml -f rocky/.ci/docker-compose.yml run --rm rocky_tests
+        run: DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f rocky/.ci/docker-compose.yml run --rm rocky_tests
 
       - name: Run integration tests
-        run: DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f rocky/base.yml -f rocky/.ci/docker-compose.yml run --rm rocky_integration_tests
+        run: DOCKER_BUILDKIT=1 COMPOSE_DOCKER_CLI_BUILD=1 docker compose -f rocky/.ci/docker-compose.yml run --rm rocky_integration

.github/workflows/test_debian_packages_on_ubuntu.yml

@@ -253,10 +253,10 @@ jobs:
           for i in {1..15}; do curl -s http://localhost:8001/_dev/health | jq .healthy | grep true -q && s=0 && break || s=$? && sleep 1 ; done
           if [ $s != 0 ]; then echo $(curl -v http://localhost:8001/_dev/health) || true && journalctl --no-pager -u kat-octopoes.service && exit $s ; fi
 
-      - name: Set up Python 3.8
+      - name: Set up Python 3.10
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.10'
           cache: pip
 
       - name: Install Robot Framework

.gitignore

@@ -442,3 +442,5 @@ nl-kat-*
 # rpki cache
 /boefjes/boefjes/plugins/kat_rpki/rpki.json
 /boefjes/boefjes/plugins/kat_rpki/rpki-meta.json
+
+*.pstat

.pre-commit-config.yaml

@@ -20,6 +20,7 @@ repos:
   - id: end-of-file-fixer
     exclude: |
       (?x)(
+      \.svcg$ |
       ^boefjes/tests/examples/rdns-nxdomain.txt$
       )
 
@@ -55,6 +56,18 @@ repos:
   hooks:
   - id: ruff
 
+- repo: https://github.com/asottile/pyupgrade
+  rev: v3.15.0
+  hooks:
+  - id: pyupgrade
+    args: [--py310-plus]
+
+- repo: https://github.com/adamchainz/django-upgrade
+  rev: 1.16.0
+  hooks:
+  - id: django-upgrade
+    args: [--target-version, "4.2"]
+
 - repo: https://github.com/psf/black
   rev: "23.12.1"
   hooks:
@@ -67,11 +80,12 @@ repos:
     additional_dependencies: ['types-PyYAML', 'types-requests', 'types-cachetools', 'types-retry', 'pydantic', 'pynacl']
     exclude: |
       (?x)(
-      ^boefjes/ |
+      ^boefjes/boefjes/plugins |
+      ^boefjes/tools |
       ^keiko/templates |
       ^mula/whitelist\.py$ |
-      ^octopoes/ |
-      ^rocky/ |
+      ^octopoes/tools |
+      ^rocky/whitelist\.py$ |
       /tests/ |
       docs/source/conf\.py$ |
       setup\.py$
@@ -87,6 +101,7 @@ repos:
       (?x)(
       \.po$ |
       \.xml$ |
+      \.svg$ |
       poetry.lock$ |
       pyproject.toml$ |
       requirements-.*.txt$ |

Makefile

@@ -132,7 +132,7 @@ poetry-dependencies:
 	done
 
 fix-poetry-merge-conflict:
-	for path in `git diff --staged --name-only | grep "pyproject.toml" | cut -d / -f 1`; do \
+	for path in `git diff --diff-filter=U --name-only | grep "poetry.lock" | cut -d / -f 1`; do \
 		echo $$path; \
 		git restore --staged $$path/poetry.lock $$path/requirements*; \
 		git checkout --theirs $$path/poetry.lock $$path/requirements*; \

boefjes/boefjes/api.py

@@ -3,7 +3,7 @@
 import multiprocessing
 from datetime import datetime, timezone
 from enum import Enum
-from typing import List, Optional
+from uuid import UUID
 
 from fastapi import Depends, FastAPI, HTTPException, Response
 from pydantic import BaseModel, ConfigDict, Field
@@ -13,16 +13,12 @@
 from boefjes.clients.bytes_client import BytesAPIClient
 from boefjes.clients.scheduler_client import SchedulerAPIClient, TaskStatus
 from boefjes.config import settings
-from boefjes.job_handler import (
-    _find_ooi_in_past,
-    get_environment_settings,
-    get_octopoes_api_connector,
-    serialize_ooi,
-)
+from boefjes.job_handler import get_environment_settings, get_octopoes_api_connector, serialize_ooi
 from boefjes.job_models import BoefjeMeta
 from boefjes.katalogus.local_repository import LocalPluginRepository, get_local_repository
 from boefjes.plugins.models import _default_mime_types
 from octopoes.models import Reference
+from octopoes.models.exception import ObjectNotFoundException
 
 app = FastAPI(title="Boefje API")
 logger = logging.getLogger(__name__)
@@ -49,7 +45,7 @@ def run():
 
 
 class BoefjeInput(BaseModel):
-    task_id: str
+    task_id: UUID
     output_url: str
     boefje_meta: BoefjeMeta
     model_config = ConfigDict(extra="forbid")
@@ -61,14 +57,14 @@ class StatusEnum(str, Enum):
 
 
 class File(BaseModel):
-    name: Optional[str] = None
+    name: str | None = None
     content: str = Field(..., contentEncoding="base64")
-    tags: Optional[List[str]] = None
+    tags: list[str] | None = None
 
 
 class BoefjeOutput(BaseModel):
     status: StatusEnum
-    files: Optional[List[File]] = None
+    files: list[File] | None = None
 
 
 def get_scheduler_client():
@@ -90,7 +86,7 @@ async def root():
 
 @app.get("/api/v0/tasks/{task_id}", response_model=BoefjeInput)
 async def boefje_input(
-    task_id: str,
+    task_id: UUID,
     scheduler_client: SchedulerAPIClient = Depends(get_scheduler_client),
     local_repository: LocalPluginRepository = Depends(get_local_repository),
 ):
@@ -107,7 +103,7 @@ async def boefje_input(
 
 @app.post("/api/v0/tasks/{task_id}")
 async def boefje_output(
-    task_id: str,
+    task_id: UUID,
     boefje_output: BoefjeOutput,
     scheduler_client: SchedulerAPIClient = Depends(get_scheduler_client),
     bytes_client: BytesAPIClient = Depends(get_bytes_client),
@@ -162,12 +158,13 @@ def create_boefje_meta(task, local_repository):
     input_ooi = task.p_item.data.input_ooi
     arguments = {}
     if input_ooi:
-        arguments["input"] = serialize_ooi(
-            _find_ooi_in_past(
-                Reference.from_str(input_ooi),
-                get_octopoes_api_connector(organization),
-            )
-        )
+        reference = Reference.from_str(input_ooi)
+        try:
+            ooi = get_octopoes_api_connector(organization).get(reference, valid_time=datetime.now(timezone.utc))
+        except ObjectNotFoundException as e:
+            raise ObjectNotFoundException(f"Object {reference} not found in Octopoes") from e
+
+        arguments["input"] = serialize_ooi(ooi)
 
     boefje_meta = BoefjeMeta(
         id=task.id,