Merge branch 'main' into fix-typing-octopoes

boefjes/boefjes/config.py

@@ -68,6 +68,10 @@ class Settings(BaseSettings):
         validation_alias="KATALOGUS_DB_URI",
     )
 
+    db_connection_pool_size: int = Field(
+        16, description="Database connection pool size", validation_alias="KATALOGUS_DB_CONNECTION_POOL_SIZE"
+    )
+
     scheduler_api: AnyHttpUrl = Field(
         ..., examples=["http://localhost:8004"], description="Mula API URL", validation_alias="SCHEDULER_API"
     )

boefjes/boefjes/sql/db.py

@@ -16,11 +16,14 @@
 
 @cache
 def get_engine() -> Engine:
-    logger.info("Connecting to database..")
+    """Returns database engine according to config settings."""
+    db_uri = make_url(name_or_url=str(settings.katalogus_db_uri))
+    db_uri_redacted = db_uri.render_as_string(hide_password=True)
+    logger.info("Connecting to database %s with pool size %s...", db_uri_redacted, settings.db_connection_pool_size)
 
-    engine = create_engine(make_url(str(settings.katalogus_db_uri)), pool_pre_ping=True, pool_size=25)
+    engine = create_engine(url=db_uri, pool_pre_ping=True, pool_size=settings.db_connection_pool_size)
 
-    logger.info("Connected to database")
+    logger.info("Connected to database %s", db_uri_redacted)
 
     return engine
 

bytes/bytes/config.py

@@ -136,6 +136,8 @@ class Settings(BaseSettings):
         None, description="OpenTelemetry endpoint", validation_alias="SPAN_EXPORT_GRPC_ENDPOINT"
     )
 
+    db_connection_pool_size: int = Field(16, description="Database connection pool size")
+
     model_config = SettingsConfigDict(env_prefix="BYTES_")
 
     @classmethod

bytes/bytes/database/db.py

@@ -2,7 +2,7 @@
 from functools import lru_cache
 
 from sqlalchemy import create_engine
-from sqlalchemy.engine import Engine
+from sqlalchemy.engine import Engine, make_url
 from sqlalchemy.orm import declarative_base
 
 logger = logging.getLogger(__name__)
@@ -11,11 +11,13 @@
 
 
 @lru_cache(maxsize=1)
-def get_engine(db_uri: str) -> Engine:
-    logger.info("Connecting to database..")
+def get_engine(db_uri: str, pool_size: int) -> Engine:
+    """Returns database engine according to config settings."""
+    db_uri_redacted = make_url(name_or_url=str(db_uri)).render_as_string(hide_password=True)
+    logger.info("Connecting to database %s with pool size %s...", db_uri_redacted, pool_size)
 
-    engine = create_engine(db_uri, pool_pre_ping=True, pool_size=25)
+    engine = create_engine(db_uri, pool_pre_ping=True, pool_size=pool_size)
 
-    logger.info("Connected to database")
+    logger.info("Connected to database %s.", db_uri_redacted)
 
     return engine

bytes/bytes/database/sql_meta_repository.py

@@ -208,7 +208,9 @@ def _get_or_create_signing_provider(self, signing_provider_url: str | None) -> S
 def create_meta_data_repository() -> Iterator[MetaDataRepository]:
     settings = get_settings()
 
-    session = sessionmaker(bind=get_engine(str(settings.db_uri)))()
+    session = sessionmaker(
+        bind=get_engine(db_uri=str(settings.db_uri), pool_size=int(settings.db_connection_pool_size))
+    )()
     repository = SQLMetaDataRepository(
         session, create_raw_repository(settings), create_hash_repository(settings), settings
     )

bytes/tests/conftest.py

@@ -72,7 +72,7 @@ def meta_repository(
     alembicArgs = ["--config", "/app/bytes/bytes/alembic.ini", "--raiseerr", "upgrade", "head"]
     alembic.config.main(argv=alembicArgs)
 
-    engine = get_engine(str(settings.db_uri))
+    engine = get_engine(db_uri=str(settings.db_uri), pool_size=settings.db_connection_pool_size)
     session = sessionmaker(bind=engine)()
 
     yield SQLMetaDataRepository(session, raw_repository, mock_hash_repository, settings)
@@ -98,9 +98,9 @@ def bytes_api_client(settings) -> Iterator[BytesAPIClient]:
 
     yield client
 
-    sessionmaker(bind=get_engine(str(settings.db_uri)), autocommit=True)().execute(
-        ";".join([f"TRUNCATE TABLE {t} CASCADE" for t in SQL_BASE.metadata.tables])
-    )
+    sessionmaker(
+        bind=get_engine(str(settings.db_uri), pool_size=settings.db_connection_pool_size), autocommit=True
+    )().execute(";".join([f"TRUNCATE TABLE {t} CASCADE" for t in SQL_BASE.metadata.tables]))
 
 
 @pytest.fixture

mula/scheduler/config/settings.py

@@ -214,6 +214,7 @@ class Settings(BaseSettings):
     db_uri: PostgresDsn = Field(
         ..., example="postgresql://xx:xx@host:5432/scheduler", description="Scheduler Postgres DB URI"
     )
+    db_connection_pool_size: int = Field(25, description="Database connection pool size")
 
     @classmethod
     def settings_customise_sources(

mula/scheduler/storage/storage.py

@@ -5,6 +5,8 @@
 
 import sqlalchemy
 
+from scheduler.config import settings
+
 logger = logging.getLogger(__name__)
 
 
@@ -16,13 +18,18 @@ def __init__(self, dsn: str) -> None:
 
         serializer = partial(json.dumps, default=str)
 
+        db_uri_redacted = sqlalchemy.engine.make_url(name_or_url=str(dsn)).render_as_string(hide_password=True)
+        pool_size = settings.Settings().db_connection_pool_size
+
+        logger.info("Connecting to database %s with pool size %s...", db_uri_redacted, pool_size)
         self.engine = sqlalchemy.create_engine(
             dsn,
             pool_pre_ping=True,
-            pool_size=25,
+            pool_size=pool_size,
             pool_recycle=300,
             json_serializer=serializer,
         )
+        logger.info("Connected to database %s.", db_uri_redacted)
 
         if self.engine is None:
             raise Exception("Invalid datastore type")

rocky/tests/conftest.py

@@ -25,8 +25,9 @@
 from octopoes.models.ooi.dns.zone import Hostname
 from octopoes.models.ooi.findings import CVEFindingType, Finding, KATFindingType, RiskLevelSeverity
 from octopoes.models.ooi.network import IPAddressV4, IPAddressV6, IPPort, Network, Protocol
-from octopoes.models.ooi.service import Service
+from octopoes.models.ooi.service import IPService, Service
 from octopoes.models.ooi.software import Software
+from octopoes.models.ooi.web import URL, SecurityTXT, Website
 from octopoes.models.origin import Origin, OriginType
 from octopoes.models.transaction import TransactionRecord
 from rocky.scheduler import Task
@@ -453,11 +454,34 @@ def hostname(network) -> Hostname:
     return Hostname(name="example.com", network=network.reference)
 
 
+@pytest.fixture
+def url(network):
+    return URL(raw="https://example.com/", network=network.reference)
+
+
+@pytest.fixture
+def website(ip_service: IPService, hostname: Hostname):
+    return Website(
+        ip_service=ip_service.reference,
+        hostname=hostname.reference,
+    )
+
+
+@pytest.fixture
+def security_txt(website: Website, url: URL):
+    return SecurityTXT(website=website.reference, url=url.reference, security_txt="example")
+
+
 @pytest.fixture
 def service() -> Service:
     return Service(name="domain")
 
 
+@pytest.fixture
+def ip_service(ip_port: IPPort, service: Service):
+    return IPService(ip_port=ip_port.reference, service=service.reference)
+
+
 @pytest.fixture
 def software() -> Software:
     return Software(name="DICOM")
@@ -549,6 +573,22 @@ def finding() -> Finding:
     )
 
 
+@pytest.fixture
+def web_report_finding_types():
+    return [
+        KATFindingType(id="KAT-NO-CSP"),
+        KATFindingType(id="KAT-CSP-VULNERABILITIES"),
+        KATFindingType(id="KAT-NO-HTTPS-REDIRECT"),
+        KATFindingType(id="KAT-NO-CERTIFICATE"),
+        KATFindingType(id="KAT-NO-SECURITY-TXT"),
+        KATFindingType(id="KAT-UNCOMMON-OPEN-PORT"),
+        KATFindingType(id="KAT-OPEN-SYSADMIN-PORT"),
+        KATFindingType(id="KAT-OPEN-DATABASE-PORT"),
+        KATFindingType(id="KAT-CERTIFICATE-EXPIRED"),
+        KATFindingType(id="KAT-CERTIFICATE-EXPIRING-SOON"),
+    ]
+
+
 @pytest.fixture
 def no_rpki_finding_type() -> KATFindingType:
     return KATFindingType(id="KAT-NO-RPKI")

rocky/tests/reports/test_web_systems_report.py

@@ -0,0 +1,92 @@
+from reports.report_types.web_system_report.report import WebSystemReport
+
+
+def test_web_report_no_findings(mock_octopoes_api_connector, valid_time, hostname, security_txt):
+    mock_octopoes_api_connector.oois = {
+        hostname.reference: hostname,
+    }
+    mock_octopoes_api_connector.queries = {
+        "Hostname.<hostname[is ResolvedHostname].address": {
+            hostname.reference: [],
+        },
+        "Hostname.<hostname[is Website].<website[is HTTPResource].<ooi[is Finding].finding_type": {
+            hostname.reference: [],
+        },
+        "Hostname.<hostname[is Website].<website[is HTTPResource].<resource[is HTTPHeader]."
+        "<ooi[is Finding].finding_type": {
+            hostname.reference: [],
+        },
+        "Hostname.<netloc[is HostnameHTTPURL].<ooi[is Finding].finding_type": {
+            hostname.reference: [],
+        },
+        "Hostname.<hostname[is Website].<ooi[is Finding].finding_type": {
+            hostname.reference: [],
+        },
+        "Hostname.<hostname[is Website].<website[is SecurityTXT]": {
+            hostname.reference: [security_txt],
+        },
+        "Hostname.<hostname[is ResolvedHostname].address.<address[is IPPort].<ooi[is Finding].finding_type": {
+            hostname.reference: [],
+        },
+        "Hostname.<hostname[is Website].certificate.<ooi[is Finding].finding_type": {
+            hostname.reference: [],
+        },
+    }
+
+    report = WebSystemReport(mock_octopoes_api_connector)
+
+    data = report.generate_data(str(hostname.reference), valid_time)
+
+    assert bool(data["web_checks"])
+
+
+def test_web_report_all_findings(
+    mock_octopoes_api_connector, valid_time, hostname, security_txt, web_report_finding_types
+):
+    mock_octopoes_api_connector.oois = {
+        hostname.reference: hostname,
+    }
+    mock_octopoes_api_connector.queries = {
+        "Hostname.<hostname[is ResolvedHostname].address": {
+            hostname.reference: [],
+        },
+        "Hostname.<hostname[is Website].<website[is HTTPResource].<ooi[is Finding].finding_type": {
+            hostname.reference: web_report_finding_types,
+        },
+        "Hostname.<hostname[is Website].<website[is HTTPResource].<resource[is HTTPHeader]."
+        "<ooi[is Finding].finding_type": {
+            hostname.reference: web_report_finding_types,
+        },
+        "Hostname.<netloc[is HostnameHTTPURL].<ooi[is Finding].finding_type": {
+            hostname.reference: web_report_finding_types,
+        },
+        "Hostname.<hostname[is Website].<ooi[is Finding].finding_type": {
+            hostname.reference: web_report_finding_types,
+        },
+        "Hostname.<hostname[is Website].<website[is SecurityTXT]": {
+            hostname.reference: [],
+        },
+        "Hostname.<hostname[is ResolvedHostname].address.<address[is IPPort]."
+        "<ooi[is Finding].finding_type": {
+            hostname.reference: web_report_finding_types,
+        },
+        "Hostname.<hostname[is Website].certificate.<ooi[is Finding].finding_type": {
+            hostname.reference: web_report_finding_types,
+        },
+    }
+
+    report = WebSystemReport(mock_octopoes_api_connector)
+
+    data = report.generate_data(str(hostname.reference), valid_time)
+
+    checks = data["web_checks"]
+
+    assert checks.has_csp == 0
+    assert checks.has_no_csp_vulnerabilities == 0
+    assert checks.redirects_http_https == 0
+    assert checks.offers_https == 0
+    assert checks.has_security_txt == 0
+    assert checks.no_uncommon_ports == 0
+    assert checks.has_certificates == 0
+    assert checks.certificates_not_expired == 0
+    assert checks.certificates_not_expiring_soon == 0