From 76cf97dace2fb62e06c19a8998cabf9aabed7b96 Mon Sep 17 00:00:00 2001 From: nscuro Date: Thu, 3 Oct 2024 01:51:53 +0200 Subject: [PATCH] Extend Compose setup and e2e tests with multiple Kafka providers Extends the Compose setup, which previously solely relied on Redpanda, with Apache Kafka and Tansu. Defaults to Apache Kafka (native) in accordance with #854. While a Tansu setup is added, Tansu can not yet be used with Hyades. It runs into errors when the API server tries to produce to it. Extends e2e tests to support testing again different Kafka providers. Currently supported are Apache Kafka, Apache Kafka native, and Redpanda. Tansu will be added once it's functional. Removes the `create-topics.sh` script and the `redpanda-init` container in favour of the new kafka topics init task (introduced in https://github.com/DependencyTrack/hyades-apiserver/pull/943). Signed-off-by: nscuro --- .env | 24 ++ .github/workflows/e2e-test.yml | 10 +- .gitignore | 2 +- compose.kafka-apache.yml | 47 +++ compose.kafka-redpanda.yml | 49 +++ compose.kafka-tansu.yml | 61 +++ compose.monitoring.yml | 61 +++ compose.yml | 191 +++++++++ docker-compose.yml | 395 ------------------ e2e/pom.xml | 5 + .../org/dependencytrack/e2e/AbstractE2ET.java | 97 +++-- .../e2e/VulnerabilityPolicyE2ET.java | 2 +- monitoring/redpanda-console.yml | 59 +++ scripts/create-topics.sh | 86 ---- 14 files changed, 565 insertions(+), 524 deletions(-) create mode 100644 .env create mode 100644 compose.kafka-apache.yml create mode 100644 compose.kafka-redpanda.yml create mode 100644 compose.kafka-tansu.yml create mode 100644 compose.monitoring.yml create mode 100644 compose.yml delete mode 100644 docker-compose.yml create mode 100644 monitoring/redpanda-console.yml delete mode 100644 scripts/create-topics.sh diff --git a/.env b/.env new file mode 100644 index 000000000..6c15939f8 --- /dev/null +++ b/.env @@ -0,0 +1,24 @@ +COMPOSE_FILE="compose.yml:compose.kafka-apache.yml" +# COMPOSE_FILE="compose.yml:compose.kafka-redpanda.yml" +# COMPOSE_FILE="compose.yml:compose.kafka-tansu.yml" +# COMPOSE_FILE="compose.yml:compose.kafka-apache.yml:compose.monitoring.yml" + +# Database connection details. +DB_USERNAME=dtrack +DB_PASSWORD=dtrack +DB_NAME=dtrack + +# Dependency-Track image tags. +HYADES_TAG="snapshot-native" +HYADES_APISERVER_TAG="snapshot" +HYADES_FRONTEND_TAG="snapshot" + +# Infrastructure image tags. +# Which ones are relevant depends on the compose files being used (see COMPOSE_FILE). +KAFKA_APACHE_TAG="3.8.0" +KAFKA_REDPANDA_TAG="v24.2.2" +KAFKA_TANSU_TAG="latest" +POSTGRES_TAG="17-alpine" + +# Tansu configuration. +KAFKA_TANSU_DB_NAME="tansu" \ No newline at end of file diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index 2a306dce7..7268e0d82 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -27,6 +27,13 @@ on: hyades-version: description: Version of Hyades services to test against default: snapshot + kafka-provider: + description: The Kafka provider to test against + type: choice + options: + - APACHE + - APACHE_NATIVE + - REDPANDA permissions: { } @@ -45,8 +52,9 @@ jobs: cache: maven - name: Test env: - APISERVER_VERSION: ${{ inputs.apiserver-version || 'snapshot' }} + HYADES_APISERVER_VERSION: ${{ inputs.apiserver-version || 'snapshot' }} HYADES_VERSION: ${{ inputs.hyades-version || 'snapshot' }} + KAFKA_PROVIDER: ${{ inputs.kafka-provider || 'APACHE_NATIVE' }} OSSINDEX_USERNAME: ${{ secrets.OSSINDEX_USERNAME }} OSSINDEX_TOKEN: ${{ secrets.OSSINDEX_TOKEN }} run: mvn -B --no-transfer-progress -pl e2e clean verify -Pe2e-all \ No newline at end of file diff --git a/.gitignore b/.gitignore index bdf800ab5..fe1aafde9 100644 --- a/.gitignore +++ b/.gitignore @@ -39,7 +39,7 @@ nb-configuration.xml *.rej # Local environment -.env +!.env # Secret key secret.key diff --git a/compose.kafka-apache.yml b/compose.kafka-apache.yml new file mode 100644 index 000000000..34b37f4c3 --- /dev/null +++ b/compose.kafka-apache.yml @@ -0,0 +1,47 @@ +# This file is part of Dependency-Track. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) OWASP Foundation. All Rights Reserved. +--- +services: + kafka: + image: "apache/kafka-native:${KAFKA_APACHE_TAG}" + environment: + CLUSTER_ID: 'RvQwrYegSUCkIPkaiAZQlQ' + KAFKA_NODE_ID: "1" + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT' + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT_HOST://localhost:9092,PLAINTEXT://kafka:29092' + KAFKA_PROCESS_ROLES: 'broker,controller' + KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093' + KAFKA_LISTENERS: 'CONTROLLER://:29093,PLAINTEXT_HOST://:9092,PLAINTEXT://:29092' + KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT' + KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER' + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + healthcheck: + test: [ "CMD", "nc", "-z", "kafka", "9092"] + interval: 5s + timeout: 3s + retries: 3 + ports: + - "9092:9092" + volumes: + - "kafka-data:/var/lib/kafka/data" + restart: unless-stopped + +volumes: + kafka-data: { } \ No newline at end of file diff --git a/compose.kafka-redpanda.yml b/compose.kafka-redpanda.yml new file mode 100644 index 000000000..4f8c63152 --- /dev/null +++ b/compose.kafka-redpanda.yml @@ -0,0 +1,49 @@ +# This file is part of Dependency-Track. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) OWASP Foundation. All Rights Reserved. +--- +services: + kafka: + image: "docker.redpanda.com/vectorized/redpanda:${KAFKA_REDPANDA_TAG}" + command: + - redpanda + - start + - --smp + - '1' + - --reserve-memory + - 0M + - --memory + - 512M + - --overprovisioned + - --node-id + - '0' + - --kafka-addr + - PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092 + - --advertise-kafka-addr + - PLAINTEXT://kafka:29092,OUTSIDE://localhost:9092 + healthcheck: + test: [ "CMD-SHELL", "rpk", "cluster", "health", "--exit-when-healthy" ] + interval: 5s + timeout: 3s + retries: 3 + ports: + - "9092:9092" + volumes: + - "redpanda-data:/var/lib/redpanda/data" + restart: unless-stopped + +volumes: + redpanda-data: { } \ No newline at end of file diff --git a/compose.kafka-tansu.yml b/compose.kafka-tansu.yml new file mode 100644 index 000000000..21957c332 --- /dev/null +++ b/compose.kafka-tansu.yml @@ -0,0 +1,61 @@ +# This file is part of Dependency-Track. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) OWASP Foundation. All Rights Reserved. +--- +services: + tansu-init: + image: "postgres:${POSTGRES_TAG}" + entrypoint: /bin/bash + command: >- + -c '((psql -d ${KAFKA_TANSU_DB_NAME} -q -c "select 1 from cluster limit 1" >/dev/null 2>/dev/null) && echo "Already initialized") + || ((psql -c "CREATE DATABASE ${KAFKA_TANSU_DB_NAME}") && (wget -q -O- https://raw.githubusercontent.com/tansu-io/tansu/refs/heads/main/work-dir/initdb.d/010-schema.sql | psql -d tansu))' + depends_on: + postgres: + condition: service_healthy + environment: + PGHOST: "postgres" + PGUSER: "${DB_USERNAME}" + PGPASSWORD: "${DB_PASSWORD}" + restart: on-failure + + kafka: + image: "ghcr.io/tansu-io/tansu:${KAFKA_TANSU_TAG}" + command: >- + --kafka-cluster-id RvQwrYegSUCkIPkaiAZQlQ + --kafka-node-id 1 + --kafka-listener-url tcp://0.0.0.0:29092 + --kafka-advertised-listener-url tcp://kafka:29092 + --storage-engine pg=postgres://${DB_USERNAME}:${DB_PASSWORD}@postgres/${KAFKA_TANSU_DB_NAME} + --work-dir /data + depends_on: + postgres: + condition: service_healthy + tansu-init: + condition: service_completed_successfully + environment: + RUST_BACKTRACE: "1" + RUST_LOG: "info" + healthcheck: + test: [ "CMD", "/tansu-server", "-h" ] + interval: 5s + timeout: 3s + retries: 3 + volumes: + - "tansu-data:/data" + restart: unless-stopped + +volumes: + tansu-data: { } \ No newline at end of file diff --git a/compose.monitoring.yml b/compose.monitoring.yml new file mode 100644 index 000000000..dc0070d79 --- /dev/null +++ b/compose.monitoring.yml @@ -0,0 +1,61 @@ +# This file is part of Dependency-Track. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) OWASP Foundation. All Rights Reserved. +--- +services: + redpanda-console: + image: docker.redpanda.com/vectorized/console:v2.7.0 + depends_on: + kafka: + condition: service_healthy + environment: + CONFIG_FILEPATH: "/tmp/config.yml" + ports: + - "127.0.0.1:28080:8080" + volumes: + - "./monitoring/redpanda-console:/tmp/config.yml:ro" + - "./proto/src/main/proto:/etc/protos:ro" + restart: unless-stopped + + prometheus: + image: prom/prometheus:v2.53.1 + container_name: dt-prometheus + ports: + - "127.0.0.1:9090:9090" + volumes: + - "./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro" + - "prometheus-data:/prometheus" + restart: unless-stopped + + grafana: + image: grafana/grafana-oss:11.1.3 + container_name: dt-grafana + depends_on: + - prometheus + environment: + GF_SECURITY_ADMIN_USER: "admin" + GF_SECURITY_ADMIN_PASSWORD: "admin" + ports: + - "127.0.0.1:3000:3000" + volumes: + - "grafana-data:/var/lib/grafana" + - "./monitoring/grafana/dashboards:/etc/dashboards:ro" + - "./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro" + restart: unless-stopped + +volumes: + grafana-data: { } + prometheus-data: { } \ No newline at end of file diff --git a/compose.yml b/compose.yml new file mode 100644 index 000000000..6fb947d6e --- /dev/null +++ b/compose.yml @@ -0,0 +1,191 @@ +# This file is part of Dependency-Track. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) OWASP Foundation. All Rights Reserved. +--- +x-db-connection-config: &dbConnectionCfg + # For API server and initializer. + ALPINE_DATABASE_URL: "jdbc:postgresql://postgres:5432/${DB_NAME}?reWriteBatchedInserts=true" + ALPINE_DATABASE_USERNAME: "${DB_USERNAME}" + ALPINE_DATABASE_PASSWORD: "${DB_PASSWORD}" + # For Quarkus-based services. + QUARKUS_DATASOURCE_JDBC_URL: "jdbc:postgresql://postgres:5432/${DB_NAME}" + QUARKUS_DATASOURCE_USERNAME: "${DB_USERNAME}" + QUARKUS_DATASOURCE_PASSWORD: "${DB_PASSWORD}" + # For PostgreSQL itself. + POSTGRES_DB: "${DB_NAME}" + POSTGRES_USER: "${DB_USERNAME}" + POSTGRES_PASSWORD: "${DB_PASSWORD}" +x-kafka-connection-config: &kafkaConnectionCfg + KAFKA_BOOTSTRAP_SERVERS: "kafka:29092" +x-secret-key-config: &secretKeyCfg + # For API server. + ALPINE_SECRET_KEY_PATH: "/var/run/secrets/.dependency-track/keys/secret.key" + # For Quarkus-based services. + SECRET_KEY_PATH: "/var/run/secrets/.dependency-track/keys/secret.key" + +services: + initializer: + image: "ghcr.io/dependencytrack/hyades-apiserver:${HYADES_APISERVER_TAG}" + depends_on: + postgres: + condition: service_healthy + environment: + <<: [ *dbConnectionCfg, *kafkaConnectionCfg ] + JAVA_OPTIONS: "-Xmx256m -XX:+UseSerialGC -XX:TieredStopAtLevel=1" + ALPINE_DATABASE_POOL_ENABLED: "false" + INIT_TASKS_ENABLED: "true" + INIT_TASKS_KAFKA_TOPICS_ENABLED: "true" + INIT_AND_EXIT: "true" + volumes: + - "secret-data:/data" + restart: on-failure + + apiserver: + image: "ghcr.io/dependencytrack/hyades-apiserver:${HYADES_APISERVER_TAG}" + depends_on: + postgres: + condition: service_healthy + kafka: + condition: service_healthy + initializer: + condition: service_completed_successfully + environment: + <<: [ *dbConnectionCfg, *kafkaConnectionCfg, *secretKeyCfg ] + # Limit maximum heap size to 1GB. + # Default would be 80% of available memory, + # which can cause problems on some workstations. + # For production deployments, the default should be used. + EXTRA_JAVA_OPTIONS: "-Xmx1g" + ALPINE_METRICS_ENABLED: "true" + INTEGRITY_CHECK_ENABLED: "true" + INIT_TASKS_ENABLED: "false" + ports: + - "127.0.0.1:8080:8080" + volumes: + - "apiserver-data:/data" + - "secret-data:/var/run/secrets:ro" + profiles: + - demo + restart: unless-stopped + + frontend: + image: "ghcr.io/dependencytrack/hyades-frontend:${HYADES_FRONTEND_TAG}" + environment: + API_BASE_URL: "http://localhost:8080" + ports: + - "127.0.0.1:8081:8080" + profiles: + - demo + restart: unless-stopped + + notification-publisher: + image: "ghcr.io/dependencytrack/hyades-notification-publisher:${HYADES_TAG}" + depends_on: + postgres: + condition: service_healthy + kafka: + condition: service_healthy + initializer: + condition: service_completed_successfully + environment: + << : [ *dbConnectionCfg, *kafkaConnectionCfg, *secretKeyCfg ] + profiles: + - demo + volumes: + - "secret-data:/var/run/secrets:ro" + restart: unless-stopped + + repo-meta-analyzer: + image: "ghcr.io/dependencytrack/hyades-repository-meta-analyzer:${HYADES_TAG}" + depends_on: + postgres: + condition: service_healthy + kafka: + condition: service_healthy + initializer: + condition: service_completed_successfully + environment: + << : [ *dbConnectionCfg, *kafkaConnectionCfg, *secretKeyCfg ] + KAFKA_STREAMS_NUM_STREAM_THREADS: "6" # Default number of input partitions is 6 + profiles: + - demo + volumes: + - "secret-data:/var/run/secrets:ro" + restart: unless-stopped + + vuln-analyzer: + image: "ghcr.io/dependencytrack/hyades-vulnerability-analyzer:${HYADES_TAG}" + depends_on: + postgres: + condition: service_healthy + kafka: + condition: service_healthy + initializer: + condition: service_completed_successfully + environment: + << : [ *dbConnectionCfg, *kafkaConnectionCfg ] + KAFKA_STREAMS_NUM_STREAM_THREADS: "12" # Default number of input partitions is 12 + SCANNER_INTERNAL_ENABLED: "true" + # SCANNER_INTERNAL_TOPIC_PARTITIONS: "3" + SCANNER_OSSINDEX_ENABLED: "true" + # SCANNER_OSSINDEX_TOPIC_PARTITIONS: "3" + # SCANNER_OSSINDEX_API_USERNAME: "email@example.com" + # SCANNER_OSSINDEX_API_TOKEN: "your-token" + # SCANNER_SNYK_ENABLED: "true" + # SCANNER_SNYK_TOPIC_PARTITIONS: "3" + # SCANNER_SNYK_API_ORG_ID: "your-org-id" + # SCANNER_SNYK_API_TOKENS: "your-token-1,your-token-2" + profiles: + - demo + restart: unless-stopped + + mirror-service: + image: "ghcr.io/dependencytrack/hyades-mirror-service:${HYADES_TAG}" + depends_on: + postgres: + condition: service_healthy + kafka: + condition: service_healthy + initializer: + condition: service_completed_successfully + environment: + << : [ *dbConnectionCfg, *kafkaConnectionCfg, *secretKeyCfg ] + KAFKA_STREAMS_NUM_STREAM_THREADS: "3" + profiles: + - demo + volumes: + - "secret-data:/var/run/secrets:ro" + restart: unless-stopped + + postgres: + image: "postgres:${POSTGRES_TAG}" + environment: + << : *dbConnectionCfg + healthcheck: + test: [ "CMD-SHELL", "pg_isready -U ${DB_USERNAME} -d ${DB_NAME}" ] + interval: 5s + timeout: 3s + retries: 3 + ports: + - "5432:5432" + volumes: + - "postgres-data:/var/lib/postgresql/data" + restart: unless-stopped + +volumes: + apiserver-data: { } + secret-data: { } + postgres-data: { } diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 123a440f9..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,395 +0,0 @@ -# This file is part of Dependency-Track. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) OWASP Foundation. All Rights Reserved. -services: - notification-publisher: - image: ghcr.io/dependencytrack/hyades-notification-publisher:snapshot-native - depends_on: - postgres: - condition: service_healthy - redpanda: - condition: service_healthy - initializer: - condition: service_completed_successfully - secret-init: - condition: service_completed_successfully - environment: - KAFKA_BOOTSTRAP_SERVERS: "dt-redpanda:29092" - QUARKUS_DATASOURCE_JDBC_URL: "jdbc:postgresql://dt-postgres:5432/dtrack" - QUARKUS_DATASOURCE_USERNAME: "dtrack" - QUARKUS_DATASOURCE_PASSWORD: "dtrack" - SECRET_KEY_PATH: "/var/run/secrets/secret.key" - ports: - # Dynamic host port binding to allow for scaling of the service. - # Scaling with Compose doesn't work when assigning static host ports. - - "8090" - profiles: - - demo - volumes: - - "secret-data:/var/run/secrets:ro" - restart: unless-stopped - - repo-meta-analyzer: - image: ghcr.io/dependencytrack/hyades-repository-meta-analyzer:snapshot-native - depends_on: - postgres: - condition: service_healthy - redpanda: - condition: service_healthy - initializer: - condition: service_completed_successfully - secret-init: - condition: service_completed_successfully - environment: - KAFKA_BOOTSTRAP_SERVERS: "dt-redpanda:29092" - KAFKA_STREAMS_NUM_STREAM_THREADS: "6" # Default number of input partitions is 6 - QUARKUS_DATASOURCE_JDBC_URL: "jdbc:postgresql://dt-postgres:5432/dtrack" - QUARKUS_DATASOURCE_USERNAME: "dtrack" - QUARKUS_DATASOURCE_PASSWORD: "dtrack" - SECRET_KEY_PATH: "/var/run/secrets/secret.key" - ports: - # Dynamic host port binding to allow for scaling of the service. - # Scaling with Compose doesn't work when assigning static host ports. - - "8091" - profiles: - - demo - volumes: - - "secret-data:/var/run/secrets:ro" - restart: unless-stopped - - vuln-analyzer: - image: ghcr.io/dependencytrack/hyades-vulnerability-analyzer:snapshot-native - depends_on: - postgres: - condition: service_healthy - redpanda: - condition: service_healthy - initializer: - condition: service_completed_successfully - environment: - KAFKA_BOOTSTRAP_SERVERS: "dt-redpanda:29092" - KAFKA_STREAMS_NUM_STREAM_THREADS: "12" # Default number of input partitions is 12 - QUARKUS_DATASOURCE_JDBC_URL: "jdbc:postgresql://dt-postgres:5432/dtrack" - QUARKUS_DATASOURCE_USERNAME: "dtrack" - QUARKUS_DATASOURCE_PASSWORD: "dtrack" - SCANNER_INTERNAL_ENABLED: "true" - # SCANNER_INTERNAL_TOPIC_PARTITIONS: "3" - SCANNER_OSSINDEX_ENABLED: "true" - # SCANNER_OSSINDEX_TOPIC_PARTITIONS: "3" - # SCANNER_OSSINDEX_API_USERNAME: "email@example.com" - # SCANNER_OSSINDEX_API_TOKEN: "your-token" - # SCANNER_SNYK_ENABLED: "true" - # SCANNER_SNYK_TOPIC_PARTITIONS: "3" - # SCANNER_SNYK_API_ORG_ID: "your-org-id" - # SCANNER_SNYK_API_TOKENS: "your-token-1,your-token-2" - ports: - # Dynamic host port binding to allow for scaling of the service. - # Scaling with Compose doesn't work when assigning static host ports. - - "8092" - profiles: - - demo - restart: unless-stopped - - mirror-service: - image: ghcr.io/dependencytrack/hyades-mirror-service:snapshot-native - depends_on: - postgres: - condition: service_healthy - redpanda: - condition: service_healthy - initializer: - condition: service_completed_successfully - secret-init: - condition: service_completed_successfully - environment: - KAFKA_BOOTSTRAP_SERVERS: "dt-redpanda:29092" - KAFKA_STREAMS_NUM_STREAM_THREADS: "3" - QUARKUS_DATASOURCE_JDBC_URL: "jdbc:postgresql://dt-postgres:5432/dtrack" - QUARKUS_DATASOURCE_USERNAME: "dtrack" - QUARKUS_DATASOURCE_PASSWORD: "dtrack" - SECRET_KEY_PATH: "/var/run/secrets/secret.key" - ports: - # Dynamic host port binding to allow for scaling of the service. - # Scaling with Compose doesn't work when assigning static host ports. - - "8093" - profiles: - - demo - volumes: - - "secret-data:/var/run/secrets:ro" - restart: unless-stopped - - initializer: - image: ghcr.io/dependencytrack/hyades-apiserver:snapshot - container_name: dt-initializer - depends_on: - postgres: - condition: service_healthy - environment: - EXTRA_JAVA_OPTIONS: "-Xmx256m" - ALPINE_DATABASE_URL: "jdbc:postgresql://dt-postgres:5432/dtrack" - ALPINE_DATABASE_USERNAME: "dtrack" - ALPINE_DATABASE_PASSWORD: "dtrack" - ALPINE_DATABASE_POOL_ENABLED: "false" - INIT_TASKS_ENABLED: "true" - INIT_AND_EXIT: "true" - restart: on-failure - - apiserver: - image: ghcr.io/dependencytrack/hyades-apiserver:snapshot - container_name: dt-apiserver - depends_on: - postgres: - condition: service_healthy - redpanda: - condition: service_healthy - initializer: - condition: service_completed_successfully - secret-init: - condition: service_completed_successfully - environment: - # Limit maximum heap size to 2GB. - # Default would be 90% of available memory, - # which can cause problems on some workstations. - # For production deployments, the default should be used. - EXTRA_JAVA_OPTIONS: "-Xmx2g" - ALPINE_DATABASE_URL: "jdbc:postgresql://dt-postgres:5432/dtrack?reWriteBatchedInserts=true" - ALPINE_DATABASE_USERNAME: "dtrack" - ALPINE_DATABASE_PASSWORD: "dtrack" - ALPINE_METRICS_ENABLED: "true" - ALPINE_SECRET_KEY_PATH: "/var/run/secrets/secret.key" - KAFKA_BOOTSTRAP_SERVERS: "dt-redpanda:29092" - INTEGRITY_CHECK_ENABLED: "true" - INIT_TASKS_ENABLED: "false" - ports: - - "127.0.0.1:8080:8080" - volumes: - - "apiserver-data:/data" - - "secret-data:/var/run/secrets:ro" - profiles: - - demo - restart: unless-stopped - - frontend: - image: ghcr.io/dependencytrack/hyades-frontend:snapshot - container_name: dt-frontend - environment: - API_BASE_URL: "http://localhost:8080" - ports: - - "127.0.0.1:8081:8080" - profiles: - - demo - restart: unless-stopped - - postgres: - image: postgres:16-alpine - container_name: dt-postgres - environment: - POSTGRES_DB: "dtrack" - POSTGRES_USER: "dtrack" - POSTGRES_PASSWORD: "dtrack" - healthcheck: - test: [ "CMD-SHELL", "pg_isready -U $${POSTGRES_USER} -d $${POSTGRES_DB}" ] - interval: 5s - timeout: 3s - retries: 3 - ports: - - "5432:5432" - volumes: - - "postgres-data:/var/lib/postgresql/data" - restart: unless-stopped - - redpanda: - image: docker.redpanda.com/vectorized/redpanda:v24.2.2 - container_name: dt-redpanda - command: - - redpanda - - start - - --smp - - '1' - - --reserve-memory - - 0M - - --memory - - 512M - - --overprovisioned - - --node-id - - '0' - - --kafka-addr - - PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092,MINIKUBE://0.0.0.0:9093 - - --advertise-kafka-addr - - PLAINTEXT://dt-redpanda:29092,OUTSIDE://localhost:9092,MINIKUBE://host.minikube.internal:9093 - - --pandaproxy-addr - - PLAINTEXT://0.0.0.0:28082,OUTSIDE://0.0.0.0:8082 - - --advertise-pandaproxy-addr - - PLAINTEXT://dt-redpanda:28082,OUTSIDE://localhost:8082 - healthcheck: - test: [ "CMD-SHELL", "rpk", "cluster", "health", "--exit-when-healthy" ] - interval: 5s - timeout: 3s - retries: 3 - ports: - # Kafka API (for use from localhost) - - "9092:9092" - # Kafka API (for use from minikube) - - "9093:9093" - volumes: - - "redpanda-data:/var/lib/redpanda/data" - restart: unless-stopped - - redpanda-init: - image: docker.redpanda.com/vectorized/redpanda:v24.2.2 - container_name: dt-redpanda-init - depends_on: - redpanda: - condition: service_healthy - entrypoint: "/bin/bash" - command: "/tmp/create-topics.sh" - user: "0" # Ensure user can read create-topics.sh - environment: - REDPANDA_BROKERS: "dt-redpanda:29092" - # KAFKA_TOPIC_PREFIX: "" - # NOTIFICATION_TOPICS_PARTITIONS: "3" - # NOTIFICATION_TOPICS_RETENTION_MS: "43200000" # 12h - # REPO_META_ANALYSIS_TOPICS_PARTITIONS: "3" - # REPO_META_ANALYSIS_TOPICS_RETENTION_MS: "43200000" # 12h - # VULN_ANALYSIS_TOPICS_PARTITIONS: "3" - # VULN_ANALYSIS_TOPICS_RETENTION_MS: "43200000" # 12h - # VULN_MIRROR_TOPICS_PARTITIONS: "3" - # VULN_MIRROR_TOPICS_RETENTION_MS: "43200000" # 12h - volumes: - - "./scripts/create-topics.sh:/tmp/create-topics.sh:ro" - restart: on-failure - - redpanda-console: - image: docker.redpanda.com/vectorized/console:v2.7.0 - container_name: dt-redpanda-console - entrypoint: "/bin/sh" - command: "-c 'echo \"$$CONSOLE_CONFIG_FILE\" > \"$$CONFIG_FILEPATH\"; /app/console'" - depends_on: - redpanda: - condition: service_healthy - environment: - CONFIG_FILEPATH: "/tmp/config.yml" - CONSOLE_CONFIG_FILE: | - kafka: - brokers: ["redpanda:29092"] - protobuf: - enabled: true - mappings: - - topicName: dtrack.notification.analyzer - valueProtoType: org.dependencytrack.notification.v1.Notification - - topicName: dtrack.notification.bom - valueProtoType: org.dependencytrack.notification.v1.Notification - - topicName: dtrack.notification.configuration - valueProtoType: org.dependencytrack.notification.v1.Notification - - topicName: dtrack.notification.datasource-mirroring - valueProtoType: org.dependencytrack.notification.v1.Notification - - topicName: dtrack.notification.file-system - valueProtoType: org.dependencytrack.notification.v1.Notification - - topicName: dtrack.notification.integration - valueProtoType: org.dependencytrack.notification.v1.Notification - - topicName: dtrack.notification.new-vulnerability - valueProtoType: org.dependencytrack.notification.v1.Notification - - topicName: dtrack.notification.new-vulnerable-dependency - valueProtoType: org.dependencytrack.notification.v1.Notification - - topicName: dtrack.notification.policy-violation - valueProtoType: org.dependencytrack.notification.v1.Notification - - topicName: dtrack.notification.project-audit-change - valueProtoType: org.dependencytrack.notification.v1.Notification - - topicName: dtrack.notification.project-created - valueProtoType: org.dependencytrack.notification.v1.Notification - - topicName: dtrack.notification.project-vuln-analysis-complete - valueProtoType: org.dependencytrack.notification.v1.Notification - - topicName: dtrack.notification.repository - valueProtoType: org.dependencytrack.notification.v1.Notification - - topicName: dtrack.notification.vex - valueProtoType: org.dependencytrack.notification.v1.Notification - - topicName: dtrack.repo-meta-analysis.component - valueProtoType: org.dependencytrack.repometaanalysis.v1.AnalysisCommand - - topicName: dtrack.repo-meta-analysis.result - valueProtoType: org.dependencytrack.repometaanalysis.v1.AnalysisResult - - topicName: dtrack.vuln-analysis.component - keyProtoType: org.dependencytrack.vulnanalysis.v1.ScanKey - valueProtoType: org.dependencytrack.vulnanalysis.v1.ScanCommand - - topicName: dtrack.vuln-analysis.scanner.result - keyProtoType: org.dependencytrack.vulnanalysis.v1.ScanKey - valueProtoType: org.dependencytrack.vulnanalysis.v1.ScannerResult - - topicName: dtrack.vuln-analysis.result - keyProtoType: org.dependencytrack.vulnanalysis.v1.ScanKey - valueProtoType: org.dependencytrack.vulnanalysis.v1.ScanResult - - topicName: dtrack.vuln-analysis.result.processed - valueProtoType: org.dependencytrack.vulnanalysis.v1.ScanResult - - topicName: dtrack.vulnerability - valueProtoType: org.cyclonedx.v1_6.Bom - - topicName: dtrack.epss - valueProtoType: org.dependencytrack.mirror.v1.EpssItem - - topicName: dtrack.notification.user - valueProtoType: org.dependencytrack.notification.v1.Notification - fileSystem: - enabled: true - paths: ["/etc/protos"] - refreshInterval: 5m - ports: - - "127.0.0.1:28080:8080" - volumes: - - "./proto/src/main/proto:/etc/protos:ro" - restart: unless-stopped - - secret-init: - image: alpine:latest - command: "/bin/sh -c 'if [ ! -f /tmp/secret/secret.key ]; then head -c 32 /dev/urandom > /tmp/secret/secret.key; fi'" - profiles: - - demo - volumes: - - "secret-data:/tmp/secret" - restart: on-failure - - prometheus: - image: prom/prometheus:v2.53.1 - container_name: dt-prometheus - ports: - - "127.0.0.1:9090:9090" - volumes: - - "./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro" - - "prometheus-data:/prometheus" - profiles: - - monitoring - restart: unless-stopped - - grafana: - image: grafana/grafana-oss:11.1.3 - container_name: dt-grafana - depends_on: - - prometheus - environment: - GF_SECURITY_ADMIN_USER: "admin" - GF_SECURITY_ADMIN_PASSWORD: "admin" - ports: - - "127.0.0.1:3000:3000" - volumes: - - "grafana-data:/var/lib/grafana" - - "./monitoring/grafana/dashboards:/etc/dashboards:ro" - - "./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro" - profiles: - - monitoring - restart: unless-stopped - -volumes: - apiserver-data: { } - secret-data: { } - postgres-data: { } - redpanda-data: { } - grafana-data: { } - prometheus-data: { } diff --git a/e2e/pom.xml b/e2e/pom.xml index 9c38104ad..a619ac920 100644 --- a/e2e/pom.xml +++ b/e2e/pom.xml @@ -86,6 +86,11 @@ 3.9.1 test + + org.testcontainers + kafka + test + org.testcontainers postgresql diff --git a/e2e/src/test/java/org/dependencytrack/e2e/AbstractE2ET.java b/e2e/src/test/java/org/dependencytrack/e2e/AbstractE2ET.java index 5f432ec7e..194bec46c 100644 --- a/e2e/src/test/java/org/dependencytrack/e2e/AbstractE2ET.java +++ b/e2e/src/test/java/org/dependencytrack/e2e/AbstractE2ET.java @@ -31,7 +31,6 @@ import org.junit.jupiter.api.BeforeEach; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.testcontainers.containers.BindMode; import org.testcontainers.containers.GenericContainer; import org.testcontainers.containers.Network; import org.testcontainers.containers.PostgreSQLContainer; @@ -39,6 +38,7 @@ import org.testcontainers.containers.startupcheck.OneShotStartupCheckStrategy; import org.testcontainers.containers.wait.strategy.Wait; import org.testcontainers.images.PullPolicy; +import org.testcontainers.kafka.KafkaContainer; import org.testcontainers.utility.DockerImageName; import org.testcontainers.utility.MountableFile; @@ -54,23 +54,35 @@ public class AbstractE2ET { - protected static DockerImageName POSTGRES_IMAGE = DockerImageName.parse("postgres:15-alpine"); - protected static DockerImageName REDPANDA_IMAGE = DockerImageName.parse("docker.redpanda.com/vectorized/redpanda:v24.2.2"); - protected static DockerImageName API_SERVER_IMAGE = DockerImageName.parse("ghcr.io/dependencytrack/hyades-apiserver") - .withTag(Optional.ofNullable(System.getenv("APISERVER_VERSION")).orElse("snapshot")); - protected static DockerImageName MIRROR_SERVICE_IMAGE = DockerImageName.parse("ghcr.io/dependencytrack/hyades-mirror-service") - .withTag(Optional.ofNullable(System.getenv("HYADES_VERSION")).orElse("snapshot")); - protected static DockerImageName NOTIFICATION_PUBLISHER_IMAGE = DockerImageName.parse("ghcr.io/dependencytrack/hyades-notification-publisher") - .withTag(Optional.ofNullable(System.getenv("HYADES_VERSION")).orElse("snapshot")); - protected static DockerImageName REPO_META_ANALYZER_IMAGE = DockerImageName.parse("ghcr.io/dependencytrack/hyades-repository-meta-analyzer") - .withTag(Optional.ofNullable(System.getenv("HYADES_VERSION")).orElse("snapshot")); - protected static DockerImageName VULN_ANALYZER_IMAGE = DockerImageName.parse("ghcr.io/dependencytrack/hyades-vulnerability-analyzer") - .withTag(Optional.ofNullable(System.getenv("HYADES_VERSION")).orElse("snapshot")); + private enum KafkaProvider { + APACHE, + APACHE_NATIVE, + REDPANDA + // TODO: TANSU + } + + private static final String HYADES_APISERVER_TAG = Optional.ofNullable(System.getenv("HYADES_APISERVER_TAG")).orElse("snapshot"); + private static final String HYADES_TAG = Optional.ofNullable(System.getenv("HYADES_TAG")).orElse("snapshot"); + private static final String KAFKA_APACHE_TAG = Optional.ofNullable(System.getenv("KAFKA_APACHE_TAG")).orElse("3.8.0"); + private static final String KAFKA_REDPANDA_TAG = Optional.ofNullable(System.getenv("KAFKA_REDPANDA_TAG")).orElse("v24.2.2"); + private static final String POSTGRES_TAG = Optional.ofNullable(System.getenv("POSTGRES_TAG")).orElse("15-alpine"); + + private static final DockerImageName KAFKA_APACHE_IMAGE = DockerImageName.parse("apache/kafka").withTag(KAFKA_APACHE_TAG); + private static final DockerImageName KAFKA_APACHE_NATIVE_IMAGE = DockerImageName.parse("apache/kafka-native").withTag(KAFKA_APACHE_TAG); + private static final DockerImageName POSTGRES_IMAGE = DockerImageName.parse("postgres").withTag(POSTGRES_TAG); + private static final DockerImageName KAFKA_REDPANDA_IMAGE = DockerImageName.parse("docker.redpanda.com/vectorized/redpanda").withTag(KAFKA_REDPANDA_TAG); + private static final DockerImageName API_SERVER_IMAGE = DockerImageName.parse("ghcr.io/dependencytrack/hyades-apiserver").withTag(HYADES_APISERVER_TAG); + private static final DockerImageName MIRROR_SERVICE_IMAGE = DockerImageName.parse("ghcr.io/dependencytrack/hyades-mirror-service").withTag(HYADES_TAG); + private static final DockerImageName NOTIFICATION_PUBLISHER_IMAGE = DockerImageName.parse("ghcr.io/dependencytrack/hyades-notification-publisher").withTag(HYADES_TAG); + private static final DockerImageName REPO_META_ANALYZER_IMAGE = DockerImageName.parse("ghcr.io/dependencytrack/hyades-repository-meta-analyzer").withTag(HYADES_TAG); + private static final DockerImageName VULN_ANALYZER_IMAGE = DockerImageName.parse("ghcr.io/dependencytrack/hyades-vulnerability-analyzer").withTag(HYADES_TAG); + + private static final KafkaProvider KAFKA_PROVIDER = Optional.ofNullable(System.getenv("KAFKA_PROVIDER")).map(KafkaProvider::valueOf).orElse(KafkaProvider.APACHE_NATIVE); protected final Logger logger = LoggerFactory.getLogger(getClass()); protected final Network internalNetwork = Network.newNetwork(); protected PostgreSQLContainer postgresContainer; - protected GenericContainer redpandaContainer; + protected GenericContainer kafkaContainer; protected GenericContainer apiServerContainer; protected GenericContainer mirrorServiceContainer; protected GenericContainer notificationPublisherContainer; @@ -82,10 +94,12 @@ public class AbstractE2ET { @BeforeEach void beforeEach() throws Exception { postgresContainer = createPostgresContainer(); - redpandaContainer = createRedpandaContainer(); - deepStart(postgresContainer, redpandaContainer).join(); + kafkaContainer = switch (KAFKA_PROVIDER) { + case APACHE, APACHE_NATIVE -> createApacheKafkaContainer(); + case REDPANDA -> createRedpandaContainer(); + }; + deepStart(postgresContainer, kafkaContainer).join(); - initializeRedpanda(); generateSecretKey(); runInitializer(); @@ -115,32 +129,32 @@ private PostgreSQLContainer createPostgresContainer() { .withNetwork(internalNetwork); } + @SuppressWarnings("resource") + protected KafkaContainer createApacheKafkaContainer() { + final DockerImageName imageName = switch (KAFKA_PROVIDER) { + case APACHE -> KAFKA_APACHE_IMAGE; + case APACHE_NATIVE -> KAFKA_APACHE_NATIVE_IMAGE; + default -> throw new IllegalArgumentException(); + }; + return new KafkaContainer(imageName) + .withNetworkAliases("kafka") + .withNetwork(internalNetwork); + } + @SuppressWarnings("resource") private GenericContainer createRedpandaContainer() { - return new GenericContainer<>(REDPANDA_IMAGE) + return new GenericContainer<>(KAFKA_REDPANDA_IMAGE) .withCommand( "redpanda", "start", "--smp", "1", "--mode", "dev-container", "--reserve-memory", "0M", "--memory", "512M", "--overprovisioned", - "--kafka-addr", "PLAINTEXT://0.0.0.0:29092", - "--advertise-kafka-addr", "PLAINTEXT://redpanda:29092" + "--kafka-addr", "PLAINTEXT://0.0.0.0:9093", + "--advertise-kafka-addr", "PLAINTEXT://kafka:9093" ) .waitingFor(Wait.forLogMessage(".*Started Kafka API server.*", 1)) - .withNetworkAliases("redpanda") + .withNetworkAliases("kafka") .withNetwork(internalNetwork); } - @SuppressWarnings("resource") - private void initializeRedpanda() { - new GenericContainer<>(REDPANDA_IMAGE) - .withCreateContainerCmdModifier(cmd -> cmd.withUser("0").withEntrypoint("/bin/bash")) - .withCommand("/tmp/create-topics.sh") - .withEnv("REDPANDA_BROKERS", "redpanda:29092") - .withFileSystemBind("../scripts/create-topics.sh", "/tmp/create-topics.sh", BindMode.READ_ONLY) - .waitingFor(Wait.forLogMessage(".*All topics created successfully.*", 1)) - .withNetwork(internalNetwork) - .start(); - } - private void generateSecretKey() throws Exception { secretKeyPath = Files.createTempFile(null, null); secretKeyPath.toFile().deleteOnExit(); @@ -157,11 +171,14 @@ private void generateSecretKey() throws Exception { private void runInitializer() { new GenericContainer<>(API_SERVER_IMAGE) .withImagePullPolicy("local".equals(API_SERVER_IMAGE.getVersionPart()) ? PullPolicy.defaultPolicy() : PullPolicy.alwaysPull()) - .withEnv("EXTRA_JAVA_OPTIONS", "-Xmx256m") + .withEnv("JAVA_OPTIONS", "-Xmx256m -XX:+UseSerialGC -XX:TieredStopAtLevel=1") .withEnv("ALPINE_DATABASE_URL", "jdbc:postgresql://postgres:5432/dtrack") .withEnv("ALPINE_DATABASE_USERNAME", "dtrack") .withEnv("ALPINE_DATABASE_PASSWORD", "dtrack") + .withEnv("ALPINE_DATABASE_POOL_ENABLED", "false") + .withEnv("KAFKA_BOOTSTRAP_SERVERS", "kafka:9093") .withEnv("INIT_TASKS_ENABLED", "true") + .withEnv("INIT_TASKS_KAFKA_TOPICS_ENABLED", "true") .withEnv("INIT_AND_EXIT", "true") .withLogConsumer(new Slf4jLogConsumer(LoggerFactory.getLogger("initializer"))) .withStartupCheckStrategy(new OneShotStartupCheckStrategy()) @@ -177,7 +194,7 @@ private GenericContainer createApiServerContainer() { .withEnv("ALPINE_DATABASE_URL", "jdbc:postgresql://postgres:5432/dtrack?reWriteBatchedInserts=true") .withEnv("ALPINE_DATABASE_USERNAME", "dtrack") .withEnv("ALPINE_DATABASE_PASSWORD", "dtrack") - .withEnv("KAFKA_BOOTSTRAP_SERVERS", "redpanda:29092") + .withEnv("KAFKA_BOOTSTRAP_SERVERS", "kafka:9093") .withEnv("INIT_TASKS_ENABLED", "false") .withEnv("ALPINE_SECRET_KEY_PATH", "/var/run/secrets/secret.key") .withCopyFileToContainer( @@ -201,7 +218,7 @@ private GenericContainer createMirrorServiceContainer() { final var container = new GenericContainer<>(MIRROR_SERVICE_IMAGE) .withImagePullPolicy("local".equals(MIRROR_SERVICE_IMAGE.getVersionPart()) ? PullPolicy.defaultPolicy() : PullPolicy.alwaysPull()) .withEnv("JAVA_OPTS", "-Xmx256m") - .withEnv("KAFKA_BOOTSTRAP_SERVERS", "redpanda:29092") + .withEnv("KAFKA_BOOTSTRAP_SERVERS", "kafka:9093") .withEnv("QUARKUS_DATASOURCE_JDBC_URL", "jdbc:postgresql://postgres:5432/dtrack") .withEnv("QUARKUS_DATASOURCE_USERNAME", "dtrack") .withEnv("QUARKUS_DATASOURCE_PASSWORD", "dtrack") @@ -226,7 +243,7 @@ private GenericContainer createNotificationPublisherContainer() { final var container = new GenericContainer<>(NOTIFICATION_PUBLISHER_IMAGE) .withImagePullPolicy("local".equals(NOTIFICATION_PUBLISHER_IMAGE.getVersionPart()) ? PullPolicy.defaultPolicy() : PullPolicy.alwaysPull()) .withEnv("JAVA_OPTS", "-Xmx256m") - .withEnv("KAFKA_BOOTSTRAP_SERVERS", "redpanda:29092") + .withEnv("KAFKA_BOOTSTRAP_SERVERS", "kafka:9093") .withEnv("QUARKUS_DATASOURCE_JDBC_URL", "jdbc:postgresql://postgres:5432/dtrack") .withEnv("QUARKUS_DATASOURCE_USERNAME", "dtrack") .withEnv("QUARKUS_DATASOURCE_PASSWORD", "dtrack") @@ -251,7 +268,7 @@ private GenericContainer createRepoMetaAnalyzerContainer() { final var container = new GenericContainer<>(REPO_META_ANALYZER_IMAGE) .withImagePullPolicy("local".equals(REPO_META_ANALYZER_IMAGE.getVersionPart()) ? PullPolicy.defaultPolicy() : PullPolicy.alwaysPull()) .withEnv("JAVA_OPTS", "-Xmx256m") - .withEnv("QUARKUS_KAFKA_STREAMS_BOOTSTRAP_SERVERS", "redpanda:29092") + .withEnv("QUARKUS_KAFKA_STREAMS_BOOTSTRAP_SERVERS", "kafka:9093") .withEnv("QUARKUS_DATASOURCE_JDBC_URL", "jdbc:postgresql://postgres:5432/dtrack") .withEnv("QUARKUS_DATASOURCE_USERNAME", "dtrack") .withEnv("QUARKUS_DATASOURCE_PASSWORD", "dtrack") @@ -276,7 +293,7 @@ private GenericContainer createVulnAnalyzerContainer() { final var container = new GenericContainer<>(VULN_ANALYZER_IMAGE) .withImagePullPolicy("local".equals(VULN_ANALYZER_IMAGE.getVersionPart()) ? PullPolicy.defaultPolicy() : PullPolicy.alwaysPull()) .withEnv("JAVA_OPTS", "-Xmx256m") - .withEnv("QUARKUS_KAFKA_STREAMS_BOOTSTRAP_SERVERS", "redpanda:29092") + .withEnv("QUARKUS_KAFKA_STREAMS_BOOTSTRAP_SERVERS", "kafka:9093") .withEnv("QUARKUS_DATASOURCE_JDBC_URL", "jdbc:postgresql://postgres:5432/dtrack") .withEnv("QUARKUS_DATASOURCE_USERNAME", "dtrack") .withEnv("QUARKUS_DATASOURCE_PASSWORD", "dtrack") @@ -342,7 +359,7 @@ void afterEach() { Optional.ofNullable(notificationPublisherContainer).ifPresent(GenericContainer::stop); Optional.ofNullable(mirrorServiceContainer).ifPresent(GenericContainer::stop); Optional.ofNullable(apiServerContainer).ifPresent(GenericContainer::stop); - Optional.ofNullable(redpandaContainer).ifPresent(GenericContainer::stop); + Optional.ofNullable(kafkaContainer).ifPresent(GenericContainer::stop); Optional.ofNullable(postgresContainer).ifPresent(GenericContainer::stop); Optional.ofNullable(internalNetwork).ifPresent(Network::close); diff --git a/e2e/src/test/java/org/dependencytrack/e2e/VulnerabilityPolicyE2ET.java b/e2e/src/test/java/org/dependencytrack/e2e/VulnerabilityPolicyE2ET.java index 1fdd8d5be..857035266 100644 --- a/e2e/src/test/java/org/dependencytrack/e2e/VulnerabilityPolicyE2ET.java +++ b/e2e/src/test/java/org/dependencytrack/e2e/VulnerabilityPolicyE2ET.java @@ -158,7 +158,7 @@ protected void customizeApiServerContainer(final GenericContainer container) // Configure policy bundle fetching to occur 5s after startup, // and every minute from then on. .withEnv("TASK_SCHEDULER_INITIAL_DELAY", "5000") - .withEnv("TASK_CRON_VULNERABILITY_POLICY_BUNDLE_FETCH", "* * * * *"); + .withEnv("TASK_VULNERABILITY_POLICY_FETCH_CRON", "* * * * *"); } @Override diff --git a/monitoring/redpanda-console.yml b/monitoring/redpanda-console.yml new file mode 100644 index 000000000..0796d814d --- /dev/null +++ b/monitoring/redpanda-console.yml @@ -0,0 +1,59 @@ +--- +kafka: + brokers: ["kafka:9092"] + protobuf: + enabled: true + mappings: + - topicName: dtrack.notification.analyzer + valueProtoType: org.dependencytrack.notification.v1.Notification + - topicName: dtrack.notification.bom + valueProtoType: org.dependencytrack.notification.v1.Notification + - topicName: dtrack.notification.configuration + valueProtoType: org.dependencytrack.notification.v1.Notification + - topicName: dtrack.notification.datasource-mirroring + valueProtoType: org.dependencytrack.notification.v1.Notification + - topicName: dtrack.notification.file-system + valueProtoType: org.dependencytrack.notification.v1.Notification + - topicName: dtrack.notification.integration + valueProtoType: org.dependencytrack.notification.v1.Notification + - topicName: dtrack.notification.new-vulnerability + valueProtoType: org.dependencytrack.notification.v1.Notification + - topicName: dtrack.notification.new-vulnerable-dependency + valueProtoType: org.dependencytrack.notification.v1.Notification + - topicName: dtrack.notification.policy-violation + valueProtoType: org.dependencytrack.notification.v1.Notification + - topicName: dtrack.notification.project-audit-change + valueProtoType: org.dependencytrack.notification.v1.Notification + - topicName: dtrack.notification.project-created + valueProtoType: org.dependencytrack.notification.v1.Notification + - topicName: dtrack.notification.project-vuln-analysis-complete + valueProtoType: org.dependencytrack.notification.v1.Notification + - topicName: dtrack.notification.repository + valueProtoType: org.dependencytrack.notification.v1.Notification + - topicName: dtrack.notification.vex + valueProtoType: org.dependencytrack.notification.v1.Notification + - topicName: dtrack.repo-meta-analysis.component + valueProtoType: org.dependencytrack.repometaanalysis.v1.AnalysisCommand + - topicName: dtrack.repo-meta-analysis.result + valueProtoType: org.dependencytrack.repometaanalysis.v1.AnalysisResult + - topicName: dtrack.vuln-analysis.component + keyProtoType: org.dependencytrack.vulnanalysis.v1.ScanKey + valueProtoType: org.dependencytrack.vulnanalysis.v1.ScanCommand + - topicName: dtrack.vuln-analysis.scanner.result + keyProtoType: org.dependencytrack.vulnanalysis.v1.ScanKey + valueProtoType: org.dependencytrack.vulnanalysis.v1.ScannerResult + - topicName: dtrack.vuln-analysis.result + keyProtoType: org.dependencytrack.vulnanalysis.v1.ScanKey + valueProtoType: org.dependencytrack.vulnanalysis.v1.ScanResult + - topicName: dtrack.vuln-analysis.result.processed + valueProtoType: org.dependencytrack.vulnanalysis.v1.ScanResult + - topicName: dtrack.vulnerability + valueProtoType: org.cyclonedx.v1_6.Bom + - topicName: dtrack.epss + valueProtoType: org.dependencytrack.mirror.v1.EpssItem + - topicName: dtrack.notification.user + valueProtoType: org.dependencytrack.notification.v1.Notification + fileSystem: + enabled: true + paths: ["/etc/protos"] + refreshInterval: 5m \ No newline at end of file diff --git a/scripts/create-topics.sh b/scripts/create-topics.sh deleted file mode 100644 index 721ff6c4f..000000000 --- a/scripts/create-topics.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash - -# This file is part of Dependency-Track. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# Copyright (c) OWASP Foundation. All Rights Reserved. - -set -euxo pipefail - -function create_topic() { - topic_config_flags="" - argument_configs=($3) - for cfg in "${argument_configs[@]}"; do - topic_config_flags="${topic_config_flags} --topic-config ${cfg}" - done - - if ! output=$(rpk topic create "$1" --partitions "$2" $topic_config_flags); then - # Don't fail the script when the rpk command failed because the topic already exists. - if [[ "$output" != *"TOPIC_ALREADY_EXISTS"* ]]; then - exit 2 - fi - fi -} - -# Wait for Redpanda to become available -rpk cluster health --watch --exit-when-healthy \ - --api-urls "$(echo "$REDPANDA_BROKERS" | sed -E 's/:[[:digit:]]+/:9644/g')" - -notification_topics=( - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.notification.analyzer" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.notification.bom" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.notification.configuration" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.notification.datasource-mirroring" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.notification.file-system" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.notification.integration" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.notification.new-vulnerability" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.notification.new-vulnerable-dependency" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.notification.policy-violation" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.notification.project-audit-change" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.notification.project-created" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.notification.project-vuln-analysis-complete" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.notification.repository" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.notification.vex" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.notification.user" -) -for topic_name in "${notification_topics[@]}"; do - create_topic "$topic_name" "${NOTIFICATION_TOPICS_PARTITIONS:-1}" "retention.ms=${NOTIFICATION_TOPICS_RETENTION_MS:-43200000}" -done - -repo_meta_analysis_topics=( - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.repo-meta-analysis.component" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.repo-meta-analysis.result" -) -for topic_name in "${repo_meta_analysis_topics[@]}"; do - create_topic "$topic_name" "${REPO_META_ANALYSIS_TOPICS_PARTITIONS:-3}" "retention.ms=${REPO_META_ANALYSIS_TOPICS_RETENTION_MS:-43200000}" -done - -vuln_analysis_topics=( - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.vuln-analysis.component" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.vuln-analysis.scanner.result" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.vuln-analysis.result" - "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.vuln-analysis.result.processed" -) -for topic_name in "${vuln_analysis_topics[@]}"; do - create_topic "$topic_name" "${VULN_ANALYSIS_TOPICS_PARTITIONS:-3}" "retention.ms=${VULN_ANALYSIS_TOPICS_RETENTION_MS:-43200000}" -done - -create_topic "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.vulnerability.mirror.command" "1" "retention.ms=${VULN_MIRROR_TOPICS_RETENTION_MS:-43200000}" -create_topic "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.vulnerability.mirror.state" "1" "cleanup.policy=compact segment.bytes=67108864 max.compaction.lag.ms=1" -create_topic "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.vulnerability.digest" "1" "cleanup.policy=compact segment.bytes=134217728" -create_topic "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.vulnerability" "${VULN_MIRROR_TOPICS_PARTITIONS:-3}" "cleanup.policy=compact" -create_topic "${DT_KAFKA_TOPIC_PREFIX:-}dtrack.epss" "${VULN_MIRROR_TOPICS_PARTITIONS:-3}" "cleanup.policy=compact" - -echo "All topics created successfully" \ No newline at end of file