Extend Compose setup and e2e tests with multiple Kafka providers

Extends the Compose setup, which previously solely relied on Redpanda, with Apache Kafka and Tansu. Defaults to Apache Kafka (native) in accordance with #854. While a Tansu setup is added, Tansu can not yet be used with Hyades. It runs into errors when the API server tries to produce to it.

Extends e2e tests to support testing again different Kafka providers. Currently supported are Apache Kafka, Apache Kafka native, and Redpanda. Tansu will be added once it's functional.

To enable all of the above, the `create-topics.sh` script had to be modified to use official Kafka CLI tools (i.e. `kafka-topics.sh`) instead of the Redpanda CLI. The official tooling uses the Kafka API and is thus provider-independent.

Signed-off-by: nscuro <nscuro@protonmail.com>

.env

@@ -0,0 +1,24 @@
+COMPOSE_FILE="compose.yml:compose.kafka-apache.yml"
+# COMPOSE_FILE="compose.yml:compose.kafka-redpanda.yml"
+# COMPOSE_FILE="compose.yml:compose.kafka-tansu.yml"
+# COMPOSE_FILE="compose.yml:compose.kafka-apache.yml:compose.monitoring.yml"
+
+# Database connection details.
+DB_USERNAME=dtrack
+DB_PASSWORD=dtrack
+DB_NAME=dtrack
+
+# Dependency-Track image tags.
+HYADES_TAG="snapshot-native"
+HYADES_APISERVER_TAG="snapshot"
+HYADES_FRONTEND_TAG="snapshot"
+
+# Infrastructure image tags.
+# Which ones are relevant depends on the compose files being used (see COMPOSE_FILE).
+KAFKA_APACHE_TAG="3.8.0"
+KAFKA_REDPANDA_TAG="v24.2.2"
+KAFKA_TANSU_TAG="latest"
+POSTGRES_TAG="17-alpine"
+
+# Tansu configuration.
+KAFKA_TANSU_DB_NAME="tansu"

.github/workflows/e2e-test.yml

@@ -27,6 +27,13 @@ on:
       hyades-version:
         description: Version of Hyades services to test against
         default: snapshot
+      kafka-provider:
+        description: The Kafka provider to test against
+        type: choice
+        options:
+        - APACHE
+        - APACHE_NATIVE
+        - REDPANDA
 
 permissions: { }
 
@@ -45,8 +52,9 @@ jobs:
         cache: maven
     - name: Test
       env:
-        APISERVER_VERSION: ${{ inputs.apiserver-version || 'snapshot' }}
+        HYADES_APISERVER_VERSION: ${{ inputs.apiserver-version || 'snapshot' }}
         HYADES_VERSION: ${{ inputs.hyades-version || 'snapshot' }}
+        KAFKA_PROVIDER: ${{ inputs.kafka-provider || 'APACHE_NATIVE' }}
         OSSINDEX_USERNAME: ${{ secrets.OSSINDEX_USERNAME }}
         OSSINDEX_TOKEN: ${{ secrets.OSSINDEX_TOKEN }}
       run: mvn -B --no-transfer-progress -pl e2e clean verify -Pe2e-all

.gitignore

@@ -39,7 +39,7 @@ nb-configuration.xml
 *.rej
 
 # Local environment
-.env
+!.env
 
 # Secret key
 secret.key

compose.kafka-apache.yml

@@ -0,0 +1,47 @@
+# This file is part of Dependency-Track.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+---
+services:
+  kafka:
+    image: "apache/kafka-native:${KAFKA_APACHE_TAG}"
+    environment:
+      CLUSTER_ID: 'RvQwrYegSUCkIPkaiAZQlQ'
+      KAFKA_NODE_ID: "1"
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT'
+      KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT_HOST://localhost:9092,PLAINTEXT://kafka:29092'
+      KAFKA_PROCESS_ROLES: 'broker,controller'
+      KAFKA_CONTROLLER_QUORUM_VOTERS: '1@kafka:29093'
+      KAFKA_LISTENERS: 'CONTROLLER://:29093,PLAINTEXT_HOST://:9092,PLAINTEXT://:29092'
+      KAFKA_INTER_BROKER_LISTENER_NAME: 'PLAINTEXT'
+      KAFKA_CONTROLLER_LISTENER_NAMES: 'CONTROLLER'
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
+      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
+      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
+      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
+    healthcheck:
+      test: [ "CMD", "nc", "-z", "kafka", "9092"]
+      interval: 5s
+      timeout: 3s
+      retries: 3
+    ports:
+    - "9092:9092"
+    volumes:
+    - "kafka-data:/var/lib/kafka/data"
+    restart: unless-stopped
+
+volumes:
+  kafka-data: { }

compose.kafka-redpanda.yml

@@ -0,0 +1,49 @@
+# This file is part of Dependency-Track.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+---
+services:
+  kafka:
+    image: "docker.redpanda.com/vectorized/redpanda:${KAFKA_REDPANDA_TAG}"
+    command:
+    - redpanda
+    - start
+    - --smp
+    - '1'
+    - --reserve-memory
+    - 0M
+    - --memory
+    - 512M
+    - --overprovisioned
+    - --node-id
+    - '0'
+    - --kafka-addr
+    - PLAINTEXT://0.0.0.0:29092,OUTSIDE://0.0.0.0:9092
+    - --advertise-kafka-addr
+    - PLAINTEXT://kafka:29092,OUTSIDE://localhost:9092
+    healthcheck:
+      test: [ "CMD-SHELL", "rpk", "cluster", "health", "--exit-when-healthy" ]
+      interval: 5s
+      timeout: 3s
+      retries: 3
+    ports:
+    - "9092:9092"
+    volumes:
+    - "redpanda-data:/var/lib/redpanda/data"
+    restart: unless-stopped
+
+volumes:
+  redpanda-data: { }

compose.kafka-tansu.yml

@@ -0,0 +1,61 @@
+# This file is part of Dependency-Track.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+---
+services:
+  tansu-init:
+    image: "postgres:${POSTGRES_TAG}"
+    entrypoint: /bin/bash
+    command: >-
+      -c '((psql -d ${KAFKA_TANSU_DB_NAME} -q -c "select 1 from cluster limit 1" >/dev/null 2>/dev/null) && echo "Already initialized")
+      || ((psql -c "CREATE DATABASE ${KAFKA_TANSU_DB_NAME}") && (wget -q -O- https://raw.githubusercontent.com/tansu-io/tansu/refs/heads/main/work-dir/initdb.d/010-schema.sql | psql -d tansu))'
+    depends_on:
+      postgres:
+        condition: service_healthy
+    environment:
+      PGHOST: "postgres"
+      PGUSER: "${DB_USERNAME}"
+      PGPASSWORD: "${DB_PASSWORD}"
+    restart: on-failure
+
+  kafka:
+    image: "ghcr.io/tansu-io/tansu:${KAFKA_TANSU_TAG}"
+    command: >-
+      --kafka-cluster-id RvQwrYegSUCkIPkaiAZQlQ
+      --kafka-node-id 1
+      --kafka-listener-url tcp://0.0.0.0:29092
+      --kafka-advertised-listener-url tcp://kafka:29092
+      --storage-engine pg=postgres://${DB_USERNAME}:${DB_PASSWORD}@postgres/${KAFKA_TANSU_DB_NAME}
+      --work-dir /data
+    depends_on:
+      postgres:
+        condition: service_healthy
+      tansu-init:
+        condition: service_completed_successfully
+    environment:
+      RUST_BACKTRACE: "1"
+      RUST_LOG: "info"
+    healthcheck:
+      test: [ "CMD", "/tansu-server", "-h" ]
+      interval: 5s
+      timeout: 3s
+      retries: 3
+    volumes:
+    - "tansu-data:/data"
+    restart: unless-stopped
+
+volumes:
+  tansu-data: { }

compose.monitoring.yml

@@ -0,0 +1,61 @@
+# This file is part of Dependency-Track.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+# Copyright (c) OWASP Foundation. All Rights Reserved.
+---
+services:
+  redpanda-console:
+    image: docker.redpanda.com/vectorized/console:v2.7.0
+    depends_on:
+      kafka:
+        condition: service_healthy
+    environment:
+      CONFIG_FILEPATH: "/tmp/config.yml"
+    ports:
+    - "127.0.0.1:28080:8080"
+    volumes:
+    - "./monitoring/redpanda-console:/tmp/config.yml:ro"
+    - "./proto/src/main/proto:/etc/protos:ro"
+    restart: unless-stopped
+
+  prometheus:
+    image: prom/prometheus:v2.53.1
+    container_name: dt-prometheus
+    ports:
+    - "127.0.0.1:9090:9090"
+    volumes:
+    - "./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro"
+    - "prometheus-data:/prometheus"
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana-oss:11.1.3
+    container_name: dt-grafana
+    depends_on:
+    - prometheus
+    environment:
+      GF_SECURITY_ADMIN_USER: "admin"
+      GF_SECURITY_ADMIN_PASSWORD: "admin"
+    ports:
+    - "127.0.0.1:3000:3000"
+    volumes:
+    - "grafana-data:/var/lib/grafana"
+    - "./monitoring/grafana/dashboards:/etc/dashboards:ro"
+    - "./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro"
+    restart: unless-stopped
+
+volumes:
+  grafana-data: { }
+  prometheus-data: { }