Pull config via @ConfigProperty in mirror-service (#1236)

.gitignore

@@ -43,9 +43,7 @@ nb-configuration.xml
 
 # Secret key
 secret.key
-!commons/src/test/resources/secret.key
-!repository-meta-analyzer/src/test/resources/secret.key
-!notification-publisher/src/test/resources/secret.key
+!*/src/test/resources/secret.key
 
 # Host volume mount for demo
 apiserver-data/

docker-compose.yml

@@ -107,12 +107,18 @@ services:
     environment:
       KAFKA_BOOTSTRAP_SERVERS: "dt-redpanda:29092"
       KAFKA_STREAMS_NUM_STREAM_THREADS: "3"
+      QUARKUS_DATASOURCE_JDBC_URL: "jdbc:postgresql://dt-postgres:5432/dtrack"
+      QUARKUS_DATASOURCE_USERNAME: "dtrack"
+      QUARKUS_DATASOURCE_PASSWORD: "dtrack"
+      SECRET_KEY_PATH: "/var/run/secrets/secret.key"
     ports:
       # Dynamic host port binding to allow for scaling of the service.
       # Scaling with Compose doesn't work when assigning static host ports.
       - "8093"
     profiles:
       - demo
+    volumes:
+      - "secret-data:/var/run/secrets:ro"
     restart: unless-stopped
 
   apiserver:

e2e/src/test/java/org/dependencytrack/e2e/AbstractE2ET.java

@@ -55,6 +55,7 @@ public class AbstractE2ET {
     protected static String POSTGRES_IMAGE = "postgres:15-alpine";
     protected static String REDPANDA_IMAGE = "docker.redpanda.com/vectorized/redpanda:v23.3.13";
     protected static String API_SERVER_IMAGE = "ghcr.io/dependencytrack/hyades-apiserver:snapshot";
+    protected static String MIRROR_SERVICE_IMAGE = "ghcr.io/dependencytrack/hyades-mirror-service:snapshot";
     protected static String NOTIFICATION_PUBLISHER_IMAGE = "ghcr.io/dependencytrack/hyades-notification-publisher:snapshot";
     protected static String REPO_META_ANALYZER_IMAGE = "ghcr.io/dependencytrack/hyades-repository-meta-analyzer:snapshot";
     protected static String VULN_ANALYZER_IMAGE = "ghcr.io/dependencytrack/hyades-vulnerability-analyzer:snapshot";
@@ -64,6 +65,7 @@ public class AbstractE2ET {
     protected PostgreSQLContainer<?> postgresContainer;
     protected GenericContainer<?> redpandaContainer;
     protected GenericContainer<?> apiServerContainer;
+    protected GenericContainer<?> mirrorServiceContainer;
     protected GenericContainer<?> notificationPublisherContainer;
     protected GenericContainer<?> repoMetaAnalyzerContainer;
     protected GenericContainer<?> vulnAnalyzerContainer;
@@ -83,10 +85,16 @@ void beforeEach() throws Exception {
         apiServerContainer = createApiServerContainer();
         apiServerContainer.start();
 
+        mirrorServiceContainer = createMirrorServiceContainer();
         notificationPublisherContainer = createNotificationPublisherContainer();
         repoMetaAnalyzerContainer = createRepoMetaAnalyzerContainer();
         vulnAnalyzerContainer = createVulnAnalyzerContainer();
-        deepStart(notificationPublisherContainer, repoMetaAnalyzerContainer, vulnAnalyzerContainer).join();
+        deepStart(
+                mirrorServiceContainer,
+                notificationPublisherContainer,
+                repoMetaAnalyzerContainer,
+                vulnAnalyzerContainer
+        ).join();
 
         apiServerClient = initializeApiServerClient();
     }
@@ -165,6 +173,31 @@ private GenericContainer<?> createApiServerContainer() {
     protected void customizeApiServerContainer(final GenericContainer<?> container) {
     }
 
+    @SuppressWarnings("resource")
+    private GenericContainer<?> createMirrorServiceContainer() {
+        final var container = new GenericContainer<>(DockerImageName.parse(MIRROR_SERVICE_IMAGE))
+                .withImagePullPolicy(PullPolicy.alwaysPull())
+                .withEnv("JAVA_OPTS", "-Xmx256m")
+                .withEnv("KAFKA_BOOTSTRAP_SERVERS", "redpanda:29092")
+                .withEnv("QUARKUS_DATASOURCE_JDBC_URL", "jdbc:postgresql://postgres:5432/dtrack")
+                .withEnv("QUARKUS_DATASOURCE_USERNAME", "dtrack")
+                .withEnv("QUARKUS_DATASOURCE_PASSWORD", "dtrack")
+                .withEnv("SECRET_KEY_PATH", "/var/run/secrets/secret.key")
+                .withCopyFileToContainer(
+                        MountableFile.forHostPath(secretKeyPath, 444),
+                        "/var/run/secrets/secret.key"
+                )
+                .withLogConsumer(new Slf4jLogConsumer(LoggerFactory.getLogger("mirror-service")))
+                .withNetworkAliases("mirror-service")
+                .withNetwork(internalNetwork)
+                .withStartupAttempts(3);
+        customizeMirrorServiceContainer(container);
+        return container;
+    }
+
+    protected void customizeMirrorServiceContainer(final GenericContainer<?> container) {
+    }
+
     @SuppressWarnings("resource")
     private GenericContainer<?> createNotificationPublisherContainer() {
         final var container = new GenericContainer<>(DockerImageName.parse(NOTIFICATION_PUBLISHER_IMAGE))
@@ -269,7 +302,7 @@ private ApiServerClient initializeApiServerClient() {
         }
 
         logger.info("Authenticating as e2e team");
-        ApiServerAuthInterceptor.setApiKey(team.apiKeys().get(0).key());
+        ApiServerAuthInterceptor.setApiKey(team.apiKeys().getFirst().key());
 
         return client;
     }
@@ -281,6 +314,7 @@ void afterEach() {
         Optional.ofNullable(vulnAnalyzerContainer).ifPresent(GenericContainer::stop);
         Optional.ofNullable(repoMetaAnalyzerContainer).ifPresent(GenericContainer::stop);
         Optional.ofNullable(notificationPublisherContainer).ifPresent(GenericContainer::stop);
+        Optional.ofNullable(mirrorServiceContainer).ifPresent(GenericContainer::stop);
         Optional.ofNullable(apiServerContainer).ifPresent(GenericContainer::stop);
         Optional.ofNullable(redpandaContainer).ifPresent(GenericContainer::stop);
         Optional.ofNullable(postgresContainer).ifPresent(GenericContainer::stop);

helm-charts/hyades/templates/mirror-service/deployment.yaml

@@ -53,6 +53,15 @@ spec:
         - name: web
           containerPort: 8093
           protocol: TCP
+        volumeMounts:
+        - name: tmp
+          mountPath: /tmp
+        {{- if (include "hyades.secretKeySecretName" .) }}
+        - name: secret-key
+          subPath: secret.key
+          mountPath: /var/run/secrets/secret.key
+          readOnly: true
+        {{- end }}
         livenessProbe:
           httpGet:
             scheme: HTTP
@@ -73,4 +82,12 @@ spec:
           periodSeconds: {{ .Values.mirrorService.probes.readiness.periodSeconds }}
           successThreshold: {{ .Values.mirrorService.probes.readiness.successThreshold }}
           timeoutSeconds: {{ .Values.mirrorService.probes.readiness.timeoutSeconds }}
+      volumes:
+      - name: tmp
+        emptyDir: {}
+            {{- with (include "hyades.secretKeySecretName" .) }}
+      - name: secret-key
+        secret:
+          secretName: {{ . }}
+            {{- end }}
   {{- end }}

mirror-service/pom.xml

@@ -38,11 +38,19 @@
             <groupId>org.dependencytrack</groupId>
             <artifactId>commons-kstreams</artifactId>
         </dependency>
+        <dependency>
+            <groupId>org.dependencytrack</groupId>
+            <artifactId>commons-persistence</artifactId>
+        </dependency>
         <dependency>
             <groupId>org.dependencytrack</groupId>
             <artifactId>proto</artifactId>
         </dependency>
 
+        <dependency>
+            <groupId>org.dependencytrack</groupId>
+            <artifactId>quarkus-config-dependencytrack</artifactId>
+        </dependency>
         <dependency>
             <groupId>io.quarkus</groupId>
             <artifactId>quarkus-kafka-client</artifactId>

mirror-service/src/main/java/org/dependencytrack/vulnmirror/KafkaStreamsTopologyProducer.java

@@ -76,7 +76,7 @@ Topology topology(final Instance<DatasourceMirror> datasourceMirrors) {
                 .foreach((datasource, value) -> datasourceMirrors.stream()
                                 .filter(mirror -> mirror.supportsDatasource(datasource))
                                 .findAny()
-                                .ifPresent(datasourceMirror -> datasourceMirror.doMirror(value)),
+                                .ifPresent(DatasourceMirror::doMirror),
                         Named.as("execute_mirror"));
 
         // For every successfully mirrored vulnerability, calculate the SHA-256 digest

mirror-service/src/main/java/org/dependencytrack/vulnmirror/datasource/DatasourceMirror.java

@@ -33,9 +33,8 @@ public interface DatasourceMirror {
     /**
      * <em>Asynchronously</em> execute a mirroring operating.
      *
-     * @param ecosystem The ecosystem to be mirrored for. Needed in case of OSV mirror only. Maybe null for other mirrors.
      * @return A {@link Future} for tracking completion of the operation
      */
-    Future<?> doMirror(String ecosystem);
+    Future<?> doMirror();
 
 }

mirror-service/src/main/java/org/dependencytrack/vulnmirror/datasource/epss/EpssClientFactory.java

@@ -31,7 +31,7 @@ class EpssClientFactory {
     }
 
     EpssDataFeed create() {
-        return epssConfig.downloadUrl()
+        return epssConfig.feedsUrl()
                 .map(EpssDataFeed::new)
                 .orElseGet(EpssDataFeed::new);
     }

mirror-service/src/main/java/org/dependencytrack/vulnmirror/datasource/epss/EpssConfig.java

@@ -19,11 +19,43 @@
 package org.dependencytrack.vulnmirror.datasource.epss;
 
 import io.smallrye.config.ConfigMapping;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Provider;
+import org.eclipse.microprofile.config.inject.ConfigProperty;
 
 import java.util.Optional;
 
-@ConfigMapping(prefix = "mirror.datasource.epss")
-public interface EpssConfig {
+/**
+ * As of Quarkus 3.9 / smallrye-config 3.7, it is not possible to use {@link ConfigMapping}
+ * interfaces with {@link Provider} fields. We need {@link Provider} fields in order to support
+ * configuration changes at runtime. Refer to <em>Injecting Dynamic Values</em> in the
+ * {@link ConfigProperty} JavaDoc for details.
+ *
+ * @see <a href="https://github.com/smallrye/smallrye-config/issues/664">Related smallrye-config issue</a>
+ */
+@ApplicationScoped
+class EpssConfig {
+
+    private final Provider<Optional<Boolean>> enabledProvider;
+    private final Provider<Optional<String>> feedsUrlProvider;
+
+    EpssConfig(
+            @ConfigProperty(name = "dtrack.vuln-source.epss.enabled") final Provider<Optional<Boolean>> enabledProvider,
+            @ConfigProperty(name = "dtrack.vuln-source.epss.feeds.url") final Provider<Optional<String>> feedsUrlProvider
+    ) {
+        this.enabledProvider = enabledProvider;
+        this.feedsUrlProvider = feedsUrlProvider;
+    }
+
+    Optional<Boolean> enabled() {
+        return enabledProvider.get();
+    }
+
+    Optional<String> feedsUrl() {
+        return feedsUrlProvider.get()
+                // NB: Dependency-Track has historically only made the base URL (e.g. https://epss.cyentia.com)
+                // configurable, but the EPSS client expects a full URL.
+                .map("%s/epss_scores-current.csv.gz"::formatted);
+    }
 
-    Optional<String> downloadUrl();
 }

mirror-service/src/main/java/org/dependencytrack/vulnmirror/datasource/epss/EpssMirror.java

@@ -38,6 +38,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Future;
 
+import static java.util.concurrent.CompletableFuture.completedFuture;
 import static org.dependencytrack.proto.notification.v1.Level.LEVEL_ERROR;
 import static org.dependencytrack.proto.notification.v1.Level.LEVEL_INFORMATIONAL;
 
@@ -47,20 +48,23 @@ class EpssMirror extends AbstractDatasourceMirror<Void> {
     private static final Logger LOGGER = LoggerFactory.getLogger(EpssMirror.class);
     private static final String NOTIFICATION_TITLE = "EPSS Mirroring";
     private final ExecutorService executorService;
-    final EpssClientFactory epssClientFactory;
+    private final EpssClientFactory epssClientFactory;
+    private final EpssConfig config;
     private final Timer durationTimer;
-    private Producer<String, byte[]> kafkaProducer;
+    private final Producer<String, byte[]> kafkaProducer;
 
     EpssMirror(@ForEpssMirror final ExecutorService executorService,
                final MirrorStateStore mirrorStateStore,
                final VulnerabilityDigestStore vulnDigestStore,
                final Producer<String, byte[]> kafkaProducer,
                @ForEpssMirror final Timer durationTimer,
-               final EpssClientFactory epssClientFactory) {
+               final EpssClientFactory epssClientFactory,
+               final EpssConfig config) {
         super(Datasource.EPSS, mirrorStateStore, vulnDigestStore, kafkaProducer, Void.class);
         this.executorService = executorService;
         this.durationTimer = durationTimer;
         this.epssClientFactory = epssClientFactory;
+        this.config = config;
         this.kafkaProducer = kafkaProducer;
     }
 
@@ -70,7 +74,12 @@ public boolean supportsDatasource(final Datasource datasource) {
     }
 
     @Override
-    public Future<?> doMirror(String ecosystem) {
+    public Future<?> doMirror() {
+        if (!config.enabled().orElse(false)) {
+            LOGGER.warn("Mirroring of the {} datasource was requested, but it is not enabled", Datasource.EPSS);
+            return completedFuture(null);
+        }
+
         return executorService.submit(() -> {
             try {
                 performMirror();

mirror-service/src/main/java/org/dependencytrack/vulnmirror/datasource/github/GitHubApiClientFactory.java

@@ -21,6 +21,7 @@
 import io.github.jeremylong.openvulnerability.client.ghsa.GitHubSecurityAdvisoryClient;
 import io.github.jeremylong.openvulnerability.client.ghsa.GitHubSecurityAdvisoryClientBuilder;
 import jakarta.enterprise.context.ApplicationScoped;
+import org.dependencytrack.common.SecretDecryptor;
 
 import java.time.Instant;
 import java.time.ZoneOffset;
@@ -32,16 +33,26 @@
 class GitHubApiClientFactory {
 
     private final GitHubConfig config;
+    private final SecretDecryptor secretDecryptor;
 
-    GitHubApiClientFactory(final GitHubConfig config) {
+    GitHubApiClientFactory(final GitHubConfig config, final SecretDecryptor secretDecryptor) {
         this.config = config;
+        this.secretDecryptor = secretDecryptor;
     }
 
     GitHubSecurityAdvisoryClient create(final long lastUpdatedEpochSeconds) {
         final GitHubSecurityAdvisoryClientBuilder builder = aGitHubSecurityAdvisoryClient();
 
         config.baseUrl().ifPresent(builder::withEndpoint);
-        config.apiKey().ifPresent(builder::withApiKey);
+        config.apiKey()
+                .map(encryptedApiKey -> {
+                    try {
+                        return secretDecryptor.decryptAsString(encryptedApiKey);
+                    } catch (Exception e) {
+                        throw new IllegalStateException("Failed to decrypt API key", e);
+                    }
+                })
+                .ifPresent(builder::withApiKey);
 
         if (lastUpdatedEpochSeconds > 0) {
             final ZonedDateTime lastUpdated = ZonedDateTime.ofInstant(Instant.ofEpochSecond(lastUpdatedEpochSeconds), ZoneOffset.UTC);

mirror-service/src/main/java/org/dependencytrack/vulnmirror/datasource/github/GitHubConfig.java

@@ -19,17 +19,54 @@
 package org.dependencytrack.vulnmirror.datasource.github;
 
 import io.smallrye.config.ConfigMapping;
-import io.smallrye.config.WithDefault;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.inject.Provider;
+import org.eclipse.microprofile.config.inject.ConfigProperty;
 
 import java.util.Optional;
 
-@ConfigMapping(prefix = "mirror.datasource.github")
-public interface GitHubConfig {
+/**
+ * As of Quarkus 3.9 / smallrye-config 3.7, it is not possible to use {@link ConfigMapping}
+ * interfaces with {@link Provider} fields. We need {@link Provider} fields in order to support
+ * configuration changes at runtime. Refer to <em>Injecting Dynamic Values</em> in the
+ * {@link ConfigProperty} JavaDoc for details.
+ *
+ * @see <a href="https://github.com/smallrye/smallrye-config/issues/664">Related smallrye-config issue</a>
+ */
+@ApplicationScoped
+class GitHubConfig {
+
+    private final Provider<Optional<Boolean>> enabledProvider;
+    private final Provider<Optional<String>> baseUrlProvider;
+    private final Provider<Optional<String>> apiKeyProvider;
+    private final Provider<Optional<Boolean>> aliasSyncEnabledProvider;
+
+    GitHubConfig(
+            @ConfigProperty(name = "dtrack.vuln-source.github.advisories.enabled") final Provider<Optional<Boolean>> enabledProvider,
+            @ConfigProperty(name = "dtrack.vuln-source.github.advisories.base.url") final Provider<Optional<String>> baseUrlProvider,
+            @ConfigProperty(name = "dtrack.vuln-source.github.advisories.access.token") final Provider<Optional<String>> apiKeyProvider,
+            @ConfigProperty(name = "dtrack.vuln-source.github.advisories.alias.sync.enabled") final Provider<Optional<Boolean>> aliasSyncEnabledProvider
+    ) {
+        this.enabledProvider = enabledProvider;
+        this.baseUrlProvider = baseUrlProvider;
+        this.apiKeyProvider = apiKeyProvider;
+        this.aliasSyncEnabledProvider = aliasSyncEnabledProvider;
+    }
+
+    Optional<Boolean> enabled() {
+        return enabledProvider.get();
+    }
+
+    Optional<String> baseUrl() {
+        return baseUrlProvider.get();
+    }
 
-    Optional<String> baseUrl();
+    Optional<String> apiKey() {
+        return apiKeyProvider.get();
+    }
 
-    Optional<String> apiKey();
+    Optional<Boolean> aliasSyncEnabled() {
+        return aliasSyncEnabledProvider.get();
+    }
 
-    @WithDefault("false")
-    boolean aliasSyncEnabled();
 }