From c1cfea5bed1745de04dc48e048e40c7769807bab Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Wed, 4 Oct 2023 14:18:43 +0100 Subject: [PATCH 01/17] WIP --- .../v1/repo_meta_analysis.proto | 16 +++++ .../RepositoryMetaAnalyzerTopology.java | 11 ++- .../java/org/hyades/model/IntegrityMeta.java | 67 +++++++++++++++++++ .../processor/MetaAnalyzerProcessor.java | 54 ++++++++++----- .../MetaAnalyzerProcessorSupplier.java | 5 +- .../repositories/AbstractMetaAnalyzer.java | 53 +++++++++++++++ .../hyades/repositories/IMetaAnalyzer.java | 9 +++ .../repositories/MavenMetaAnalyzer.java | 19 ++++++ .../hyades/repositories/NpmMetaAnalyzer.java | 24 +++++++ .../hyades/repositories/PypiMetaAnalyzer.java | 23 +++++++ .../processor/MetaAnalyzerProcessorTest.java | 25 +++---- 11 files changed, 268 insertions(+), 38 deletions(-) create mode 100644 repository-meta-analyzer/src/main/java/org/hyades/model/IntegrityMeta.java diff --git a/proto/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto b/proto/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto index a58d96c27..4b57a7088 100644 --- a/proto/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto +++ b/proto/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto @@ -11,6 +11,8 @@ option java_package = "org.hyades.proto.repometaanalysis.v1"; message AnalysisCommand { // The component that shall be analyzed. Component component = 1; + bool fetch_integrity_data = 2; + bool fetch_latest_version = 3; } message AnalysisResult { @@ -25,6 +27,9 @@ message AnalysisResult { // When the latest version was published. optional google.protobuf.Timestamp published = 4; + + // Integrity metadata of the component. + optional IntegrityMeta integrity_meta = 5; } message Component { @@ -35,3 +40,14 @@ message Component { // Internal components will only be looked up in internal repositories. optional bool internal = 2; } + +message IntegrityMeta { + optional string md5 = 1; + optional string sha1 = 2; + optional string sha256 = 3; + optional string sha512 = 4; + // When the component current version last modified. + optional google.protobuf.Timestamp current_version_last_modified = 5; + // Complete URL to fetch integrity metadata of the component. + optional string repository_url = 6; +} diff --git a/repository-meta-analyzer/src/main/java/org/hyades/RepositoryMetaAnalyzerTopology.java b/repository-meta-analyzer/src/main/java/org/hyades/RepositoryMetaAnalyzerTopology.java index 0026c5cbf..ff8c6c96c 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/RepositoryMetaAnalyzerTopology.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/RepositoryMetaAnalyzerTopology.java @@ -17,7 +17,6 @@ import org.hyades.proto.KafkaProtobufSerde; import org.hyades.proto.repometaanalysis.v1.AnalysisCommand; import org.hyades.proto.repometaanalysis.v1.AnalysisResult; -import org.hyades.proto.repometaanalysis.v1.Component; import org.hyades.repositories.RepositoryAnalyzerFactory; import org.hyades.serde.KafkaPurlSerde; @@ -57,19 +56,17 @@ public Topology topology(final RepositoryAnalyzerFactory analyzerFactory, .withName("command-by-purl-coordinates")); commandStream - .mapValues((purl, command) -> command.getComponent(), - Named.as("map_to_component")) .split(Named.as("applicable_analyzer")) - .branch((purl, component) -> analyzerFactory.hasApplicableAnalyzer(purl), Branched - .withConsumer(stream -> stream + .branch((purl, command) -> analyzerFactory.hasApplicableAnalyzer(purl), Branched + .withConsumer(stream -> stream .processValues(analyzerProcessorSupplier, Named.as("analyze_component")) .to(KafkaTopic.REPO_META_ANALYSIS_RESULT.getName(), Produced .with(purlSerde, scanResultSerde) .withName(processorNameProduce(KafkaTopic.REPO_META_ANALYSIS_RESULT, "analysis_result")))) .withName("-found")) .defaultBranch(Branched - .withConsumer(stream -> stream - .mapValues((purl, component) -> AnalysisResult.newBuilder().setComponent(component).build(), + .withConsumer(stream -> stream + .mapValues((purl, command) -> AnalysisResult.newBuilder().setComponent(command.getComponent()).build(), Named.as("map_unmatched_component_to_empty_result")) .to(KafkaTopic.REPO_META_ANALYSIS_RESULT.getName(), Produced .with(purlSerde, scanResultSerde) diff --git a/repository-meta-analyzer/src/main/java/org/hyades/model/IntegrityMeta.java b/repository-meta-analyzer/src/main/java/org/hyades/model/IntegrityMeta.java new file mode 100644 index 000000000..00cdee2d2 --- /dev/null +++ b/repository-meta-analyzer/src/main/java/org/hyades/model/IntegrityMeta.java @@ -0,0 +1,67 @@ +package org.hyades.model; + +import java.io.Serializable; +import java.util.Date; + +public class IntegrityMeta implements Serializable { + + private String md5; + + private String sha1; + + private String sha256; + + private String sha512; + + private Date currentVersionLastModified; + + private String repositoryUrl; + + public String getMd5() { + return md5; + } + + public void setMd5(String md5) { + this.md5 = md5; + } + + public String getSha1() { + return sha1; + } + + public void setSha1(String sha1) { + this.sha1 = sha1; + } + + public String getSha256() { + return sha256; + } + + public void setSha256(String sha256) { + this.sha256 = sha256; + } + + public Date getCurrentVersionLastModified() { + return currentVersionLastModified; + } + + public void setCurrentVersionLastModified(Date currentVersionLastModified) { + this.currentVersionLastModified = currentVersionLastModified; + } + + public void setSha512(String sha512) { + this.sha512 = sha512; + } + + public String getSha512() { + return sha512; + } + + public String getRepositoryUrl() { + return repositoryUrl; + } + + public void setRepositoryUrl(String repositoryUrl) { + this.repositoryUrl = repositoryUrl; + } +} \ No newline at end of file diff --git a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java index 3d946ba01..2873a4dc5 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java @@ -13,8 +13,10 @@ import org.hyades.persistence.model.Repository; import org.hyades.persistence.model.RepositoryType; import org.hyades.persistence.repository.RepoEntityRepository; +import org.hyades.proto.repometaanalysis.v1.AnalysisCommand; import org.hyades.proto.repometaanalysis.v1.AnalysisResult; import org.hyades.proto.repometaanalysis.v1.Component; +import org.hyades.proto.repometaanalysis.v1.IntegrityMeta; import org.hyades.repositories.IMetaAnalyzer; import org.hyades.repositories.RepositoryAnalyzerFactory; import org.slf4j.Logger; @@ -24,7 +26,7 @@ import java.util.NoSuchElementException; import java.util.Optional; -class MetaAnalyzerProcessor extends ContextualFixedKeyProcessor { +class MetaAnalyzerProcessor extends ContextualFixedKeyProcessor { private static final Logger LOGGER = LoggerFactory.getLogger(MetaAnalyzerProcessor.class); @@ -44,9 +46,9 @@ class MetaAnalyzerProcessor extends ContextualFixedKeyProcessor record) { + public void process(final FixedKeyRecord record) { final PackageURL purlWithoutVersion = record.key(); - final Component component = record.value(); + final Component component = record.value().getComponent(); // NOTE: Do not use purlWithoutVersion for the analysis! // It only contains the type, namespace and name, but is missing the @@ -93,7 +95,7 @@ public void process(final FixedKeyRecord record) { LOGGER.debug("Cache miss (analyzer: {}, purl: {}, repository: {})", analyzer.getName(), purl, repository.getIdentifier()); } - final Optional optionalResult = analyze(analyzer, repository, component); + final Optional optionalResult = analyze(analyzer, repository, record.value()); if (optionalResult.isPresent()) { context().forward(record .withValue(optionalResult.get()) @@ -109,7 +111,7 @@ public void process(final FixedKeyRecord record) { .withTimestamp(context().currentSystemTimeMs())); } - private Optional analyze(final IMetaAnalyzer analyzer, final Repository repository, final Component component) { + private Optional analyze(final IMetaAnalyzer analyzer, final Repository repository, final AnalysisCommand analysisCommand) { analyzer.setRepositoryBaseUrl(repository.getUrl()); boolean isAuthenticationRequired = Optional.ofNullable(repository.isAuthenticationRequired()).orElse(false); if (isAuthenticationRequired) { @@ -120,14 +122,20 @@ private Optional analyze(final IMetaAnalyzer analyzer, final Rep } } + // Analyzers still work with "legacy" data models, + // allowing us to avoid major refactorings of the original code. + final var component = new org.hyades.persistence.model.Component(); + component.setPurl(analysisCommand.getComponent().getPurl()); LOGGER.debug("Performing meta analysis on purl: {}", component.getPurl()); - final MetaModel metaModel; + MetaModel metaModel = null; + org.hyades.model.IntegrityMeta integrityMeta = null; try { - // Analyzers still work with "legacy" data models, - // allowing us to avoid major refactorings of the original code. - final var analyzerComponent = new org.hyades.persistence.model.Component(); - analyzerComponent.setPurl(component.getPurl()); - metaModel = analyzer.analyze(analyzerComponent); + if (analysisCommand.getFetchLatestVersion()) { + metaModel = analyzer.analyze(component); + } + if (analysisCommand.getFetchIntegrityData()) { + integrityMeta = analyzer.getIntegrityMeta(component); + } } catch (Exception e) { LOGGER.error("Failed to analyze {} using {} with repository {}", component.getPurl(), analyzer.getName(), repository.getIdentifier(), e); @@ -135,21 +143,33 @@ private Optional analyze(final IMetaAnalyzer analyzer, final Rep } final AnalysisResult.Builder resultBuilder = AnalysisResult.newBuilder() - .setComponent(component) + .setComponent(analysisCommand.getComponent()) .setRepository(repository.getIdentifier()); - if (metaModel.getLatestVersion() != null) { + if (metaModel != null && metaModel.getLatestVersion() != null) { resultBuilder.setLatestVersion(metaModel.getLatestVersion()); if (metaModel.getPublishedTimestamp() != null) { resultBuilder.setPublished(Timestamp.newBuilder() .setSeconds(metaModel.getPublishedTimestamp().getTime() / 1000)); } - final AnalysisResult result = resultBuilder.build(); LOGGER.debug("Found component metadata for: {} using repository: {} ({})", component.getPurl(), repository.getIdentifier(), repository.getType()); - return Optional.of(result); } - - return Optional.empty(); + if (integrityMeta != null) { + IntegrityMeta.Builder metaBuilder = IntegrityMeta.newBuilder() + .setMd5(integrityMeta.getMd5()) + .setSha1(integrityMeta.getSha1()) + .setSha256(integrityMeta.getSha256()) + .setSha512(integrityMeta.getSha512()) + .setRepositoryUrl(integrityMeta.getRepositoryUrl()); + if (integrityMeta.getCurrentVersionLastModified() != null) { + metaBuilder.setCurrentVersionLastModified(Timestamp.newBuilder() + .setSeconds(integrityMeta.getCurrentVersionLastModified().getTime() / 1000)); + } + resultBuilder.setIntegrityMeta(metaBuilder); + LOGGER.debug("Found integrity metadata for: {} using repository: {} ({})", + component.getPurl(), repository.getIdentifier(), repository.getType()); + } + return Optional.of(resultBuilder.build()); } private PackageURL mustParsePurl(final String purl) { diff --git a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessorSupplier.java b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessorSupplier.java index 2bbd5f65c..6d27d5b50 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessorSupplier.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessorSupplier.java @@ -8,12 +8,13 @@ import org.apache.kafka.streams.processor.api.FixedKeyProcessorSupplier; import org.hyades.common.SecretDecryptor; import org.hyades.persistence.repository.RepoEntityRepository; +import org.hyades.proto.repometaanalysis.v1.AnalysisCommand; import org.hyades.proto.repometaanalysis.v1.AnalysisResult; import org.hyades.proto.repometaanalysis.v1.Component; import org.hyades.repositories.RepositoryAnalyzerFactory; @ApplicationScoped -public class MetaAnalyzerProcessorSupplier implements FixedKeyProcessorSupplier { +public class MetaAnalyzerProcessorSupplier implements FixedKeyProcessorSupplier { private final RepoEntityRepository repoEntityRepository; private final RepositoryAnalyzerFactory analyzerFactory; @@ -31,7 +32,7 @@ public MetaAnalyzerProcessorSupplier(final RepoEntityRepository repoEntityReposi } @Override - public FixedKeyProcessor get() { + public FixedKeyProcessor get() { return new MetaAnalyzerProcessor(repoEntityRepository, analyzerFactory, secretDecryptor, cache); } diff --git a/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java index 575019eb2..97b551508 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java @@ -19,11 +19,16 @@ package org.hyades.repositories; import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.impl.client.CloseableHttpClient; +import org.hyades.commonutil.DateUtil; import org.hyades.commonutil.HttpUtil; +import org.hyades.model.IntegrityMeta; import org.hyades.persistence.model.Component; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,6 +82,11 @@ public void setRepositoryUsernameAndPassword(String username, String password) { this.password = StringUtils.trimToNull(password); } + @Override + public IntegrityMeta getIntegrityMeta(Component component) { + throw new UnsupportedOperationException("This analyzer does not support Integrity Metadata."); + } + protected void handleUnexpectedHttpResponse(final Logger logger, String url, final int statusCode, final String statusText, final Component component) { logger.debug("HTTP Status : " + statusCode + " " + statusText); logger.debug(" - RepositoryType URL : " + url); @@ -116,4 +126,47 @@ protected CloseableHttpResponse processHttpRequest(String url) throws IOExceptio return httpClient.execute(request); } + protected CloseableHttpResponse processHttpHeadRequest(String url) throws IOException { + final HttpUriRequest request = new HttpHead(url); + request.addHeader("accept", "application/json"); + if (username != null || password != null) { + request.addHeader("Authorization", HttpUtil.basicAuthHeaderValue(username, password)); + } + return httpClient.execute(request); + } + + protected IntegrityMeta extractIntegrityModelFromResponse(CloseableHttpResponse response, IntegrityMeta integrityMeta) { + var headers = response.getAllHeaders(); + for (var header : headers) { + if (header.getName().equalsIgnoreCase("X-Checksum-MD5")) { + integrityMeta.setMd5(header.getValue()); + } else if (header.getName().equalsIgnoreCase("X-Checksum-SHA1")) { + integrityMeta.setSha1(header.getValue()); + } else if (header.getName().equalsIgnoreCase("X-Checksum-SHA256")) { + integrityMeta.setSha256(header.getValue()); + } else if (header.getName().equalsIgnoreCase("X-Checksum-SHA512")) { + integrityMeta.setSha512(header.getValue()); + } else if (header.getName().equalsIgnoreCase("Last-Modified") && header.getValue() != null) { + integrityMeta.setCurrentVersionLastModified(DateUtil.parseDate(header.getValue())); + } + } + return integrityMeta; + } + + public IntegrityMeta fetchIntegrityMeta(String url, Logger logger, Component component) { + var integrityMeta = new IntegrityMeta(); + integrityMeta.setRepositoryUrl(url); + try (final CloseableHttpResponse response = processHttpHeadRequest(url)) { + final StatusLine status = response.getStatusLine(); + if (status.getStatusCode() == HttpStatus.SC_OK) { + return extractIntegrityModelFromResponse(response, integrityMeta); + } else { + handleUnexpectedHttpResponse(logger, url, status.getStatusCode(), status.getReasonPhrase(), component); + } + } catch (Exception ex) { + logger.warn("Head request for integrity meta failed. Not caching response for component with purl:{}", component.getPurl()); + handleRequestException(logger, ex); + } + return integrityMeta; + } } diff --git a/repository-meta-analyzer/src/main/java/org/hyades/repositories/IMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/hyades/repositories/IMetaAnalyzer.java index 5a84aead8..739cd340f 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/repositories/IMetaAnalyzer.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/repositories/IMetaAnalyzer.java @@ -19,6 +19,7 @@ package org.hyades.repositories; import org.apache.http.impl.client.CloseableHttpClient; +import org.hyades.model.IntegrityMeta; import org.hyades.model.MetaAnalyzerException; import org.hyades.model.MetaModel; import org.hyades.persistence.model.Component; @@ -81,4 +82,12 @@ public interface IMetaAnalyzer { String getName(); + /** + * The component meta data to analyze for integrity. + * @param component the component to analyze + * @return an IntegrityMeta object + * @throws MetaAnalyzerException in case of any issue during metadata generation + * @since 3.1.0 + */ + IntegrityMeta getIntegrityMeta(Component component); } diff --git a/repository-meta-analyzer/src/main/java/org/hyades/repositories/MavenMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/hyades/repositories/MavenMetaAnalyzer.java index ad6c4f546..4c1b002fa 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/repositories/MavenMetaAnalyzer.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/repositories/MavenMetaAnalyzer.java @@ -25,6 +25,7 @@ import org.apache.http.client.methods.CloseableHttpResponse; import org.hyades.commonutil.DateUtil; import org.hyades.commonutil.XmlUtil; +import org.hyades.model.IntegrityMeta; import org.hyades.model.MetaModel; import org.hyades.persistence.model.Component; import org.hyades.persistence.model.RepositoryType; @@ -118,4 +119,22 @@ public MetaModel analyze(final Component component) { public String getName() { return this.getClass().getSimpleName(); } + + @Override + public IntegrityMeta getIntegrityMeta(Component component) { + if (component != null) { + var packageUrl = component.getPurl(); + if (packageUrl != null) { + String type = "jar"; + if (packageUrl.getQualifiers() != null) { + type = packageUrl.getQualifiers().getOrDefault("type", "jar"); + } + final String mavenGavUrl = packageUrl.getNamespace().replaceAll("\\.", "/") + "/" + packageUrl.getName(); + final String url = baseUrl + "/" + mavenGavUrl + "/" + packageUrl.getVersion() + "/" + packageUrl.getName() + "-" + packageUrl.getVersion() + "." + type; + var integrityMeta = new IntegrityMeta(); + return fetchIntegrityMeta(url, LOGGER, component); + } + } + return null; + } } diff --git a/repository-meta-analyzer/src/main/java/org/hyades/repositories/NpmMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/hyades/repositories/NpmMetaAnalyzer.java index a609bcf5a..1518598f8 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/repositories/NpmMetaAnalyzer.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/repositories/NpmMetaAnalyzer.java @@ -21,6 +21,7 @@ import com.github.packageurl.PackageURL; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.util.EntityUtils; +import org.hyades.model.IntegrityMeta; import org.hyades.model.MetaAnalyzerException; import org.hyades.model.MetaModel; import org.hyades.persistence.model.Component; @@ -98,6 +99,29 @@ public MetaModel analyze(final Component component) { return meta; } + @Override + public IntegrityMeta getIntegrityMeta(Component component) { + if (component != null) { + var packageUrl = component.getPurl(); + if (packageUrl != null) { + String type = "tgz"; + if (packageUrl.getQualifiers() != null) { + type = packageUrl.getQualifiers().getOrDefault("type", "tgz"); + } + String npmArtifactoryUrl = ""; + if (packageUrl.getNamespace() != null) { + npmArtifactoryUrl += packageUrl.getNamespace().replaceAll("\\.", "/") + "/"; + } + npmArtifactoryUrl += packageUrl.getName(); + final String url = this.baseUrl + "/" + npmArtifactoryUrl + "/-/" + + npmArtifactoryUrl + "-" + packageUrl.getVersion() + "." + type; + var integrityMeta = new IntegrityMeta(); + return fetchIntegrityMeta(url, LOGGER, component); + } + } + return null; + } + @Override public String getName() { return this.getClass().getSimpleName(); diff --git a/repository-meta-analyzer/src/main/java/org/hyades/repositories/PypiMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/hyades/repositories/PypiMetaAnalyzer.java index 133d76de3..975137ad7 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/repositories/PypiMetaAnalyzer.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/repositories/PypiMetaAnalyzer.java @@ -21,6 +21,7 @@ import com.github.packageurl.PackageURL; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.util.EntityUtils; +import org.hyades.model.IntegrityMeta; import org.hyades.model.MetaAnalyzerException; import org.hyades.model.MetaModel; import org.hyades.persistence.model.Component; @@ -123,6 +124,28 @@ private MetaModel setTimeStamp(MetaModel meta, String updateTime) { return meta; } + @Override + public IntegrityMeta getIntegrityMeta(Component component) { + if (component != null) { + var packageUrl = component.getPurl(); + if (packageUrl != null) { + String type = "tar.gz"; + if (packageUrl.getQualifiers() != null) { + type = packageUrl.getQualifiers().getOrDefault("type", "tar.gz"); + } + String pypiArtifactoryUrl = ""; + if (packageUrl.getNamespace() != null) { + pypiArtifactoryUrl += packageUrl.getNamespace().replaceAll("\\.", "/") + "/"; + } + pypiArtifactoryUrl += packageUrl.getName() + "/" + packageUrl.getVersion(); + String url = this.baseUrl + "/" + pypiArtifactoryUrl + "/" + + packageUrl.getName() + "-" + packageUrl.getVersion() + "." + type; + return fetchIntegrityMeta(url, LOGGER, component); + } + } + return null; + } + @Override public String getName() { return this.getClass().getSimpleName(); diff --git a/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java b/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java index 93e8bc5c9..e0c6d96f3 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java @@ -22,6 +22,7 @@ import org.hyades.persistence.model.RepositoryType; import org.hyades.persistence.repository.RepoEntityRepository; import org.hyades.proto.KafkaProtobufSerde; +import org.hyades.proto.repometaanalysis.v1.AnalysisCommand; import org.hyades.proto.repometaanalysis.v1.AnalysisResult; import org.hyades.proto.repometaanalysis.v1.Component; import org.hyades.repositories.RepositoryAnalyzerFactory; @@ -51,7 +52,7 @@ public Map getConfigOverrides() { private static final String TEST_PURL_JACKSON_BIND = "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4"; private TopologyTestDriver testDriver; - private TestInputTopic inputTopic; + private TestInputTopic inputTopic; private TestOutputTopic outputTopic; @Inject RepoEntityRepository repoEntityRepository; @@ -73,7 +74,7 @@ public Map getConfigOverrides() { void beforeEach() { final var processorSupplier = new MetaAnalyzerProcessorSupplier(repoEntityRepository, analyzerFactory, secretDecryptor, cache); - final var valueSerde = new KafkaProtobufSerde<>(Component.parser()); + final var valueSerde = new KafkaProtobufSerde<>(AnalysisCommand.parser()); final var purlSerde = new KafkaPurlSerde(); final var valueSerdeResult = new KafkaProtobufSerde<>(AnalysisResult.parser()); @@ -96,8 +97,8 @@ void afterEach() { @Test void testWithNoSupportedRepositoryTypes() throws Exception { - final TestRecord inputRecord = new TestRecord<>(new PackageURL(TEST_PURL_JACKSON_BIND), Component.newBuilder() - .setPurl(TEST_PURL_JACKSON_BIND).build()); + final TestRecord inputRecord = new TestRecord<>(new PackageURL(TEST_PURL_JACKSON_BIND), AnalysisCommand.newBuilder().setComponent(Component.newBuilder() + .setPurl(TEST_PURL_JACKSON_BIND)).build()); inputTopic.pipeInput(inputRecord); assertThat(outputTopic.getQueueSize()).isEqualTo(1); assertThat(outputTopic.readRecordsToList()).satisfiesExactly( @@ -108,8 +109,8 @@ record -> { @Test void testMalformedPurl() throws Exception { - final TestRecord inputRecord = new TestRecord<>(new PackageURL(TEST_PURL_JACKSON_BIND), Component.newBuilder() - .setPurl("invalid purl").build()); + final TestRecord inputRecord = new TestRecord<>(new PackageURL(TEST_PURL_JACKSON_BIND), AnalysisCommand.newBuilder().setComponent(Component.newBuilder() + .setPurl("invalid purl")).build()); Assertions.assertThrows(StreamsException.class, () -> { inputTopic.pipeInput(inputRecord); }, "no exception thrown"); @@ -118,8 +119,8 @@ void testMalformedPurl() throws Exception { @Test void testNoAnalyzerApplicable() throws Exception { - final TestRecord inputRecord = new TestRecord<>(new PackageURL("pkg:test/com.fasterxml.jackson.core/jackson-databind@2.13.4"), Component.newBuilder() - .setPurl("pkg:test/com.fasterxml.jackson.core/jackson-databind@2.13.4").build()); + final TestRecord inputRecord = new TestRecord<>(new PackageURL("pkg:test/com.fasterxml.jackson.core/jackson-databind@2.13.4"), AnalysisCommand.newBuilder().setComponent(Component.newBuilder() + .setPurl("pkg:test/com.fasterxml.jackson.core/jackson-databind@2.13.4")).build()); inputTopic.pipeInput(inputRecord); assertThat(outputTopic.getQueueSize()).isEqualTo(1); assertThat(outputTopic.readRecordsToList()).satisfiesExactly( @@ -137,8 +138,8 @@ void testInternalRepositoryExternalComponent() throws MalformedPackageURLExcepti ('MAVEN',true, 'central', true, 'test.com', false,1); """).executeUpdate(); - final TestRecord inputRecord = new TestRecord<>(new PackageURL("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4"), Component.newBuilder() - .setPurl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4").setInternal(false).build()); + final TestRecord inputRecord = new TestRecord<>(new PackageURL("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4"), AnalysisCommand.newBuilder().setComponent(Component.newBuilder() + .setPurl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4").setInternal(false)).build()); inputTopic.pipeInput(inputRecord); assertThat(outputTopic.getQueueSize()).isEqualTo(1); assertThat(outputTopic.readRecordsToList()).satisfiesExactly( @@ -156,8 +157,8 @@ void testExternalRepositoryInternalComponent() throws MalformedPackageURLExcepti ('MAVEN',true, 'central', false, 'test.com', false,1); """).executeUpdate(); - final TestRecord inputRecord = new TestRecord<>(new PackageURL("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4"), Component.newBuilder() - .setPurl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4").setInternal(true).build()); + final TestRecord inputRecord = new TestRecord<>(new PackageURL("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4"), AnalysisCommand.newBuilder().setComponent(Component.newBuilder() + .setPurl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4").setInternal(true)).build()); inputTopic.pipeInput(inputRecord); assertThat(outputTopic.getQueueSize()).isEqualTo(1); assertThat(outputTopic.readRecordsToList()).satisfiesExactly( From dc45bcad7e3cdc6c3daa149e44b8b10f9dd0a172 Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Thu, 5 Oct 2023 10:38:15 +0100 Subject: [PATCH 02/17] test WIP --- .../MetaAnalyzerProcessorSupplier.java | 1 - .../repositories/AbstractMetaAnalyzer.java | 17 +++-- .../repositories/MavenMetaAnalyzer.java | 3 +- .../hyades/repositories/NpmMetaAnalyzer.java | 3 +- .../hyades/repositories/PypiMetaAnalyzer.java | 2 +- .../repositories/MavenMetaAnalyzerTest.java | 71 +++++++++++++++++++ 6 files changed, 86 insertions(+), 11 deletions(-) diff --git a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessorSupplier.java b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessorSupplier.java index 6d27d5b50..5b4de4e53 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessorSupplier.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessorSupplier.java @@ -10,7 +10,6 @@ import org.hyades.persistence.repository.RepoEntityRepository; import org.hyades.proto.repometaanalysis.v1.AnalysisCommand; import org.hyades.proto.repometaanalysis.v1.AnalysisResult; -import org.hyades.proto.repometaanalysis.v1.Component; import org.hyades.repositories.RepositoryAnalyzerFactory; @ApplicationScoped diff --git a/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java index 97b551508..98b30b9de 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java @@ -26,7 +26,6 @@ import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.impl.client.CloseableHttpClient; -import org.hyades.commonutil.DateUtil; import org.hyades.commonutil.HttpUtil; import org.hyades.model.IntegrityMeta; import org.hyades.persistence.model.Component; @@ -34,6 +33,8 @@ import org.slf4j.LoggerFactory; import java.io.IOException; +import java.text.ParseException; +import java.text.SimpleDateFormat; /** * Base abstract class that all IMetaAnalyzer implementations should likely extend. @@ -45,6 +46,8 @@ public abstract class AbstractMetaAnalyzer implements IMetaAnalyzer { final Logger logger = LoggerFactory.getLogger(this.getClass()); + private final String GMT_FORMAT = "EEE, dd MMM yyyy HH:mm:ss Z"; + protected CloseableHttpClient httpClient; protected String baseUrl; @@ -135,7 +138,7 @@ protected CloseableHttpResponse processHttpHeadRequest(String url) throws IOExce return httpClient.execute(request); } - protected IntegrityMeta extractIntegrityModelFromResponse(CloseableHttpResponse response, IntegrityMeta integrityMeta) { + protected IntegrityMeta extractIntegrityModelFromResponse(CloseableHttpResponse response, IntegrityMeta integrityMeta, Component component) { var headers = response.getAllHeaders(); for (var header : headers) { if (header.getName().equalsIgnoreCase("X-Checksum-MD5")) { @@ -147,19 +150,23 @@ protected IntegrityMeta extractIntegrityModelFromResponse(CloseableHttpResponse } else if (header.getName().equalsIgnoreCase("X-Checksum-SHA512")) { integrityMeta.setSha512(header.getValue()); } else if (header.getName().equalsIgnoreCase("Last-Modified") && header.getValue() != null) { - integrityMeta.setCurrentVersionLastModified(DateUtil.parseDate(header.getValue())); + try { + integrityMeta.setCurrentVersionLastModified(new SimpleDateFormat(GMT_FORMAT).parse(header.getValue())); + } catch (ParseException pe) { + logger.warn("Parsing LastModified date failed while extracting integrity meta for purl: {}", component.getPurl()); + } } } return integrityMeta; } - public IntegrityMeta fetchIntegrityMeta(String url, Logger logger, Component component) { + public IntegrityMeta fetchIntegrityMeta(String url, Component component) { var integrityMeta = new IntegrityMeta(); integrityMeta.setRepositoryUrl(url); try (final CloseableHttpResponse response = processHttpHeadRequest(url)) { final StatusLine status = response.getStatusLine(); if (status.getStatusCode() == HttpStatus.SC_OK) { - return extractIntegrityModelFromResponse(response, integrityMeta); + return extractIntegrityModelFromResponse(response, integrityMeta, component); } else { handleUnexpectedHttpResponse(logger, url, status.getStatusCode(), status.getReasonPhrase(), component); } diff --git a/repository-meta-analyzer/src/main/java/org/hyades/repositories/MavenMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/hyades/repositories/MavenMetaAnalyzer.java index 4c1b002fa..4c3c73a6a 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/repositories/MavenMetaAnalyzer.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/repositories/MavenMetaAnalyzer.java @@ -131,8 +131,7 @@ public IntegrityMeta getIntegrityMeta(Component component) { } final String mavenGavUrl = packageUrl.getNamespace().replaceAll("\\.", "/") + "/" + packageUrl.getName(); final String url = baseUrl + "/" + mavenGavUrl + "/" + packageUrl.getVersion() + "/" + packageUrl.getName() + "-" + packageUrl.getVersion() + "." + type; - var integrityMeta = new IntegrityMeta(); - return fetchIntegrityMeta(url, LOGGER, component); + return fetchIntegrityMeta(url, component); } } return null; diff --git a/repository-meta-analyzer/src/main/java/org/hyades/repositories/NpmMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/hyades/repositories/NpmMetaAnalyzer.java index 1518598f8..3849b3157 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/repositories/NpmMetaAnalyzer.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/repositories/NpmMetaAnalyzer.java @@ -115,8 +115,7 @@ public IntegrityMeta getIntegrityMeta(Component component) { npmArtifactoryUrl += packageUrl.getName(); final String url = this.baseUrl + "/" + npmArtifactoryUrl + "/-/" + npmArtifactoryUrl + "-" + packageUrl.getVersion() + "." + type; - var integrityMeta = new IntegrityMeta(); - return fetchIntegrityMeta(url, LOGGER, component); + return fetchIntegrityMeta(url, component); } } return null; diff --git a/repository-meta-analyzer/src/main/java/org/hyades/repositories/PypiMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/hyades/repositories/PypiMetaAnalyzer.java index 975137ad7..07163eb4d 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/repositories/PypiMetaAnalyzer.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/repositories/PypiMetaAnalyzer.java @@ -140,7 +140,7 @@ public IntegrityMeta getIntegrityMeta(Component component) { pypiArtifactoryUrl += packageUrl.getName() + "/" + packageUrl.getVersion(); String url = this.baseUrl + "/" + pypiArtifactoryUrl + "/" + packageUrl.getName() + "-" + packageUrl.getVersion() + "." + type; - return fetchIntegrityMeta(url, LOGGER, component); + return fetchIntegrityMeta(url, component); } } return null; diff --git a/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java b/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java index 7c7f0e969..79abb9829 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java @@ -19,24 +19,48 @@ package org.hyades.repositories; import com.github.packageurl.PackageURL; +import com.github.tomakehurst.wiremock.client.WireMock; +import com.github.tomakehurst.wiremock.http.Body; +import com.github.tomakehurst.wiremock.http.ContentTypeHeader; +import com.github.tomakehurst.wiremock.junit5.WireMockExtension; +import jakarta.ws.rs.core.MediaType; import org.apache.http.impl.client.HttpClients; import org.hyades.model.MetaModel; import org.hyades.persistence.model.Component; import org.hyades.persistence.model.RepositoryType; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; + +import static com.github.tomakehurst.wiremock.client.WireMock.containing; +import static com.github.tomakehurst.wiremock.core.WireMockConfiguration.wireMockConfig; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; class MavenMetaAnalyzerTest { private IMetaAnalyzer analyzer; + @RegisterExtension + static WireMockExtension wireMock = WireMockExtension.newInstance() + .options(wireMockConfig().dynamicPort()) + .build(); + @BeforeEach void beforeEach() { analyzer = new MavenMetaAnalyzer(); analyzer.setHttpClient(HttpClients.createDefault()); } + @AfterEach + void afterEach() { + wireMock.resetAll(); + } + @Test void testAnalyzer() throws Exception { Component component = new Component(); @@ -89,5 +113,52 @@ void testIOException() { Assertions.assertEquals(metaModel.getComponent(), component); } + + @Test + void testGetIntegrityMetaComponentNull() { + var integrityModel = analyzer.getIntegrityMeta(null); + Assertions.assertNull(integrityModel); + } + + @Test + void testGetIntegrityMeta404() { + Component component = new Component(); + component.setPurl("pkg:maven/typo3/package-empty-result@v1.2.0"); + var integrityMeta = analyzer.getIntegrityMeta(component); + assertNotNull(integrityMeta); + assertEquals("https://repo1.maven.org/maven2/typo3/package-empty-result/v1.2.0/package-empty-result-v1.2.0.jar", integrityMeta.getRepositoryUrl()); + assertNull(integrityMeta.getSha1()); + assertNull(integrityMeta.getMd5()); + assertNull(integrityMeta.getSha256()); + assertNull(integrityMeta.getSha512()); + assertNull(integrityMeta.getCurrentVersionLastModified()); + } + + @Test + void testGetIntegrityModel200() { + Component component = new Component(); + component.setPurl("pkg:maven/typo3/package-empty-result@v1.2.0"); + wireMock.stubFor(WireMock.head(WireMock.anyUrl()).withHeader("accept", containing("application/json")) + .willReturn(WireMock.aResponse() + .withStatus(200) + .withResponseBody(Body.ofBinaryOrText(""" + + """.getBytes(), new ContentTypeHeader(MediaType.APPLICATION_JSON)) + ) + .withHeader("X-CheckSum-MD5", "md5hash") + .withHeader("X-Checksum-SHA1", "sha1hash") + .withHeader("X-Checksum-SHA512", "sha512hash") + .withHeader("X-Checksum-SHA256", "sha256hash") + .withHeader("Last-Modified", "Mon, 07 Jul 2022 14:00:00 GMT"))); + analyzer.setRepositoryBaseUrl(wireMock.baseUrl()); + var integrityMeta = analyzer.getIntegrityMeta(component); + assertNotNull(integrityMeta); + assertThat(integrityMeta.getRepositoryUrl()).contains("/typo3/package-empty-result/v1.2.0/package-empty-result-v1.2.0.jar"); + assertEquals("md5hash", integrityMeta.getMd5()); + assertEquals("sha1hash", integrityMeta.getSha1()); + assertEquals("sha256hash", integrityMeta.getSha256()); + assertEquals("sha512hash", integrityMeta.getSha512()); + assertEquals("Thu Jul 07 15:00:00 IST 2022", integrityMeta.getCurrentVersionLastModified().toString()); + } } From 1c0219e2348b8edce0a2db0cb23804f22a937eba Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Thu, 5 Oct 2023 13:38:16 +0100 Subject: [PATCH 03/17] tests added --- .../repositories/MavenMetaAnalyzerTest.java | 2 +- .../repositories/NpmMetaAnalyzerTest.java | 87 +++++++++++++++++++ .../repositories/PypiMetaAnalyzerTest.java | 86 ++++++++++++++++++ 3 files changed, 174 insertions(+), 1 deletion(-) diff --git a/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java b/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java index 79abb9829..fdf4b19ee 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java @@ -149,7 +149,7 @@ void testGetIntegrityModel200() { .withHeader("X-Checksum-SHA1", "sha1hash") .withHeader("X-Checksum-SHA512", "sha512hash") .withHeader("X-Checksum-SHA256", "sha256hash") - .withHeader("Last-Modified", "Mon, 07 Jul 2022 14:00:00 GMT"))); + .withHeader("Last-Modified", "Thu, 07 Jul 2022 14:00:00 GMT"))); analyzer.setRepositoryBaseUrl(wireMock.baseUrl()); var integrityMeta = analyzer.getIntegrityMeta(component); assertNotNull(integrityMeta); diff --git a/repository-meta-analyzer/src/test/java/org/hyades/repositories/NpmMetaAnalyzerTest.java b/repository-meta-analyzer/src/test/java/org/hyades/repositories/NpmMetaAnalyzerTest.java index c2594e95b..9e6d2060c 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/repositories/NpmMetaAnalyzerTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/repositories/NpmMetaAnalyzerTest.java @@ -25,6 +25,7 @@ import org.hyades.persistence.model.Component; import org.hyades.persistence.model.RepositoryType; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -32,6 +33,10 @@ import org.mockserver.client.MockServerClient; import org.mockserver.integration.ClientAndServer; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.mockserver.model.HttpRequest.request; import static org.mockserver.model.HttpResponse.response; @@ -52,6 +57,11 @@ void beforeEach() { analyzer.setHttpClient(HttpClients.createDefault()); } + @AfterEach + void afterEach() { + mockServer.reset(); + } + @AfterAll static void afterClass() { mockServer.stop(); @@ -145,4 +155,81 @@ void testAnalyzerReturnEmptyResultWithBraces() throws Exception { metaModel.getPublishedTimestamp() ); } + + @Test + void testAnalyzerReturnIntegrityResult() { + Component component = new Component(); + component.setPurl("pkg:npm/typo3/package-empty-result@v1.2.0"); + analyzer.setRepositoryBaseUrl(String.format("http://localhost:%d", mockServer.getPort())); + new MockServerClient("localhost", mockServer.getPort()) + .when( + request() + .withMethod("HEAD") + .withPath("/typo3/package-empty-result/-/typo3/package-empty-result-v1.2.0.tgz") + ) + .respond( + response() + .withStatusCode(200) + .withHeader("X-Checksum-MD5", "md5hash") + .withHeader("X-Checksum-SHA1", "sha1hash") + .withHeader("X-Checksum-SHA256", "sha256hash") + .withHeader("X-Checksum-SHA512", "sha512hash") + .withHeader("Last-Modified", "Thu, 07 Jul 2022 14:00:00 GMT") + ); + + var integrityMeta = analyzer.getIntegrityMeta(component); + assertNotNull(integrityMeta); + assertThat(integrityMeta.getRepositoryUrl()).contains("/typo3/package-empty-result/-/typo3/package-empty-result-v1.2.0.tgz"); + assertEquals("md5hash", integrityMeta.getMd5()); + assertEquals("sha1hash", integrityMeta.getSha1()); + assertEquals("sha256hash", integrityMeta.getSha256()); + assertEquals("sha512hash", integrityMeta.getSha512()); + assertEquals("Thu Jul 07 15:00:00 IST 2022", integrityMeta.getCurrentVersionLastModified().toString()); + } + + @Test + void testIntegrityResultForPurlWithoutNamespace() { + Component component = new Component(); + component.setPurl("pkg:npm/amazon-s3-uri@0.0.1"); + analyzer.setRepositoryBaseUrl(String.format("http://localhost:%d", mockServer.getPort())); + new MockServerClient("localhost", mockServer.getPort()) + .when( + request() + .withMethod("HEAD") + .withPath("/amazon-s3-uri/-/amazon-s3-uri-0.0.1.tgz") + ) + .respond( + response() + .withStatusCode(200) + ); + + var integrityMeta = analyzer.getIntegrityMeta(component); + assertNotNull(integrityMeta); + assertThat(integrityMeta.getRepositoryUrl()).contains("/amazon-s3-uri/-/amazon-s3-uri-0.0.1.tgz"); + } + + @Test + void testIntegrityAnalyzerException() { + Component component = new Component(); + component.setPurl("pkg:npm/amazon-s3-uri@0.0.1"); + analyzer.setRepositoryBaseUrl(String.format("http://localhost:%d", mockServer.getPort())); + new MockServerClient("localhost", mockServer.getPort()) + .when( + request() + .withMethod("HEAD") + .withPath("/amazon-s3-uri/-/amazon-s3-uri-0.0.1.tgz") + ) + .respond( + response() + .withStatusCode(400) + ); + var integrityMeta = analyzer.getIntegrityMeta(component); + assertNotNull(integrityMeta); + assertThat(integrityMeta.getRepositoryUrl()).contains("/amazon-s3-uri/-/amazon-s3-uri-0.0.1.tgz"); + assertNull(integrityMeta.getSha1()); + assertNull(integrityMeta.getMd5()); + assertNull(integrityMeta.getSha256()); + assertNull(integrityMeta.getSha512()); + assertNull(integrityMeta.getCurrentVersionLastModified()); + } } diff --git a/repository-meta-analyzer/src/test/java/org/hyades/repositories/PypiMetaAnalyzerTest.java b/repository-meta-analyzer/src/test/java/org/hyades/repositories/PypiMetaAnalyzerTest.java index 254608c02..77ef862ba 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/repositories/PypiMetaAnalyzerTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/repositories/PypiMetaAnalyzerTest.java @@ -25,6 +25,7 @@ import org.hyades.persistence.model.Component; import org.hyades.persistence.model.RepositoryType; import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; @@ -32,6 +33,10 @@ import org.mockserver.client.MockServerClient; import org.mockserver.integration.ClientAndServer; +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.mockserver.model.HttpRequest.request; import static org.mockserver.model.HttpResponse.response; @@ -52,6 +57,11 @@ void beforeEach() { analyzer.setHttpClient(HttpClients.createDefault()); } + @AfterEach + void afterEach() { + mockServer.reset(); + } + @AfterAll static void afterClass() { mockServer.stop(); @@ -144,4 +154,80 @@ void testAnalyzerReturnEmptyResultWithBraces() throws Exception { metaModel.getPublishedTimestamp() ); } + + @Test + void testAnalyzerReturnIntegrityResult() { + Component component = new Component(); + component.setPurl("pkg:pypi/typo3/package-ok-result@v1.2.0"); + analyzer.setRepositoryBaseUrl(String.format("http://localhost:%d", mockServer.getPort())); + new MockServerClient("localhost", mockServer.getPort()) + .when( + request() + .withMethod("HEAD") + .withPath("/typo3/package-ok-result/v1.2.0/package-ok-result-v1.2.0.tar.gz") + ) + .respond( + response() + .withStatusCode(200) + .withHeader("X-Checksum-MD5", "md5hash") + .withHeader("X-Checksum-SHA1", "sha1hash") + .withHeader("X-Checksum-SHA256", "sha256hash") + .withHeader("X-Checksum-SHA512", "sha512hash") + .withHeader("Last-Modified", "Thu, 07 Jul 2022 14:00:00 GMT") + ); + + var integrityMeta = analyzer.getIntegrityMeta(component); + assertNotNull(integrityMeta); + assertThat(integrityMeta.getRepositoryUrl()).contains("/typo3/package-ok-result/v1.2.0/package-ok-result-v1.2.0.tar.gz"); + assertEquals("md5hash", integrityMeta.getMd5()); + assertEquals("sha1hash", integrityMeta.getSha1()); + assertEquals("sha256hash", integrityMeta.getSha256()); + assertEquals("sha512hash", integrityMeta.getSha512()); + assertEquals("Thu Jul 07 15:00:00 IST 2022", integrityMeta.getCurrentVersionLastModified().toString()); + } + + @Test + void testIntegrityAnalyzerException() { + Component component = new Component(); + component.setPurl("pkg:pypi/typo1/package-no-result@v1.2.0"); + analyzer.setRepositoryBaseUrl(String.format("http://localhost:%d", mockServer.getPort())); + new MockServerClient("localhost", mockServer.getPort()) + .when( + request() + .withMethod("HEAD") + ) + .respond( + response() + .withStatusCode(400) + ); + var integrityMeta = analyzer.getIntegrityMeta(component); + assertNotNull(integrityMeta); + assertThat(integrityMeta.getRepositoryUrl()).contains("/typo1/package-no-result/v1.2.0/package-no-result-v1.2.0.tar.gz"); + assertNull(integrityMeta.getSha1()); + assertNull(integrityMeta.getMd5()); + assertNull(integrityMeta.getSha256()); + assertNull(integrityMeta.getSha512()); + assertNull(integrityMeta.getCurrentVersionLastModified()); + } + + @Test + void testIntegrityResultForPurlWithoutNamespace() { + Component component = new Component(); + component.setPurl("pkg:pypi/package-result@v1.2.0"); + analyzer.setRepositoryBaseUrl(String.format("http://localhost:%d", mockServer.getPort())); + new MockServerClient("localhost", mockServer.getPort()) + .when( + request() + .withMethod("HEAD") + .withPath("/package-result/v1.2.0/package-result-v1.2.0.tar.gz") + ) + .respond( + response() + .withStatusCode(200) + ); + + var integrityMeta = analyzer.getIntegrityMeta(component); + assertNotNull(integrityMeta); + assertThat(integrityMeta.getRepositoryUrl()).contains("/package-result/v1.2.0/package-result-v1.2.0.tar.gz"); + } } From b026a45daf0c7f879e4b76b7776f4ea4e948d2f2 Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Thu, 5 Oct 2023 18:02:33 +0100 Subject: [PATCH 04/17] added IT --- .../processor/MetaAnalyzerProcessor.java | 22 +- .../repositories/AbstractMetaAnalyzer.java | 2 +- .../org/hyades/RepositoryMetaAnalyzerIT.java | 277 +++++++++++++++++- .../repositories/HexMetaAnalyzerTest.java | 8 + 4 files changed, 298 insertions(+), 11 deletions(-) diff --git a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java index 2873a4dc5..8d4ae522b 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java @@ -155,12 +155,22 @@ private Optional analyze(final IMetaAnalyzer analyzer, final Rep component.getPurl(), repository.getIdentifier(), repository.getType()); } if (integrityMeta != null) { - IntegrityMeta.Builder metaBuilder = IntegrityMeta.newBuilder() - .setMd5(integrityMeta.getMd5()) - .setSha1(integrityMeta.getSha1()) - .setSha256(integrityMeta.getSha256()) - .setSha512(integrityMeta.getSha512()) - .setRepositoryUrl(integrityMeta.getRepositoryUrl()); + IntegrityMeta.Builder metaBuilder = IntegrityMeta.newBuilder(); + if (integrityMeta.getMd5() != null) { + metaBuilder.setMd5(integrityMeta.getMd5()); + } + if (integrityMeta.getSha1() != null) { + metaBuilder.setSha1(integrityMeta.getSha1()); + } + if (integrityMeta.getSha256() != null) { + metaBuilder.setSha256(integrityMeta.getSha256()); + } + if (integrityMeta.getSha512() != null) { + metaBuilder.setSha512(integrityMeta.getSha512()); + } + if (integrityMeta.getRepositoryUrl() != null) { + metaBuilder.setRepositoryUrl(integrityMeta.getRepositoryUrl()); + } if (integrityMeta.getCurrentVersionLastModified() != null) { metaBuilder.setCurrentVersionLastModified(Timestamp.newBuilder() .setSeconds(integrityMeta.getCurrentVersionLastModified().getTime() / 1000)); diff --git a/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java index 98b30b9de..97e7d5362 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java @@ -87,7 +87,7 @@ public void setRepositoryUsernameAndPassword(String username, String password) { @Override public IntegrityMeta getIntegrityMeta(Component component) { - throw new UnsupportedOperationException("This analyzer does not support Integrity Metadata."); + throw new UnsupportedOperationException("Skipping integrity meta fetch as this analyzer does not support Integrity Metadata."); } protected void handleUnexpectedHttpResponse(final Logger logger, String url, final int statusCode, final String statusText, final Component component) { diff --git a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java index b3b08ea32..d7648e64b 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java @@ -37,18 +37,21 @@ @Suite @SelectClasses(value = { - RepositoryMetaAnalyzerIT.WithValidPurl.class, + RepositoryMetaAnalyzerIT.WithValidPurlLatestVersionEnabled.class, + RepositoryMetaAnalyzerIT.WithValidPurlWithIntegrityRepoUnsupported.class, RepositoryMetaAnalyzerIT.WithInvalidPurl.class, RepositoryMetaAnalyzerIT.NoCapableAnalyzer.class, - RepositoryMetaAnalyzerIT.InternalAnalyzerNonInternalComponent.class + RepositoryMetaAnalyzerIT.InternalAnalyzerNonInternalComponent.class, + RepositoryMetaAnalyzerIT.WithValidPurlWithBothLatestVersionAndIntegrityEnabled.class, + RepositoryMetaAnalyzerIT.WithValidPurlIntegrityMetaEnabled.class }) class RepositoryMetaAnalyzerIT { @QuarkusIntegrationTest @QuarkusTestResource(KafkaCompanionResource.class) @QuarkusTestResource(WireMockTestResource.class) - @TestProfile(WithValidPurl.TestProfile.class) - static class WithValidPurl { + @TestProfile(WithValidPurlLatestVersionEnabled.TestProfile.class) + static class WithValidPurlLatestVersionEnabled{ public static class TestProfile implements QuarkusTestProfile {} @@ -92,6 +95,15 @@ void beforeEach() throws Exception { } """.getBytes(), new ContentTypeHeader(MediaType.APPLICATION_JSON)) ))); + wireMockServer.stubFor(WireMock.head(WireMock.anyUrl()) + .willReturn(WireMock.aResponse() + .withStatus(200) + .withHeader("X-Checksum-MD5", "md5hash") + .withHeader("X-Checksum-SHA1", "sha1hash") + .withHeader("X-Checksum-SHA256", "sha256hash") + .withHeader("X-Checksum-SHA512", "sha512hash") + .withHeader("Last-Modified", "Wed, 06 Jul 2022 14:00:00 GMT") + )); } @Test @@ -99,6 +111,8 @@ void test() { final var command = AnalysisCommand.newBuilder() .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() .setPurl("pkg:golang/github.com/acme/acme-lib@9.1.1")) + .setFetchLatestVersion(true) + .setFetchIntegrityData(false) .build(); kafkaCompanion @@ -123,6 +137,69 @@ record -> { assertThat(result.getLatestVersion()).isEqualTo("v6.6.6"); assertThat(result.hasPublished()).isTrue(); assertThat(result.getPublished().getSeconds()).isEqualTo(1664402372); + assertThat(result.hasIntegrityMeta()).isFalse(); + } + ); + } + } + + @QuarkusIntegrationTest + @QuarkusTestResource(KafkaCompanionResource.class) + @QuarkusTestResource(WireMockTestResource.class) + @TestProfile(WithValidPurlWithIntegrityRepoUnsupported.TestProfile.class) + static class WithValidPurlWithIntegrityRepoUnsupported{ + + public static class TestProfile implements QuarkusTestProfile {} + + @InjectKafkaCompanion + KafkaCompanion kafkaCompanion; + + @InjectWireMock + WireMockServer wireMockServer; + + @BeforeEach + void beforeEach() throws Exception { + try (final Connection connection = DriverManager.getConnection( + ConfigProvider.getConfig().getValue("quarkus.datasource.jdbc.url", String.class), + ConfigProvider.getConfig().getValue("quarkus.datasource.username", String.class), + ConfigProvider.getConfig().getValue("quarkus.datasource.password", String.class))) { + final PreparedStatement ps = connection.prepareStatement(""" + INSERT INTO "REPOSITORY" ("ENABLED", "IDENTIFIER", "INTERNAL", "PASSWORD", "RESOLUTION_ORDER", "TYPE", "URL", "AUTHENTICATIONREQUIRED") + VALUES ('true', 'test', false, NULL, 1, 'GO_MODULES', 'http://localhost:%d', false); + """.formatted(wireMockServer.port())); + ps.execute(); + } + } + + @Test + void test() { + final var command = AnalysisCommand.newBuilder() + .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() + .setPurl("pkg:golang/github.com/acme/acme-lib@9.1.1")) + .setFetchLatestVersion(false) + .setFetchIntegrityData(true) + .build(); + + kafkaCompanion + .produce(Serdes.String(), new KafkaProtobufSerde<>(AnalysisCommand.parser())) + .fromRecords(new ProducerRecord<>(KafkaTopic.REPO_META_ANALYSIS_COMMAND.getName(), "foo", command)); + + final List> results = kafkaCompanion + .consume(Serdes.String(), new KafkaProtobufSerde<>(AnalysisResult.parser())) + .fromTopics(KafkaTopic.REPO_META_ANALYSIS_RESULT.getName(), 1, Duration.ofSeconds(5)) + .awaitCompletion() + .getRecords(); + + assertThat(results).satisfiesExactly( + record -> { + assertThat(record.key()).isEqualTo("pkg:golang/github.com/acme/acme-lib"); + assertThat(record.value()).isNotNull(); + final AnalysisResult result = record.value(); + assertThat(result.hasComponent()).isTrue(); + assertThat(result.hasRepository()).isFalse(); + assertThat(result.hasLatestVersion()).isFalse(); + assertThat(result.hasPublished()).isFalse(); + assertThat(result.hasIntegrityMeta()).isFalse(); } ); } @@ -231,6 +308,7 @@ record -> { assertThat(result.hasRepository()).isFalse(); assertThat(result.hasLatestVersion()).isFalse(); assertThat(result.hasPublished()).isFalse(); + assertThat(result.hasIntegrityMeta()).isFalse(); } ); } @@ -291,6 +369,197 @@ record -> { assertThat(result.hasRepository()).isFalse(); assertThat(result.hasLatestVersion()).isFalse(); assertThat(result.hasPublished()).isFalse(); + assertThat(result.hasIntegrityMeta()).isFalse(); + } + ); + } + } + + @QuarkusIntegrationTest + @QuarkusTestResource(KafkaCompanionResource.class) + @QuarkusTestResource(WireMockTestResource.class) + @TestProfile(WithValidPurlWithBothLatestVersionAndIntegrityEnabled.TestProfile.class) + static class WithValidPurlWithBothLatestVersionAndIntegrityEnabled { + + public static class TestProfile implements QuarkusTestProfile {} + + @InjectKafkaCompanion + KafkaCompanion kafkaCompanion; + + @InjectWireMock + WireMockServer wireMockServer; + + @BeforeEach + void beforeEach() throws Exception { + try (final Connection connection = DriverManager.getConnection( + ConfigProvider.getConfig().getValue("quarkus.datasource.jdbc.url", String.class), + ConfigProvider.getConfig().getValue("quarkus.datasource.username", String.class), + ConfigProvider.getConfig().getValue("quarkus.datasource.password", String.class))) { + final PreparedStatement ps = connection.prepareStatement(""" + INSERT INTO "REPOSITORY" ("ENABLED", "IDENTIFIER", "INTERNAL", "PASSWORD", "RESOLUTION_ORDER", "TYPE", "URL", "AUTHENTICATIONREQUIRED") + VALUES ('true', 'test', false, NULL, 1, 'NPM', 'http://localhost:%d', false); + """.formatted(wireMockServer.port())); + ps.execute(); + } + + wireMockServer.stubFor(WireMock.get(WireMock.anyUrl()) + .willReturn(WireMock.aResponse() + .withStatus(200) + .withResponseBody(Body.ofBinaryOrText(""" + { + "latest": "v6.6.6" + } + """.getBytes(), new ContentTypeHeader(MediaType.APPLICATION_JSON)) + ))); + wireMockServer.stubFor(WireMock.head(WireMock.anyUrl()) + .willReturn(WireMock.aResponse() + .withStatus(200) + .withHeader("X-Checksum-MD5", "md5hash") + .withHeader("X-Checksum-SHA1", "sha1hash") + .withHeader("Last-Modified", "Wed, 06 Jul 2022 14:00:00 GMT") + )); + } + + @Test + void testBoth() { + final var command = AnalysisCommand.newBuilder() + .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() + .setPurl("pkg:npm/amazon-s3-uri@0.0.1")) + .setFetchLatestVersion(true) + .setFetchIntegrityData(true) + .build(); + + kafkaCompanion + .produce(Serdes.String(), new KafkaProtobufSerde<>(AnalysisCommand.parser())) + .fromRecords(new ProducerRecord<>(KafkaTopic.REPO_META_ANALYSIS_COMMAND.getName(), "foo", command)); + + final List> results = kafkaCompanion + .consume(Serdes.String(), new KafkaProtobufSerde<>(AnalysisResult.parser())) + .fromTopics(KafkaTopic.REPO_META_ANALYSIS_RESULT.getName(), 1, Duration.ofSeconds(5)) + .awaitCompletion() + .getRecords(); + + assertThat(results).satisfiesExactly( + record -> { + assertThat(record.key()).isEqualTo("pkg:npm/amazon-s3-uri"); + assertThat(record.value()).isNotNull(); + final AnalysisResult result = record.value(); + assertThat(result.hasComponent()).isTrue(); + assertThat(result.getRepository()).isEqualTo("test"); + assertThat(result.getLatestVersion()).isEqualTo("v6.6.6"); + assertThat(result.hasPublished()).isFalse(); + assertThat(result.hasIntegrityMeta()).isTrue(); + final var integrityMeta = result.getIntegrityMeta(); + assertThat(integrityMeta).isNotNull(); + } + ); + } + } + + @QuarkusIntegrationTest + @QuarkusTestResource(KafkaCompanionResource.class) + @QuarkusTestResource(WireMockTestResource.class) + @TestProfile(WithValidPurlIntegrityMetaEnabled.TestProfile.class) + static class WithValidPurlIntegrityMetaEnabled { + + public static class TestProfile implements QuarkusTestProfile {} + + @InjectKafkaCompanion + KafkaCompanion kafkaCompanion; + + @InjectWireMock + WireMockServer wireMockServer; + + @BeforeEach + void beforeEach() throws Exception { + try (final Connection connection = DriverManager.getConnection( + ConfigProvider.getConfig().getValue("quarkus.datasource.jdbc.url", String.class), + ConfigProvider.getConfig().getValue("quarkus.datasource.username", String.class), + ConfigProvider.getConfig().getValue("quarkus.datasource.password", String.class))) { + final PreparedStatement ps = connection.prepareStatement(""" + INSERT INTO "REPOSITORY" ("ENABLED", "IDENTIFIER", "INTERNAL", "PASSWORD", "RESOLUTION_ORDER", "TYPE", "URL", "AUTHENTICATIONREQUIRED") + VALUES ('true', 'test', false, NULL, 1, 'NPM', 'http://localhost:%d', false); + """.formatted(wireMockServer.port())); + ps.execute(); + } + wireMockServer.stubFor(WireMock.head(WireMock.anyUrl()) + .willReturn(WireMock.aResponse() + .withStatus(200) + .withHeader("X-Checksum-MD5", "md5hash") + .withHeader("X-Checksum-SHA1", "sha1hash") + .withHeader("Last-Modified", "Wed, 06 Jul 2022 14:00:00 GMT") + )); + } + +// @Test +// void testLatestVersionOnly() { +// final var command = AnalysisCommand.newBuilder() +// .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() +// .setPurl("pkg:npm/github.com/acme/acme-lib@9.1.1")) +// .setFetchLatestVersion(true) +// .setFetchIntegrityData(false) +// .build(); +// +// kafkaCompanion +// .produce(Serdes.String(), new KafkaProtobufSerde<>(AnalysisCommand.parser())) +// .fromRecords(new ProducerRecord<>(KafkaTopic.REPO_META_ANALYSIS_COMMAND.getName(), "foo", command)); +// +// final List> results = kafkaCompanion +// .consume(Serdes.String(), new KafkaProtobufSerde<>(AnalysisResult.parser())) +// .fromTopics(KafkaTopic.REPO_META_ANALYSIS_RESULT.getName(), 1, Duration.ofSeconds(5)) +// .awaitCompletion() +// .getRecords(); +// +// assertThat(results).satisfiesExactly( +// record -> { +// assertThat(record.key()).isEqualTo("pkg:npm/github.com/acme/acme-lib"); +// assertThat(record.value()).isNotNull(); +// final AnalysisResult result = record.value(); +// assertThat(result.hasComponent()).isTrue(); +// assertThat(result.hasRepository()).isTrue(); +// assertThat(result.getRepository()).isEqualTo("test"); +// assertThat(result.hasLatestVersion()).isTrue(); +// assertThat(result.getLatestVersion()).isEqualTo("v6.6.6"); +// assertThat(result.hasPublished()).isFalse(); +// assertThat(result.hasIntegrityMeta()).isFalse(); +// } +// ); +// } + + @Test + void testIntegrityMetaOnly() { + final var command = AnalysisCommand.newBuilder() + .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() + .setPurl("pkg:npm/amazon-s3-uri@0.0.1")) + .setFetchLatestVersion(false) + .setFetchIntegrityData(true) + .build(); + + kafkaCompanion + .produce(Serdes.String(), new KafkaProtobufSerde<>(AnalysisCommand.parser())) + .fromRecords(new ProducerRecord<>(KafkaTopic.REPO_META_ANALYSIS_COMMAND.getName(), "foo", command)); + + final List> results = kafkaCompanion + .consume(Serdes.String(), new KafkaProtobufSerde<>(AnalysisResult.parser())) + .fromTopics(KafkaTopic.REPO_META_ANALYSIS_RESULT.getName(), 1, Duration.ofSeconds(5)) + .awaitCompletion() + .getRecords(); + + assertThat(results).satisfiesExactly( + record -> { + assertThat(record.key()).isEqualTo("pkg:npm/amazon-s3-uri"); + assertThat(record.value()).isNotNull(); + final AnalysisResult result = record.value(); + assertThat(result.hasComponent()).isTrue(); + assertThat(result.getRepository()).isEqualTo("test"); + assertThat(result.hasLatestVersion()).isFalse(); + assertThat(result.hasPublished()).isFalse(); + assertThat(result.hasIntegrityMeta()).isTrue(); + final var integrityMeta = result.getIntegrityMeta(); + assertThat(integrityMeta.getMd5()).isEqualTo("md5hash"); + assertThat(integrityMeta.getSha1()).isEqualTo("sha1hash"); + assertThat(integrityMeta.getCurrentVersionLastModified().getSeconds()).isEqualTo(1657116000); + assertThat(integrityMeta.getRepositoryUrl()).contains("/amazon-s3-uri/-/amazon-s3-uri-0.0.1.tgz"); } ); } diff --git a/repository-meta-analyzer/src/test/java/org/hyades/repositories/HexMetaAnalyzerTest.java b/repository-meta-analyzer/src/test/java/org/hyades/repositories/HexMetaAnalyzerTest.java index 19cd199db..bf2b8124b 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/repositories/HexMetaAnalyzerTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/repositories/HexMetaAnalyzerTest.java @@ -32,6 +32,7 @@ import org.mockserver.client.MockServerClient; import org.mockserver.integration.ClientAndServer; +import static org.junit.Assert.assertThrows; import static org.mockserver.model.HttpRequest.request; import static org.mockserver.model.HttpResponse.response; @@ -145,4 +146,11 @@ void testAnalyzerReturnEmptyResultWithBraces() throws Exception { metaModel.getPublishedTimestamp() ); } + + @Test + void testIntegrityAnalyzerNotSupported() { + Component component = new Component(); + component.setPurl("pkg:pypi/typo3/package-empty-result@v1.2.0"); + assertThrows(UnsupportedOperationException.class, () -> analyzer.getIntegrityMeta(component)); + } } From b215faff4f03439599ca0d305ada5015d239aa29 Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Thu, 5 Oct 2023 18:08:06 +0100 Subject: [PATCH 05/17] Update RepositoryMetaAnalyzerTopologyTest.java --- .../test/java/org/hyades/RepositoryMetaAnalyzerTopologyTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerTopologyTest.java b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerTopologyTest.java index 29307b3af..bcb62690d 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerTopologyTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerTopologyTest.java @@ -183,6 +183,7 @@ void testMetaOutput() { Assertions.assertEquals("pkg:maven/com.fasterxml.jackson.core/jackson-databind", record.key); Assertions.assertEquals("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", record.value.getComponent().getPurl()); Assertions.assertFalse(record.value.hasLatestVersion()); + Assertions.assertFalse(record.value.hasIntegrityMeta()); } @AfterEach From d7d8c70f3fffed60aeb5a33f5b9cb016c928e438 Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Fri, 6 Oct 2023 09:36:50 +0100 Subject: [PATCH 06/17] fix test fail --- .../java/org/hyades/repositories/MavenMetaAnalyzerTest.java | 2 +- .../test/java/org/hyades/repositories/NpmMetaAnalyzerTest.java | 2 +- .../test/java/org/hyades/repositories/PypiMetaAnalyzerTest.java | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java b/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java index fdf4b19ee..204af512f 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java @@ -158,7 +158,7 @@ void testGetIntegrityModel200() { assertEquals("sha1hash", integrityMeta.getSha1()); assertEquals("sha256hash", integrityMeta.getSha256()); assertEquals("sha512hash", integrityMeta.getSha512()); - assertEquals("Thu Jul 07 15:00:00 IST 2022", integrityMeta.getCurrentVersionLastModified().toString()); + assertNotNull(integrityMeta.getCurrentVersionLastModified()); } } diff --git a/repository-meta-analyzer/src/test/java/org/hyades/repositories/NpmMetaAnalyzerTest.java b/repository-meta-analyzer/src/test/java/org/hyades/repositories/NpmMetaAnalyzerTest.java index 9e6d2060c..10df80d6a 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/repositories/NpmMetaAnalyzerTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/repositories/NpmMetaAnalyzerTest.java @@ -184,7 +184,7 @@ void testAnalyzerReturnIntegrityResult() { assertEquals("sha1hash", integrityMeta.getSha1()); assertEquals("sha256hash", integrityMeta.getSha256()); assertEquals("sha512hash", integrityMeta.getSha512()); - assertEquals("Thu Jul 07 15:00:00 IST 2022", integrityMeta.getCurrentVersionLastModified().toString()); + assertNotNull(integrityMeta.getCurrentVersionLastModified()); } @Test diff --git a/repository-meta-analyzer/src/test/java/org/hyades/repositories/PypiMetaAnalyzerTest.java b/repository-meta-analyzer/src/test/java/org/hyades/repositories/PypiMetaAnalyzerTest.java index 77ef862ba..af1ab41f5 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/repositories/PypiMetaAnalyzerTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/repositories/PypiMetaAnalyzerTest.java @@ -183,7 +183,7 @@ void testAnalyzerReturnIntegrityResult() { assertEquals("sha1hash", integrityMeta.getSha1()); assertEquals("sha256hash", integrityMeta.getSha256()); assertEquals("sha512hash", integrityMeta.getSha512()); - assertEquals("Thu Jul 07 15:00:00 IST 2022", integrityMeta.getCurrentVersionLastModified().toString()); + assertNotNull(integrityMeta.getCurrentVersionLastModified()); } @Test From 8dcb75f275673a7e0829c8456668ddd928d3ce69 Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Fri, 6 Oct 2023 11:41:02 +0100 Subject: [PATCH 07/17] added more test --- .../org/hyades/RepositoryMetaAnalyzerIT.java | 35 -------- .../processor/MetaAnalyzerProcessorTest.java | 86 +++++++++++++++++++ 2 files changed, 86 insertions(+), 35 deletions(-) diff --git a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java index d7648e64b..e97bff80b 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java @@ -491,41 +491,6 @@ void beforeEach() throws Exception { )); } -// @Test -// void testLatestVersionOnly() { -// final var command = AnalysisCommand.newBuilder() -// .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() -// .setPurl("pkg:npm/github.com/acme/acme-lib@9.1.1")) -// .setFetchLatestVersion(true) -// .setFetchIntegrityData(false) -// .build(); -// -// kafkaCompanion -// .produce(Serdes.String(), new KafkaProtobufSerde<>(AnalysisCommand.parser())) -// .fromRecords(new ProducerRecord<>(KafkaTopic.REPO_META_ANALYSIS_COMMAND.getName(), "foo", command)); -// -// final List> results = kafkaCompanion -// .consume(Serdes.String(), new KafkaProtobufSerde<>(AnalysisResult.parser())) -// .fromTopics(KafkaTopic.REPO_META_ANALYSIS_RESULT.getName(), 1, Duration.ofSeconds(5)) -// .awaitCompletion() -// .getRecords(); -// -// assertThat(results).satisfiesExactly( -// record -> { -// assertThat(record.key()).isEqualTo("pkg:npm/github.com/acme/acme-lib"); -// assertThat(record.value()).isNotNull(); -// final AnalysisResult result = record.value(); -// assertThat(result.hasComponent()).isTrue(); -// assertThat(result.hasRepository()).isTrue(); -// assertThat(result.getRepository()).isEqualTo("test"); -// assertThat(result.hasLatestVersion()).isTrue(); -// assertThat(result.getLatestVersion()).isEqualTo("v6.6.6"); -// assertThat(result.hasPublished()).isFalse(); -// assertThat(result.hasIntegrityMeta()).isFalse(); -// } -// ); -// } - @Test void testIntegrityMetaOnly() { final var command = AnalysisCommand.newBuilder() diff --git a/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java b/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java index e0c6d96f3..52f423d15 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java @@ -2,6 +2,8 @@ import com.github.packageurl.MalformedPackageURLException; import com.github.packageurl.PackageURL; +import com.github.tomakehurst.wiremock.http.Body; +import com.github.tomakehurst.wiremock.http.ContentTypeHeader; import io.quarkus.cache.Cache; import io.quarkus.cache.CacheName; import io.quarkus.test.TestTransaction; @@ -10,6 +12,7 @@ import io.quarkus.test.junit.TestProfile; import jakarta.inject.Inject; import jakarta.persistence.EntityManager; +import jakarta.ws.rs.core.MediaType; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.TestInputTopic; import org.apache.kafka.streams.TestOutputTopic; @@ -27,14 +30,20 @@ import org.hyades.proto.repometaanalysis.v1.Component; import org.hyades.repositories.RepositoryAnalyzerFactory; import org.hyades.serde.KafkaPurlSerde; +import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockserver.client.MockServerClient; +import org.mockserver.integration.ClientAndServer; import java.util.Map; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockserver.model.HttpRequest.request; +import static org.mockserver.model.HttpResponse.response; @QuarkusTest @TestProfile(MetaAnalyzerProcessorTest.TestProfile.class) @@ -49,6 +58,8 @@ public Map getConfigOverrides() { } } + private static ClientAndServer mockServer; + private static final String TEST_PURL_JACKSON_BIND = "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4"; private TopologyTestDriver testDriver; @@ -70,6 +81,11 @@ public Map getConfigOverrides() { @Inject SecretDecryptor secretDecryptor; + @BeforeAll + static void beforeClass() { + mockServer = ClientAndServer.startClientAndServer(1080); + } + @BeforeEach void beforeEach() { final var processorSupplier = new MetaAnalyzerProcessorSupplier(repoEntityRepository, analyzerFactory, secretDecryptor, cache); @@ -95,6 +111,11 @@ void afterEach() { cache.invalidateAll().await().indefinitely(); } + @AfterAll + static void afterClass() { + mockServer.stop(); + } + @Test void testWithNoSupportedRepositoryTypes() throws Exception { final TestRecord inputRecord = new TestRecord<>(new PackageURL(TEST_PURL_JACKSON_BIND), AnalysisCommand.newBuilder().setComponent(Component.newBuilder() @@ -168,4 +189,69 @@ record -> { } + @Test + @TestTransaction + void testRepoMetaWithIntegrityMetaWithAuth() throws Exception { + entityManager.createNativeQuery(""" + INSERT INTO "REPOSITORY" ("TYPE", "ENABLED","IDENTIFIER", "INTERNAL", "URL", "AUTHENTICATIONREQUIRED", "RESOLUTION_ORDER", "USERNAME", "PASSWORD") VALUES + ('NPM', true, 'central', true, :url, true, 1, 'username', :encryptedPassword); + """) + .setParameter("encryptedPassword", secretDecryptor.encryptAsString("password")) + .setParameter("url", String.format("http://localhost:%d", mockServer.getPort())) + .executeUpdate(); + + new MockServerClient("localhost", mockServer.getPort()) + .when( + request() + .withMethod("GET") + .withPath("/-/package/%40apollo%2Ffederation/dist-tags") + ) + .respond( + response() + .withStatusCode(200) + .withBody(Body.ofBinaryOrText(""" + { + "latest": "v6.6.6" + } + """.getBytes(), new ContentTypeHeader(MediaType.APPLICATION_JSON)).asBytes() + )); + + new MockServerClient("localhost", mockServer.getPort()) + .when( + request() + .withMethod("HEAD") + .withPath("/@apollo/federation/-/@apollo/federation-0.19.1.tgz") + ) + .respond( + response() + .withStatusCode(200) + .withHeader("X-Checksum-MD5", "md5hash") + ); + + final TestRecord inputRecord = new TestRecord<>(new PackageURL("pkg:npm/@apollo/federation@0.19.1"), + AnalysisCommand.newBuilder() + .setComponent(Component.newBuilder() + .setPurl("pkg:npm/@apollo/federation@0.19.1") + .setInternal(true)) + .setFetchIntegrityData(true) + .setFetchLatestVersion(true).build()); + + inputTopic.pipeInput(inputRecord); + assertThat(outputTopic.getQueueSize()).isEqualTo(1); + assertThat(outputTopic.readRecordsToList()).satisfiesExactly( + record -> { + assertThat(record.key().getType()).isEqualTo(RepositoryType.NPM.toString().toLowerCase()); + assertThat(record.value()).isNotNull(); + final AnalysisResult result = record.value(); + assertThat(result.hasComponent()).isTrue(); + assertThat(result.getRepository()).isEqualTo("central"); + assertThat(result.getLatestVersion()).isEqualTo("v6.6.6"); + assertThat(result.hasPublished()).isFalse(); + assertThat(result.hasIntegrityMeta()).isTrue(); + final var integrityMeta = result.getIntegrityMeta(); + assertThat(integrityMeta.getMd5()).isEqualTo("md5hash"); + assertThat(integrityMeta.getRepositoryUrl()).contains("/@apollo/federation/-/@apollo/federation-0.19.1.tgz"); + }); + + } } \ No newline at end of file From 746a1b5679343853eb539de8f8aa7d49a2b3ced4 Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Mon, 9 Oct 2023 10:39:49 +0100 Subject: [PATCH 08/17] move date parsing to util --- .../main/java/org/hyades/commonutil/DateUtil.java | 11 ++++++----- .../java/org/hyades/commonutil/DateUtilTest.java | 14 +++++++++++++- .../hyades/repositories/AbstractMetaAnalyzer.java | 9 ++------- .../org/hyades/repositories/MavenMetaAnalyzer.java | 3 ++- 4 files changed, 23 insertions(+), 14 deletions(-) diff --git a/commons/src/main/java/org/hyades/commonutil/DateUtil.java b/commons/src/main/java/org/hyades/commonutil/DateUtil.java index dec61132e..d14286085 100644 --- a/commons/src/main/java/org/hyades/commonutil/DateUtil.java +++ b/commons/src/main/java/org/hyades/commonutil/DateUtil.java @@ -30,16 +30,17 @@ private DateUtil() { } /** - * Convenience method that parses a date in yyyyMMddHHmmss format and + * Convenience method that parses a date in give format and * returns a Date object. If the parsing fails, null is returned. - * @param yyyyMMddHHmmss the date string to parse + * @param date the date string to parse + * @param dateFormat the date format * @return a Date object * @since 3.1.0 */ - public static Date parseDate(final String yyyyMMddHHmmss) { - final SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHHmmss"); + public static Date parseDate(final String date, final String dateFormat) { + final SimpleDateFormat format = new SimpleDateFormat(dateFormat); try { - return format.parse(yyyyMMddHHmmss); + return format.parse(date); } catch (ParseException e) { return null; } diff --git a/commons/src/test/java/org/hyades/commonutil/DateUtilTest.java b/commons/src/test/java/org/hyades/commonutil/DateUtilTest.java index 751399f60..8214d8390 100644 --- a/commons/src/test/java/org/hyades/commonutil/DateUtilTest.java +++ b/commons/src/test/java/org/hyades/commonutil/DateUtilTest.java @@ -31,7 +31,7 @@ public class DateUtilTest { @Test public void testParseDate() { - Date date = DateUtil.parseDate("20191231153012"); + Date date = DateUtil.parseDate("20191231153012", "yyyyMMddHHmmss"); LocalDateTime localDateTime = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); Assertions.assertEquals(Month.DECEMBER, localDateTime.getMonth()); Assertions.assertEquals(31, localDateTime.getDayOfMonth()); @@ -41,6 +41,18 @@ public void testParseDate() { Assertions.assertEquals(12, localDateTime.getSecond()); } + @Test + public void testParseGMTDate() { + Date date = DateUtil.parseDate("Thu, 07 Jul 2022 14:00:00 GMT", "EEE, dd MMM yyyy HH:mm:ss Z"); + LocalDateTime localDateTime = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); + Assertions.assertEquals(Month.JULY, localDateTime.getMonth()); + Assertions.assertEquals(7, localDateTime.getDayOfMonth()); + Assertions.assertEquals(2022, localDateTime.getYear()); + Assertions.assertEquals(15, localDateTime.getHour()); + Assertions.assertEquals(0, localDateTime.getMinute()); + Assertions.assertEquals(0, localDateTime.getSecond()); + } + @Test public void testToISO8601() { Date date = Date.from(LocalDateTime.of(2019, Month.JANUARY, 31, 15, 30, 12).toInstant(ZoneOffset.UTC)); diff --git a/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java index 97e7d5362..2e2d9af6c 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java @@ -26,6 +26,7 @@ import org.apache.http.client.methods.HttpHead; import org.apache.http.client.methods.HttpUriRequest; import org.apache.http.impl.client.CloseableHttpClient; +import org.hyades.commonutil.DateUtil; import org.hyades.commonutil.HttpUtil; import org.hyades.model.IntegrityMeta; import org.hyades.persistence.model.Component; @@ -33,8 +34,6 @@ import org.slf4j.LoggerFactory; import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; /** * Base abstract class that all IMetaAnalyzer implementations should likely extend. @@ -150,11 +149,7 @@ protected IntegrityMeta extractIntegrityModelFromResponse(CloseableHttpResponse } else if (header.getName().equalsIgnoreCase("X-Checksum-SHA512")) { integrityMeta.setSha512(header.getValue()); } else if (header.getName().equalsIgnoreCase("Last-Modified") && header.getValue() != null) { - try { - integrityMeta.setCurrentVersionLastModified(new SimpleDateFormat(GMT_FORMAT).parse(header.getValue())); - } catch (ParseException pe) { - logger.warn("Parsing LastModified date failed while extracting integrity meta for purl: {}", component.getPurl()); - } + integrityMeta.setCurrentVersionLastModified(DateUtil.parseDate(header.getValue(), GMT_FORMAT)); } } return integrityMeta; diff --git a/repository-meta-analyzer/src/main/java/org/hyades/repositories/MavenMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/hyades/repositories/MavenMetaAnalyzer.java index 4c3c73a6a..d72a89a7b 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/repositories/MavenMetaAnalyzer.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/repositories/MavenMetaAnalyzer.java @@ -53,6 +53,7 @@ public class MavenMetaAnalyzer extends AbstractMetaAnalyzer { private static final Logger LOGGER = LoggerFactory.getLogger(MavenMetaAnalyzer.class); private static final String DEFAULT_BASE_URL = "https://repo1.maven.org/maven2"; private static final String REPO_METADATA_URL = "/%s/maven-metadata.xml"; + private final String MAVEN_REPO_DATE_FORMAT = "yyyyMMddHHmmss"; MavenMetaAnalyzer() { this.baseUrl = DEFAULT_BASE_URL; @@ -100,7 +101,7 @@ public MetaModel analyze(final Component component) { meta.setLatestVersion(release != null ? release : latest); if (lastUpdated != null) { - meta.setPublishedTimestamp(DateUtil.parseDate(lastUpdated)); + meta.setPublishedTimestamp(DateUtil.parseDate(lastUpdated, MAVEN_REPO_DATE_FORMAT)); } } } From 606b2c79eddd382520b68fbc26b7093969d5f906 Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Mon, 9 Oct 2023 10:47:32 +0100 Subject: [PATCH 09/17] update repositoryUrl to metaSourceUrl --- .../repometaanalysis/v1/repo_meta_analysis.proto | 2 +- .../src/main/java/org/hyades/model/IntegrityMeta.java | 10 +++++----- .../org/hyades/processor/MetaAnalyzerProcessor.java | 4 ++-- .../org/hyades/repositories/AbstractMetaAnalyzer.java | 6 +++--- .../test/java/org/hyades/RepositoryMetaAnalyzerIT.java | 2 +- .../hyades/processor/MetaAnalyzerProcessorTest.java | 2 +- .../org/hyades/repositories/NpmMetaAnalyzerTest.java | 6 +++--- .../org/hyades/repositories/PypiMetaAnalyzerTest.java | 6 +++--- 8 files changed, 19 insertions(+), 19 deletions(-) diff --git a/proto/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto b/proto/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto index 4b57a7088..6c5ea0383 100644 --- a/proto/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto +++ b/proto/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto @@ -49,5 +49,5 @@ message IntegrityMeta { // When the component current version last modified. optional google.protobuf.Timestamp current_version_last_modified = 5; // Complete URL to fetch integrity metadata of the component. - optional string repository_url = 6; + optional string meta_source_url = 6; } diff --git a/repository-meta-analyzer/src/main/java/org/hyades/model/IntegrityMeta.java b/repository-meta-analyzer/src/main/java/org/hyades/model/IntegrityMeta.java index 00cdee2d2..849af0c85 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/model/IntegrityMeta.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/model/IntegrityMeta.java @@ -15,7 +15,7 @@ public class IntegrityMeta implements Serializable { private Date currentVersionLastModified; - private String repositoryUrl; + private String metaSourceUrl; public String getMd5() { return md5; @@ -57,11 +57,11 @@ public String getSha512() { return sha512; } - public String getRepositoryUrl() { - return repositoryUrl; + public String getMetaSourceUrl() { + return metaSourceUrl; } - public void setRepositoryUrl(String repositoryUrl) { - this.repositoryUrl = repositoryUrl; + public void setMetaSourceUrl(String metaSourceUrl) { + this.metaSourceUrl = metaSourceUrl; } } \ No newline at end of file diff --git a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java index 8d4ae522b..e56cb5116 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java @@ -168,8 +168,8 @@ private Optional analyze(final IMetaAnalyzer analyzer, final Rep if (integrityMeta.getSha512() != null) { metaBuilder.setSha512(integrityMeta.getSha512()); } - if (integrityMeta.getRepositoryUrl() != null) { - metaBuilder.setRepositoryUrl(integrityMeta.getRepositoryUrl()); + if (integrityMeta.getMetaSourceUrl() != null) { + metaBuilder.setMetaSourceUrl(integrityMeta.getMetaSourceUrl()); } if (integrityMeta.getCurrentVersionLastModified() != null) { metaBuilder.setCurrentVersionLastModified(Timestamp.newBuilder() diff --git a/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java index 2e2d9af6c..19643f16a 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/repositories/AbstractMetaAnalyzer.java @@ -137,7 +137,7 @@ protected CloseableHttpResponse processHttpHeadRequest(String url) throws IOExce return httpClient.execute(request); } - protected IntegrityMeta extractIntegrityModelFromResponse(CloseableHttpResponse response, IntegrityMeta integrityMeta, Component component) { + protected IntegrityMeta extractIntegrityModelFromResponse(CloseableHttpResponse response, IntegrityMeta integrityMeta) { var headers = response.getAllHeaders(); for (var header : headers) { if (header.getName().equalsIgnoreCase("X-Checksum-MD5")) { @@ -157,11 +157,11 @@ protected IntegrityMeta extractIntegrityModelFromResponse(CloseableHttpResponse public IntegrityMeta fetchIntegrityMeta(String url, Component component) { var integrityMeta = new IntegrityMeta(); - integrityMeta.setRepositoryUrl(url); + integrityMeta.setMetaSourceUrl(url); try (final CloseableHttpResponse response = processHttpHeadRequest(url)) { final StatusLine status = response.getStatusLine(); if (status.getStatusCode() == HttpStatus.SC_OK) { - return extractIntegrityModelFromResponse(response, integrityMeta, component); + return extractIntegrityModelFromResponse(response, integrityMeta); } else { handleUnexpectedHttpResponse(logger, url, status.getStatusCode(), status.getReasonPhrase(), component); } diff --git a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java index e97bff80b..d94ee8085 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java @@ -524,7 +524,7 @@ record -> { assertThat(integrityMeta.getMd5()).isEqualTo("md5hash"); assertThat(integrityMeta.getSha1()).isEqualTo("sha1hash"); assertThat(integrityMeta.getCurrentVersionLastModified().getSeconds()).isEqualTo(1657116000); - assertThat(integrityMeta.getRepositoryUrl()).contains("/amazon-s3-uri/-/amazon-s3-uri-0.0.1.tgz"); + assertThat(integrityMeta.getMetaSourceUrl()).contains("/amazon-s3-uri/-/amazon-s3-uri-0.0.1.tgz"); } ); } diff --git a/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java b/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java index 52f423d15..0efb59beb 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java @@ -250,7 +250,7 @@ record -> { assertThat(result.hasIntegrityMeta()).isTrue(); final var integrityMeta = result.getIntegrityMeta(); assertThat(integrityMeta.getMd5()).isEqualTo("md5hash"); - assertThat(integrityMeta.getRepositoryUrl()).contains("/@apollo/federation/-/@apollo/federation-0.19.1.tgz"); + assertThat(integrityMeta.getMetaSourceUrl()).contains("/@apollo/federation/-/@apollo/federation-0.19.1.tgz"); }); } diff --git a/repository-meta-analyzer/src/test/java/org/hyades/repositories/NpmMetaAnalyzerTest.java b/repository-meta-analyzer/src/test/java/org/hyades/repositories/NpmMetaAnalyzerTest.java index 10df80d6a..882f19736 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/repositories/NpmMetaAnalyzerTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/repositories/NpmMetaAnalyzerTest.java @@ -179,7 +179,7 @@ void testAnalyzerReturnIntegrityResult() { var integrityMeta = analyzer.getIntegrityMeta(component); assertNotNull(integrityMeta); - assertThat(integrityMeta.getRepositoryUrl()).contains("/typo3/package-empty-result/-/typo3/package-empty-result-v1.2.0.tgz"); + assertThat(integrityMeta.getMetaSourceUrl()).contains("/typo3/package-empty-result/-/typo3/package-empty-result-v1.2.0.tgz"); assertEquals("md5hash", integrityMeta.getMd5()); assertEquals("sha1hash", integrityMeta.getSha1()); assertEquals("sha256hash", integrityMeta.getSha256()); @@ -205,7 +205,7 @@ void testIntegrityResultForPurlWithoutNamespace() { var integrityMeta = analyzer.getIntegrityMeta(component); assertNotNull(integrityMeta); - assertThat(integrityMeta.getRepositoryUrl()).contains("/amazon-s3-uri/-/amazon-s3-uri-0.0.1.tgz"); + assertThat(integrityMeta.getMetaSourceUrl()).contains("/amazon-s3-uri/-/amazon-s3-uri-0.0.1.tgz"); } @Test @@ -225,7 +225,7 @@ void testIntegrityAnalyzerException() { ); var integrityMeta = analyzer.getIntegrityMeta(component); assertNotNull(integrityMeta); - assertThat(integrityMeta.getRepositoryUrl()).contains("/amazon-s3-uri/-/amazon-s3-uri-0.0.1.tgz"); + assertThat(integrityMeta.getMetaSourceUrl()).contains("/amazon-s3-uri/-/amazon-s3-uri-0.0.1.tgz"); assertNull(integrityMeta.getSha1()); assertNull(integrityMeta.getMd5()); assertNull(integrityMeta.getSha256()); diff --git a/repository-meta-analyzer/src/test/java/org/hyades/repositories/PypiMetaAnalyzerTest.java b/repository-meta-analyzer/src/test/java/org/hyades/repositories/PypiMetaAnalyzerTest.java index af1ab41f5..4470e30b0 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/repositories/PypiMetaAnalyzerTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/repositories/PypiMetaAnalyzerTest.java @@ -178,7 +178,7 @@ void testAnalyzerReturnIntegrityResult() { var integrityMeta = analyzer.getIntegrityMeta(component); assertNotNull(integrityMeta); - assertThat(integrityMeta.getRepositoryUrl()).contains("/typo3/package-ok-result/v1.2.0/package-ok-result-v1.2.0.tar.gz"); + assertThat(integrityMeta.getMetaSourceUrl()).contains("/typo3/package-ok-result/v1.2.0/package-ok-result-v1.2.0.tar.gz"); assertEquals("md5hash", integrityMeta.getMd5()); assertEquals("sha1hash", integrityMeta.getSha1()); assertEquals("sha256hash", integrityMeta.getSha256()); @@ -202,7 +202,7 @@ void testIntegrityAnalyzerException() { ); var integrityMeta = analyzer.getIntegrityMeta(component); assertNotNull(integrityMeta); - assertThat(integrityMeta.getRepositoryUrl()).contains("/typo1/package-no-result/v1.2.0/package-no-result-v1.2.0.tar.gz"); + assertThat(integrityMeta.getMetaSourceUrl()).contains("/typo1/package-no-result/v1.2.0/package-no-result-v1.2.0.tar.gz"); assertNull(integrityMeta.getSha1()); assertNull(integrityMeta.getMd5()); assertNull(integrityMeta.getSha256()); @@ -228,6 +228,6 @@ void testIntegrityResultForPurlWithoutNamespace() { var integrityMeta = analyzer.getIntegrityMeta(component); assertNotNull(integrityMeta); - assertThat(integrityMeta.getRepositoryUrl()).contains("/package-result/v1.2.0/package-result-v1.2.0.tar.gz"); + assertThat(integrityMeta.getMetaSourceUrl()).contains("/package-result/v1.2.0/package-result-v1.2.0.tar.gz"); } } From deaa045168afb1de19d0fe1fa0556641bae4e85e Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Mon, 9 Oct 2023 11:02:06 +0100 Subject: [PATCH 10/17] removed if blocks --- .../processor/MetaAnalyzerProcessor.java | 27 ++++++------------- .../repositories/MavenMetaAnalyzerTest.java | 4 +-- 2 files changed, 10 insertions(+), 21 deletions(-) diff --git a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java index e56cb5116..594147558 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java @@ -156,25 +156,14 @@ private Optional analyze(final IMetaAnalyzer analyzer, final Rep } if (integrityMeta != null) { IntegrityMeta.Builder metaBuilder = IntegrityMeta.newBuilder(); - if (integrityMeta.getMd5() != null) { - metaBuilder.setMd5(integrityMeta.getMd5()); - } - if (integrityMeta.getSha1() != null) { - metaBuilder.setSha1(integrityMeta.getSha1()); - } - if (integrityMeta.getSha256() != null) { - metaBuilder.setSha256(integrityMeta.getSha256()); - } - if (integrityMeta.getSha512() != null) { - metaBuilder.setSha512(integrityMeta.getSha512()); - } - if (integrityMeta.getMetaSourceUrl() != null) { - metaBuilder.setMetaSourceUrl(integrityMeta.getMetaSourceUrl()); - } - if (integrityMeta.getCurrentVersionLastModified() != null) { - metaBuilder.setCurrentVersionLastModified(Timestamp.newBuilder() - .setSeconds(integrityMeta.getCurrentVersionLastModified().getTime() / 1000)); - } + Optional.ofNullable(integrityMeta.getMd5()).ifPresent(hash -> metaBuilder.setMd5(hash)); + Optional.ofNullable(integrityMeta.getSha1()).ifPresent(hash -> metaBuilder.setSha1(hash)); + Optional.ofNullable(integrityMeta.getSha256()).ifPresent(hash -> metaBuilder.setSha256(hash)); + Optional.ofNullable(integrityMeta.getSha512()).ifPresent(hash -> metaBuilder.setSha512(hash)); + Optional.ofNullable(integrityMeta.getMetaSourceUrl()).ifPresent(url -> metaBuilder.setMetaSourceUrl(url)); + Optional.ofNullable(integrityMeta.getCurrentVersionLastModified()).ifPresent(date -> + metaBuilder.setCurrentVersionLastModified(Timestamp.newBuilder() + .setSeconds(date.getTime() / 1000))); resultBuilder.setIntegrityMeta(metaBuilder); LOGGER.debug("Found integrity metadata for: {} using repository: {} ({})", component.getPurl(), repository.getIdentifier(), repository.getType()); diff --git a/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java b/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java index 204af512f..c966a75c6 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/repositories/MavenMetaAnalyzerTest.java @@ -126,7 +126,7 @@ void testGetIntegrityMeta404() { component.setPurl("pkg:maven/typo3/package-empty-result@v1.2.0"); var integrityMeta = analyzer.getIntegrityMeta(component); assertNotNull(integrityMeta); - assertEquals("https://repo1.maven.org/maven2/typo3/package-empty-result/v1.2.0/package-empty-result-v1.2.0.jar", integrityMeta.getRepositoryUrl()); + assertEquals("https://repo1.maven.org/maven2/typo3/package-empty-result/v1.2.0/package-empty-result-v1.2.0.jar", integrityMeta.getMetaSourceUrl()); assertNull(integrityMeta.getSha1()); assertNull(integrityMeta.getMd5()); assertNull(integrityMeta.getSha256()); @@ -153,7 +153,7 @@ void testGetIntegrityModel200() { analyzer.setRepositoryBaseUrl(wireMock.baseUrl()); var integrityMeta = analyzer.getIntegrityMeta(component); assertNotNull(integrityMeta); - assertThat(integrityMeta.getRepositoryUrl()).contains("/typo3/package-empty-result/v1.2.0/package-empty-result-v1.2.0.jar"); + assertThat(integrityMeta.getMetaSourceUrl()).contains("/typo3/package-empty-result/v1.2.0/package-empty-result-v1.2.0.jar"); assertEquals("md5hash", integrityMeta.getMd5()); assertEquals("sha1hash", integrityMeta.getSha1()); assertEquals("sha256hash", integrityMeta.getSha256()); From f77c34e311ff0f199853189ab8037350c4460962 Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Tue, 10 Oct 2023 10:38:59 +0100 Subject: [PATCH 11/17] refactoring --- .../RepositoryMetaAnalyzerTopology.java | 23 +- .../processor/MetaAnalyzerProcessor.java | 220 ++++++++++-------- .../main/java/org/hyades/util/PurlUtil.java | 26 +++ .../RepositoryMetaAnalyzerTopologyTest.java | 68 +++++- 4 files changed, 212 insertions(+), 125 deletions(-) create mode 100644 repository-meta-analyzer/src/main/java/org/hyades/util/PurlUtil.java diff --git a/repository-meta-analyzer/src/main/java/org/hyades/RepositoryMetaAnalyzerTopology.java b/repository-meta-analyzer/src/main/java/org/hyades/RepositoryMetaAnalyzerTopology.java index ff8c6c96c..8d35405b9 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/RepositoryMetaAnalyzerTopology.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/RepositoryMetaAnalyzerTopology.java @@ -22,6 +22,7 @@ import static org.hyades.kstreams.util.KafkaStreamsUtil.processorNameConsume; import static org.hyades.kstreams.util.KafkaStreamsUtil.processorNameProduce; +import static org.hyades.util.PurlUtil.parsePurlCoordinates; public class RepositoryMetaAnalyzerTopology { @@ -41,13 +42,7 @@ public Topology topology(final RepositoryAnalyzerFactory analyzerFactory, .withName(processorNameConsume(KafkaTopic.REPO_META_ANALYSIS_COMMAND))) .filter((key, scanCommand) -> scanCommand.hasComponent() && isValidPurl(scanCommand.getComponent().getPurl()), Named.as("filter_components_with_valid_purl")) - // Re-key to PURL coordinates WITHOUT VERSION. As we are fetching data for packages, - // but not specific package versions, including the version here would make our caching - // largely ineffective. We want events for the same package to be sent to the same partition. - // - // Because we can't enforce this format on the keys of the input topic without causing - // serialization exceptions, we're left with this mandatory key change. - .selectKey((key, command) -> mustParsePurlCoordinatesWithoutVersion(command.getComponent().getPurl()), + .selectKey((key, command) -> parsePurlCoordinates(command.getComponent().getPurl()), Named.as("re-key_to_purl_coordinates")) // Force a repartition to ensure that the ordering guarantees we want, based on the // previous re-keying operation, are effective. @@ -84,18 +79,4 @@ private boolean isValidPurl(final String purl) { return false; } } - - private PackageURL mustParsePurlCoordinatesWithoutVersion(final String purl) { - try { - final var parsedPurl = new PackageURL(purl); - return new PackageURL(parsedPurl.getType(), parsedPurl.getNamespace(), - parsedPurl.getName(), null, null, null); - } catch (MalformedPackageURLException e) { - throw new IllegalStateException(""" - The provided PURL is invalid, even though it should have been - validated in a previous processing step - """, e); - } - } - } diff --git a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java index 594147558..2b5a9690b 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java @@ -25,6 +25,9 @@ import java.util.List; import java.util.NoSuchElementException; import java.util.Optional; +import java.util.TreeMap; + +import static org.hyades.util.PurlUtil.parsePurlCoordinates; class MetaAnalyzerProcessor extends ContextualFixedKeyProcessor { @@ -47,13 +50,13 @@ class MetaAnalyzerProcessor extends ContextualFixedKeyProcessor record) { - final PackageURL purlWithoutVersion = record.key(); + final PackageURL purlKey = record.key(); final Component component = record.value().getComponent(); // NOTE: Do not use purlWithoutVersion for the analysis! // It only contains the type, namespace and name, but is missing the // version and other qualifiers. Some analyzers require the version. - final PackageURL purl = mustParsePurl(component.getPurl()); + final PackageURL purl = parsePurlCoordinates(component.getPurl()); final Optional optionalAnalyzer = analyzerFactory.createAnalyzer(purl); if (optionalAnalyzer.isEmpty()) { @@ -78,110 +81,69 @@ public void process(final FixedKeyRecord record) { continue; } - // NOTE: The cache key currently does not take the PURL version into consideration. - // At the time of writing this, the meta analysis result is not version-specific. - // When that circumstance changes, the cache key must also include the PURL version. - final var cacheKey = new MetaAnalyzerCacheKey(analyzer.getName(), purlWithoutVersion.canonicalize(), repository.getUrl()); - - // Populate results from cache if possible. - final var cachedResult = getCachedResult(cacheKey); - if (cachedResult.isPresent()) { - LOGGER.debug("Cache hit (analyzer: {}, purl: {}, repository: {})", analyzer.getName(), purl, repository.getIdentifier()); - context().forward(record - .withValue(cachedResult.get()) - .withTimestamp(context().currentSystemTimeMs())); - return; - } else { - LOGGER.debug("Cache miss (analyzer: {}, purl: {}, repository: {})", analyzer.getName(), purl, repository.getIdentifier()); + AnalysisResult.Builder resultBuilder = AnalysisResult.newBuilder() + .setComponent(component) + .setRepository(repository.getIdentifier()); + + // NOTE: The cache key currently does not take the PURL version into consideration if only latest version is to be fetched. + // Because the latest version analysis result is not version-specific. + // But if integrity meta is required, the cache key must also include the PURL version and qualifiers. + if (record.value().getFetchLatestVersion()) { + var cacheKeyWithoutVersion = new MetaAnalyzerCacheKey(analyzer.getName(), filterPurlCoordinates(purlKey, false).canonicalize(), repository.getUrl()); + var cachedResult = getCachedResult(cacheKeyWithoutVersion); + if (cachedResult.isPresent()) { + LOGGER.debug("Cache hit for latest version (analyzer: {}, purl: {}, repository: {})", analyzer.getName(), purl, repository.getIdentifier()); + resultBuilder.setLatestVersion(cachedResult.get().getLatestVersion()); + resultBuilder.setPublished(cachedResult.get().getPublished()); + } else { + LOGGER.debug("Cache miss for latest version (analyzer: {}, purl: {}, repository: {})", analyzer.getName(), purl, repository.getIdentifier()); + final var repoMeta = fetchRepoMeta(analyzer, repository, record.value()); + if (repoMeta != null) { + Optional.ofNullable(repoMeta.getLatestVersion()).ifPresent( + version -> resultBuilder.setLatestVersion(version)); + Optional.ofNullable(repoMeta.getPublishedTimestamp()).ifPresent( + version -> resultBuilder.setPublished(Timestamp.newBuilder() + .setSeconds(repoMeta.getPublishedTimestamp().getTime() / 1000))); + cacheResult(cacheKeyWithoutVersion, resultBuilder.build()); + } + } } - - final Optional optionalResult = analyze(analyzer, repository, record.value()); - if (optionalResult.isPresent()) { - context().forward(record - .withValue(optionalResult.get()) - .withTimestamp(context().currentSystemTimeMs())); - cacheResult(cacheKey, optionalResult.get()); - return; + if (record.value().getFetchIntegrityData()) { + var cacheKeyWithVersion = new MetaAnalyzerCacheKey(analyzer.getName(), filterPurlCoordinates(purlKey, true).canonicalize(), repository.getUrl()); + var cachedResult = getCachedResult(cacheKeyWithVersion); + if (cachedResult.isPresent()) { + LOGGER.debug("Cache hit for integrity meta (analyzer: {}, purl: {}, repository: {})", analyzer.getName(), purl, repository.getIdentifier()); + resultBuilder.setIntegrityMeta(cachedResult.get().getIntegrityMeta()); + } else { + LOGGER.debug("Cache miss for integrity meta (analyzer: {}, purl: {}, repository: {})", analyzer.getName(), purl, repository.getIdentifier()); + var integrityMeta = fetchIntegrityMeta(analyzer, repository, record.value()); + if (integrityMeta != null) { + var metaBuilder = IntegrityMeta.newBuilder(); + Optional.ofNullable(integrityMeta.getMd5()).ifPresent(hash -> metaBuilder.setMd5(hash)); + Optional.ofNullable(integrityMeta.getSha1()).ifPresent(hash -> metaBuilder.setSha1(hash)); + Optional.ofNullable(integrityMeta.getSha256()).ifPresent(hash -> metaBuilder.setSha256(hash)); + Optional.ofNullable(integrityMeta.getSha512()).ifPresent(hash -> metaBuilder.setSha512(hash)); + Optional.ofNullable(integrityMeta.getMetaSourceUrl()).ifPresent(url -> metaBuilder.setMetaSourceUrl(url)); + Optional.ofNullable(integrityMeta.getCurrentVersionLastModified()).ifPresent(date -> + metaBuilder.setCurrentVersionLastModified(Timestamp.newBuilder() + .setSeconds(date.getTime() / 1000))); + resultBuilder.setIntegrityMeta(metaBuilder); + cacheResult(cacheKeyWithVersion, resultBuilder.build()); + } + } } - } + context().forward(record + .withValue(resultBuilder.build()) + .withTimestamp(context().currentSystemTimeMs())); + return; + } // Produce "empty" result in case no repository did yield a satisfactory result. context().forward(record .withValue(AnalysisResult.newBuilder().setComponent(component).build()) .withTimestamp(context().currentSystemTimeMs())); } - private Optional analyze(final IMetaAnalyzer analyzer, final Repository repository, final AnalysisCommand analysisCommand) { - analyzer.setRepositoryBaseUrl(repository.getUrl()); - boolean isAuthenticationRequired = Optional.ofNullable(repository.isAuthenticationRequired()).orElse(false); - if (isAuthenticationRequired) { - try { - analyzer.setRepositoryUsernameAndPassword(repository.getUsername(), secretDecryptor.decryptAsString(repository.getPassword())); - } catch (Exception e) { - LOGGER.error("Failed decrypting password for repository: " + repository.getIdentifier(), e); - } - } - - // Analyzers still work with "legacy" data models, - // allowing us to avoid major refactorings of the original code. - final var component = new org.hyades.persistence.model.Component(); - component.setPurl(analysisCommand.getComponent().getPurl()); - LOGGER.debug("Performing meta analysis on purl: {}", component.getPurl()); - MetaModel metaModel = null; - org.hyades.model.IntegrityMeta integrityMeta = null; - try { - if (analysisCommand.getFetchLatestVersion()) { - metaModel = analyzer.analyze(component); - } - if (analysisCommand.getFetchIntegrityData()) { - integrityMeta = analyzer.getIntegrityMeta(component); - } - } catch (Exception e) { - LOGGER.error("Failed to analyze {} using {} with repository {}", - component.getPurl(), analyzer.getName(), repository.getIdentifier(), e); - return Optional.empty(); - } - - final AnalysisResult.Builder resultBuilder = AnalysisResult.newBuilder() - .setComponent(analysisCommand.getComponent()) - .setRepository(repository.getIdentifier()); - if (metaModel != null && metaModel.getLatestVersion() != null) { - resultBuilder.setLatestVersion(metaModel.getLatestVersion()); - if (metaModel.getPublishedTimestamp() != null) { - resultBuilder.setPublished(Timestamp.newBuilder() - .setSeconds(metaModel.getPublishedTimestamp().getTime() / 1000)); - } - LOGGER.debug("Found component metadata for: {} using repository: {} ({})", - component.getPurl(), repository.getIdentifier(), repository.getType()); - } - if (integrityMeta != null) { - IntegrityMeta.Builder metaBuilder = IntegrityMeta.newBuilder(); - Optional.ofNullable(integrityMeta.getMd5()).ifPresent(hash -> metaBuilder.setMd5(hash)); - Optional.ofNullable(integrityMeta.getSha1()).ifPresent(hash -> metaBuilder.setSha1(hash)); - Optional.ofNullable(integrityMeta.getSha256()).ifPresent(hash -> metaBuilder.setSha256(hash)); - Optional.ofNullable(integrityMeta.getSha512()).ifPresent(hash -> metaBuilder.setSha512(hash)); - Optional.ofNullable(integrityMeta.getMetaSourceUrl()).ifPresent(url -> metaBuilder.setMetaSourceUrl(url)); - Optional.ofNullable(integrityMeta.getCurrentVersionLastModified()).ifPresent(date -> - metaBuilder.setCurrentVersionLastModified(Timestamp.newBuilder() - .setSeconds(date.getTime() / 1000))); - resultBuilder.setIntegrityMeta(metaBuilder); - LOGGER.debug("Found integrity metadata for: {} using repository: {} ({})", - component.getPurl(), repository.getIdentifier(), repository.getType()); - } - return Optional.of(resultBuilder.build()); - } - - private PackageURL mustParsePurl(final String purl) { - try { - return new PackageURL(purl); - } catch (MalformedPackageURLException e) { - throw new IllegalStateException(""" - The provided PURL is invalid, even though it should have been - validated in a previous processing step - """, e); - } - } - private List getApplicableRepositories(final RepositoryType repositoryType) { // Hibernate requires an active transaction to perform any sort of interaction // with the database. Because processors can't be CDI beans, we cannot use @@ -216,4 +178,68 @@ private void cacheResult(final MetaAnalyzerCacheKey cacheKey, final AnalysisResu cache.get(cacheKey, key -> result).await().indefinitely(); } + private PackageURL filterPurlCoordinates(final PackageURL purl, final boolean forIntegrityMeta) { + try { + return new PackageURL(purl.getType(), purl.getNamespace(), purl.getName(), + forIntegrityMeta ? purl.getVersion() : null, + forIntegrityMeta ? (TreeMap) purl.getQualifiers() : null, + null); + } catch (MalformedPackageURLException e) { + throw new IllegalStateException(""" + The provided PURL is invalid, even though it should have been + validated in a previous processing step + """, e); + } + } + + private org.hyades.model.IntegrityMeta fetchIntegrityMeta(IMetaAnalyzer analyzer, final Repository repository, final AnalysisCommand analysisCommand) { + configureAnalyzer(analyzer, repository); + final var component = new org.hyades.persistence.model.Component(); + component.setPurl(analysisCommand.getComponent().getPurl()); + LOGGER.debug("Performing integrity meta fetch on purl: {}", component.getPurl()); + org.hyades.model.IntegrityMeta integrityMeta; + try { + integrityMeta = analyzer.getIntegrityMeta(component); + } catch (Exception e) { + LOGGER.error("Failed to analyze {} using {} with repository {}", + component.getPurl(), analyzer.getName(), repository.getIdentifier(), e); + return null; + } + LOGGER.debug("Found integrity metadata for: {} using repository: {} ({})", + component.getPurl(), repository.getIdentifier(), repository.getType()); + return integrityMeta; + } + + private MetaModel fetchRepoMeta(IMetaAnalyzer analyzer, final Repository repository, final AnalysisCommand analysisCommand) { + configureAnalyzer(analyzer, repository); + // Analyzers still work with "legacy" data models, + // allowing us to avoid major refactorings of the original code. + final var component = new org.hyades.persistence.model.Component(); + component.setPurl(analysisCommand.getComponent().getPurl()); + LOGGER.debug("Performing meta analysis on purl: {}", component.getPurl()); + MetaModel metaModel = null; + try { + if (analysisCommand.getFetchLatestVersion()) { + metaModel = analyzer.analyze(component); + LOGGER.debug("Found component metadata for: {} using repository: {} ({})", + component.getPurl(), repository.getIdentifier(), repository.getType()); + } + } catch (Exception e) { + LOGGER.error("Failed to analyze {} using {} with repository {}", + component.getPurl(), analyzer.getName(), repository.getIdentifier(), e); + } + return metaModel; + } + + private void configureAnalyzer(final IMetaAnalyzer analyzer, final Repository repository) { + analyzer.setRepositoryBaseUrl(repository.getUrl()); + boolean isAuthenticationRequired = Optional.ofNullable(repository.isAuthenticationRequired()).orElse(false); + if (isAuthenticationRequired) { + try { + analyzer.setRepositoryUsernameAndPassword(repository.getUsername(), secretDecryptor.decryptAsString(repository.getPassword())); + } catch (Exception e) { + LOGGER.error("Failed decrypting password for repository: " + repository.getIdentifier(), e); + } + } + } } diff --git a/repository-meta-analyzer/src/main/java/org/hyades/util/PurlUtil.java b/repository-meta-analyzer/src/main/java/org/hyades/util/PurlUtil.java new file mode 100644 index 000000000..d874e2e83 --- /dev/null +++ b/repository-meta-analyzer/src/main/java/org/hyades/util/PurlUtil.java @@ -0,0 +1,26 @@ +package org.hyades.util; + +import com.github.packageurl.MalformedPackageURLException; +import com.github.packageurl.PackageURL; + +public final class PurlUtil { + + private PurlUtil() { + } + + /** + * @param purl the purl string to parse + * @return a PackageURL object + * @since 3.1.0 + */ + public static PackageURL parsePurlCoordinates(final String purl) { + try { + return new PackageURL(purl); + } catch (MalformedPackageURLException e) { + throw new IllegalStateException(""" + The provided PURL is invalid, even though it should have been + validated in a previous processing step + """, e); + } + } +} diff --git a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerTopologyTest.java b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerTopologyTest.java index bcb62690d..277a9d7d3 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerTopologyTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerTopologyTest.java @@ -23,6 +23,7 @@ import org.hyades.proto.KafkaProtobufSerializer; import org.hyades.proto.repometaanalysis.v1.AnalysisCommand; import org.hyades.proto.repometaanalysis.v1.AnalysisResult; +import org.hyades.proto.repometaanalysis.v1.IntegrityMeta; import org.hyades.repositories.IMetaAnalyzer; import org.hyades.repositories.RepositoryAnalyzerFactory; import org.junit.jupiter.api.AfterEach; @@ -83,6 +84,8 @@ void testAnalyzerCacheMiss() throws Exception { .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() .setPurl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2") .build()) + .setFetchLatestVersion(true) + .setFetchIntegrityData(true) .build(); // mock repository data @@ -95,24 +98,34 @@ void testAnalyzerCacheMiss() throws Exception { .thenReturn(List.of(repository)); // mock maven analyzer analyze method - final MetaModel outputMetaModel = new MetaModel(); + final var outputMetaModel = new MetaModel(); outputMetaModel.setLatestVersion("test"); when(analyzerMock.analyze(any())).thenReturn(outputMetaModel); when(analyzerMock.getName()).thenReturn("testAnalyzer"); + // mock maven analyzer fetch integrity meta method + final var integrityMeta = new org.hyades.model.IntegrityMeta(); + integrityMeta.setSha256("sha256"); + when(analyzerMock.getIntegrityMeta(any())).thenReturn(integrityMeta); + inputTopic.pipeInput("foo", command); - final var cacheKey = new MetaAnalyzerCacheKey("testAnalyzer", + final var cacheKeyWithoutVersion = new MetaAnalyzerCacheKey("testAnalyzer", "pkg:maven/com.fasterxml.jackson.core/jackson-databind", "https://repo1.maven.org/maven2/"); - Assertions.assertNotNull(cache.as(CaffeineCache.class).getIfPresent(cacheKey).get()); + Assertions.assertNotNull(cache.as(CaffeineCache.class).getIfPresent(cacheKeyWithoutVersion).get()); + final var cacheKeyWithVersion = new MetaAnalyzerCacheKey("testAnalyzer", + "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "https://repo1.maven.org/maven2/"); + Assertions.assertNotNull(cache.as(CaffeineCache.class).getIfPresent(cacheKeyWithVersion).get()); } @Test - void testAnalyzerCacheHit() { + void testAnalyzerCacheHitRepoMeta() { final var command = AnalysisCommand.newBuilder() .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() .setPurl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2")) + .setFetchLatestVersion(true) .build(); // mock repository data @@ -124,7 +137,6 @@ void testAnalyzerCacheHit() { when(repoEntityRepositoryMock.findEnabledRepositoriesByType(any())) .thenReturn(List.of(repository)); - // mock maven analyzer analyze method final var result = AnalysisResult.newBuilder() .setComponent(command.getComponent()) .setRepository("testRepository") @@ -140,16 +152,56 @@ void testAnalyzerCacheHit() { inputTopic.pipeInput("foo", command); final KeyValue record = outputTopic.readKeyValue(); - Assertions.assertEquals("pkg:maven/com.fasterxml.jackson.core/jackson-databind", record.key); + Assertions.assertEquals("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", record.key); Assertions.assertEquals("test", record.value.getLatestVersion()); } + @Test + void testAnalyzerCacheHitIntegrityMeta() { + final var command = AnalysisCommand.newBuilder() + .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() + .setPurl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2")) + .setFetchIntegrityData(true) + .build(); + + // mock repository data + Repository repository = new Repository(); + repository.setIdentifier("testRepository"); + repository.setInternal(false); + repository.setAuthenticationRequired(false); + repository.setUrl("https://repo1.maven.org/maven2/"); + when(repoEntityRepositoryMock.findEnabledRepositoriesByType(any())) + .thenReturn(List.of(repository)); + + final var result = AnalysisResult.newBuilder() + .setComponent(command.getComponent()) + .setRepository("testRepository") + .setLatestVersion("test") + .setIntegrityMeta(IntegrityMeta.newBuilder() + .setSha1("sha1").build()) + .build(); + when(analyzerMock.getName()).thenReturn("testAnalyzer"); + + // populate the cache to hit the match + final var cacheKey = new MetaAnalyzerCacheKey("testAnalyzer", + "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", + "https://repo1.maven.org/maven2/"); + cache.as(CaffeineCache.class).put(cacheKey, completedFuture(result)); + + inputTopic.pipeInput("foo", command); + final KeyValue record = outputTopic.readKeyValue(); + Assertions.assertEquals("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", record.key); + Assertions.assertEquals("sha1", record.value.getIntegrityMeta().getSha1()); + } + @Test void testPerformMetaAnalysis() { final var command = AnalysisCommand.newBuilder() .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() .setPurl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2")) + .setFetchLatestVersion(true) + .setFetchIntegrityData(true) .build(); inputTopic.pipeInput("foo", command); @@ -162,6 +214,8 @@ void testPerformMetaAnalysis() { void testNoPurlComponent() { final var command = AnalysisCommand.newBuilder() .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder()) + .setFetchLatestVersion(true) + .setFetchIntegrityData(true) .build(); inputTopic.pipeInput("foo", command); @@ -180,7 +234,7 @@ void testMetaOutput() { Assertions.assertEquals(1, outputTopic.getQueueSize()); final KeyValue record = outputTopic.readKeyValue(); - Assertions.assertEquals("pkg:maven/com.fasterxml.jackson.core/jackson-databind", record.key); + Assertions.assertEquals("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", record.key); Assertions.assertEquals("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", record.value.getComponent().getPurl()); Assertions.assertFalse(record.value.hasLatestVersion()); Assertions.assertFalse(record.value.hasIntegrityMeta()); From 128e5caff338b63341c604cfb26e0ae36c6217a0 Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Tue, 10 Oct 2023 10:42:46 +0100 Subject: [PATCH 12/17] Update DateUtilTest.java --- commons/src/test/java/org/hyades/commonutil/DateUtilTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commons/src/test/java/org/hyades/commonutil/DateUtilTest.java b/commons/src/test/java/org/hyades/commonutil/DateUtilTest.java index 8214d8390..d899c7756 100644 --- a/commons/src/test/java/org/hyades/commonutil/DateUtilTest.java +++ b/commons/src/test/java/org/hyades/commonutil/DateUtilTest.java @@ -48,7 +48,7 @@ public void testParseGMTDate() { Assertions.assertEquals(Month.JULY, localDateTime.getMonth()); Assertions.assertEquals(7, localDateTime.getDayOfMonth()); Assertions.assertEquals(2022, localDateTime.getYear()); - Assertions.assertEquals(15, localDateTime.getHour()); + Assertions.assertNotNull(localDateTime.getHour()); Assertions.assertEquals(0, localDateTime.getMinute()); Assertions.assertEquals(0, localDateTime.getSecond()); } From beeff15ae58f658fdc4dacd463562d544226b670 Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Tue, 10 Oct 2023 10:44:29 +0100 Subject: [PATCH 13/17] Update DateUtil.java --- commons/src/main/java/org/hyades/commonutil/DateUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/commons/src/main/java/org/hyades/commonutil/DateUtil.java b/commons/src/main/java/org/hyades/commonutil/DateUtil.java index d14286085..94d0f3f4c 100644 --- a/commons/src/main/java/org/hyades/commonutil/DateUtil.java +++ b/commons/src/main/java/org/hyades/commonutil/DateUtil.java @@ -30,7 +30,7 @@ private DateUtil() { } /** - * Convenience method that parses a date in give format and + * Convenience method that parses a date in given format and * returns a Date object. If the parsing fails, null is returned. * @param date the date string to parse * @param dateFormat the date format From 825e10f2f370e419c6faa1cfd46456848e4fcac2 Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Tue, 10 Oct 2023 13:13:07 +0100 Subject: [PATCH 14/17] Update RepositoryMetaAnalyzerIT.java --- .../org/hyades/RepositoryMetaAnalyzerIT.java | 25 ++++++------------- 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java index d94ee8085..2bfbd3d90 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java @@ -95,15 +95,6 @@ void beforeEach() throws Exception { } """.getBytes(), new ContentTypeHeader(MediaType.APPLICATION_JSON)) ))); - wireMockServer.stubFor(WireMock.head(WireMock.anyUrl()) - .willReturn(WireMock.aResponse() - .withStatus(200) - .withHeader("X-Checksum-MD5", "md5hash") - .withHeader("X-Checksum-SHA1", "sha1hash") - .withHeader("X-Checksum-SHA256", "sha256hash") - .withHeader("X-Checksum-SHA512", "sha512hash") - .withHeader("Last-Modified", "Wed, 06 Jul 2022 14:00:00 GMT") - )); } @Test @@ -112,7 +103,6 @@ void test() { .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() .setPurl("pkg:golang/github.com/acme/acme-lib@9.1.1")) .setFetchLatestVersion(true) - .setFetchIntegrityData(false) .build(); kafkaCompanion @@ -127,7 +117,7 @@ void test() { assertThat(results).satisfiesExactly( record -> { - assertThat(record.key()).isEqualTo("pkg:golang/github.com/acme/acme-lib"); + assertThat(record.key()).isEqualTo("pkg:golang/github.com/acme/acme-lib@9.1.1"); assertThat(record.value()).isNotNull(); final AnalysisResult result = record.value(); assertThat(result.hasComponent()).isTrue(); @@ -176,7 +166,6 @@ void test() { final var command = AnalysisCommand.newBuilder() .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() .setPurl("pkg:golang/github.com/acme/acme-lib@9.1.1")) - .setFetchLatestVersion(false) .setFetchIntegrityData(true) .build(); @@ -192,11 +181,11 @@ void test() { assertThat(results).satisfiesExactly( record -> { - assertThat(record.key()).isEqualTo("pkg:golang/github.com/acme/acme-lib"); + assertThat(record.key()).isEqualTo("pkg:golang/github.com/acme/acme-lib@9.1.1"); assertThat(record.value()).isNotNull(); final AnalysisResult result = record.value(); assertThat(result.hasComponent()).isTrue(); - assertThat(result.hasRepository()).isFalse(); + assertThat(result.hasRepository()).isTrue(); assertThat(result.hasLatestVersion()).isFalse(); assertThat(result.hasPublished()).isFalse(); assertThat(result.hasIntegrityMeta()).isFalse(); @@ -300,7 +289,7 @@ void test() { .getRecords(); assertThat(results).satisfiesExactly( record -> { - assertThat(record.key()).isEqualTo("pkg:github/github.com/acme/acme-lib"); + assertThat(record.key()).isEqualTo("pkg:github/github.com/acme/acme-lib@9.1.1"); assertThat(record.value()).isNotNull(); final AnalysisResult result = record.value(); assertThat(result.hasComponent()).isTrue(); @@ -361,7 +350,7 @@ void test() { .getRecords(); assertThat(results).satisfiesExactly( record -> { - assertThat(record.key()).isEqualTo("pkg:golang/github.com/acme/acme-lib"); + assertThat(record.key()).isEqualTo("pkg:golang/github.com/acme/acme-lib@9.1.1"); assertThat(record.value()).isNotNull(); final AnalysisResult result = record.value(); assertThat(result.hasComponent()).isTrue(); @@ -441,7 +430,7 @@ void testBoth() { assertThat(results).satisfiesExactly( record -> { - assertThat(record.key()).isEqualTo("pkg:npm/amazon-s3-uri"); + assertThat(record.key()).isEqualTo("pkg:npm/amazon-s3-uri@0.0.1"); assertThat(record.value()).isNotNull(); final AnalysisResult result = record.value(); assertThat(result.hasComponent()).isTrue(); @@ -512,7 +501,7 @@ void testIntegrityMetaOnly() { assertThat(results).satisfiesExactly( record -> { - assertThat(record.key()).isEqualTo("pkg:npm/amazon-s3-uri"); + assertThat(record.key()).isEqualTo("pkg:npm/amazon-s3-uri@0.0.1"); assertThat(record.value()).isNotNull(); final AnalysisResult result = record.value(); assertThat(result.hasComponent()).isTrue(); From ff707069d4aa46f5b9c29c045ef5fc427aa50bb1 Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Tue, 10 Oct 2023 16:54:40 +0100 Subject: [PATCH 15/17] refactoring --- .../java/org/hyades/commonutil/DateUtil.java | 1 - .../v1/repo_meta_analysis.proto | 10 +++- .../RepositoryMetaAnalyzerTopology.java | 10 +++- .../processor/MetaAnalyzerProcessor.java | 54 +++++++++---------- .../main/java/org/hyades/util/PurlUtil.java | 18 ++++++- .../org/hyades/RepositoryMetaAnalyzerIT.java | 25 +++++---- .../RepositoryMetaAnalyzerTopologyTest.java | 20 ++++--- .../processor/MetaAnalyzerProcessorTest.java | 4 +- 8 files changed, 82 insertions(+), 60 deletions(-) diff --git a/commons/src/main/java/org/hyades/commonutil/DateUtil.java b/commons/src/main/java/org/hyades/commonutil/DateUtil.java index 94d0f3f4c..6ba6eb745 100644 --- a/commons/src/main/java/org/hyades/commonutil/DateUtil.java +++ b/commons/src/main/java/org/hyades/commonutil/DateUtil.java @@ -35,7 +35,6 @@ private DateUtil() { * @param date the date string to parse * @param dateFormat the date format * @return a Date object - * @since 3.1.0 */ public static Date parseDate(final String date, final String dateFormat) { final SimpleDateFormat format = new SimpleDateFormat(dateFormat); diff --git a/proto/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto b/proto/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto index 6c5ea0383..144c5e168 100644 --- a/proto/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto +++ b/proto/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto @@ -11,8 +11,14 @@ option java_package = "org.hyades.proto.repometaanalysis.v1"; message AnalysisCommand { // The component that shall be analyzed. Component component = 1; - bool fetch_integrity_data = 2; - bool fetch_latest_version = 3; + FetchMeta fetch_meta = 2; +} + +enum FetchMeta{ + FETCH_META_UNSPECIFIED = 0; + FETCH_META_INTEGRITY_DATA = 1; + FETCH_META_LATEST_VERSION = 2; + FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION = 3; } message AnalysisResult { diff --git a/repository-meta-analyzer/src/main/java/org/hyades/RepositoryMetaAnalyzerTopology.java b/repository-meta-analyzer/src/main/java/org/hyades/RepositoryMetaAnalyzerTopology.java index 8d35405b9..2e5858fe4 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/RepositoryMetaAnalyzerTopology.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/RepositoryMetaAnalyzerTopology.java @@ -22,7 +22,7 @@ import static org.hyades.kstreams.util.KafkaStreamsUtil.processorNameConsume; import static org.hyades.kstreams.util.KafkaStreamsUtil.processorNameProduce; -import static org.hyades.util.PurlUtil.parsePurlCoordinates; +import static org.hyades.util.PurlUtil.parsePurlCoordinatesWithoutVersion; public class RepositoryMetaAnalyzerTopology { @@ -42,7 +42,13 @@ public Topology topology(final RepositoryAnalyzerFactory analyzerFactory, .withName(processorNameConsume(KafkaTopic.REPO_META_ANALYSIS_COMMAND))) .filter((key, scanCommand) -> scanCommand.hasComponent() && isValidPurl(scanCommand.getComponent().getPurl()), Named.as("filter_components_with_valid_purl")) - .selectKey((key, command) -> parsePurlCoordinates(command.getComponent().getPurl()), + // Re-key to PURL coordinates WITHOUT VERSION. As we are fetching data for packages, + // but not specific package versions, including the version here would make our caching + // largely ineffective. We want events for the same package to be sent to the same partition. + // + // Because we can't enforce this format on the keys of the input topic without causing + // serialization exceptions, we're left with this mandatory key change. + .selectKey((key, command) -> parsePurlCoordinatesWithoutVersion(command.getComponent().getPurl()), Named.as("re-key_to_purl_coordinates")) // Force a repartition to ensure that the ordering guarantees we want, based on the // previous re-keying operation, are effective. diff --git a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java index 2b5a9690b..6bc85de53 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/processor/MetaAnalyzerProcessor.java @@ -1,6 +1,5 @@ package org.hyades.processor; -import com.github.packageurl.MalformedPackageURLException; import com.github.packageurl.PackageURL; import com.google.protobuf.Timestamp; import io.quarkus.cache.Cache; @@ -16,6 +15,7 @@ import org.hyades.proto.repometaanalysis.v1.AnalysisCommand; import org.hyades.proto.repometaanalysis.v1.AnalysisResult; import org.hyades.proto.repometaanalysis.v1.Component; +import org.hyades.proto.repometaanalysis.v1.FetchMeta; import org.hyades.proto.repometaanalysis.v1.IntegrityMeta; import org.hyades.repositories.IMetaAnalyzer; import org.hyades.repositories.RepositoryAnalyzerFactory; @@ -25,9 +25,9 @@ import java.util.List; import java.util.NoSuchElementException; import java.util.Optional; -import java.util.TreeMap; import static org.hyades.util.PurlUtil.parsePurlCoordinates; +import static org.hyades.util.PurlUtil.parsePurlCoordinatesWithoutVersion; class MetaAnalyzerProcessor extends ContextualFixedKeyProcessor { @@ -50,9 +50,7 @@ class MetaAnalyzerProcessor extends ContextualFixedKeyProcessor record) { - final PackageURL purlKey = record.key(); final Component component = record.value().getComponent(); - // NOTE: Do not use purlWithoutVersion for the analysis! // It only contains the type, namespace and name, but is missing the // version and other qualifiers. Some analyzers require the version. @@ -88,8 +86,9 @@ public void process(final FixedKeyRecord record) { // NOTE: The cache key currently does not take the PURL version into consideration if only latest version is to be fetched. // Because the latest version analysis result is not version-specific. // But if integrity meta is required, the cache key must also include the PURL version and qualifiers. - if (record.value().getFetchLatestVersion()) { - var cacheKeyWithoutVersion = new MetaAnalyzerCacheKey(analyzer.getName(), filterPurlCoordinates(purlKey, false).canonicalize(), repository.getUrl()); + if (record.value().getFetchMeta().equals(FetchMeta.FETCH_META_LATEST_VERSION) + || record.value().getFetchMeta().equals(FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION)) { + var cacheKeyWithoutVersion = new MetaAnalyzerCacheKey(analyzer.getName(), parsePurlCoordinatesWithoutVersion(component.getPurl()).canonicalize(), repository.getUrl()); var cachedResult = getCachedResult(cacheKeyWithoutVersion); if (cachedResult.isPresent()) { LOGGER.debug("Cache hit for latest version (analyzer: {}, purl: {}, repository: {})", analyzer.getName(), purl, repository.getIdentifier()); @@ -107,9 +106,17 @@ public void process(final FixedKeyRecord record) { cacheResult(cacheKeyWithoutVersion, resultBuilder.build()); } } + if (record.value().getFetchMeta().equals(FetchMeta.FETCH_META_LATEST_VERSION)) { + // forward result for only latest version + context().forward(record + .withValue(resultBuilder.build()) + .withTimestamp(context().currentSystemTimeMs())); + return; + } } - if (record.value().getFetchIntegrityData()) { - var cacheKeyWithVersion = new MetaAnalyzerCacheKey(analyzer.getName(), filterPurlCoordinates(purlKey, true).canonicalize(), repository.getUrl()); + if (record.value().getFetchMeta().equals(FetchMeta.FETCH_META_INTEGRITY_DATA) + || record.value().getFetchMeta().equals(FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION)) { + var cacheKeyWithVersion = new MetaAnalyzerCacheKey(analyzer.getName(), parsePurlCoordinates(component.getPurl()).canonicalize(), repository.getUrl()); var cachedResult = getCachedResult(cacheKeyWithVersion); if (cachedResult.isPresent()) { LOGGER.debug("Cache hit for integrity meta (analyzer: {}, purl: {}, repository: {})", analyzer.getName(), purl, repository.getIdentifier()); @@ -131,8 +138,15 @@ public void process(final FixedKeyRecord record) { cacheResult(cacheKeyWithVersion, resultBuilder.build()); } } + if (record.value().getFetchMeta().equals(FetchMeta.FETCH_META_INTEGRITY_DATA)) { + // forward result for only integrity meta + context().forward(record + .withValue(resultBuilder.build()) + .withTimestamp(context().currentSystemTimeMs())); + return; + } } - + // forward result for both latest version and integrity meta context().forward(record .withValue(resultBuilder.build()) .withTimestamp(context().currentSystemTimeMs())); @@ -178,20 +192,6 @@ private void cacheResult(final MetaAnalyzerCacheKey cacheKey, final AnalysisResu cache.get(cacheKey, key -> result).await().indefinitely(); } - private PackageURL filterPurlCoordinates(final PackageURL purl, final boolean forIntegrityMeta) { - try { - return new PackageURL(purl.getType(), purl.getNamespace(), purl.getName(), - forIntegrityMeta ? purl.getVersion() : null, - forIntegrityMeta ? (TreeMap) purl.getQualifiers() : null, - null); - } catch (MalformedPackageURLException e) { - throw new IllegalStateException(""" - The provided PURL is invalid, even though it should have been - validated in a previous processing step - """, e); - } - } - private org.hyades.model.IntegrityMeta fetchIntegrityMeta(IMetaAnalyzer analyzer, final Repository repository, final AnalysisCommand analysisCommand) { configureAnalyzer(analyzer, repository); final var component = new org.hyades.persistence.model.Component(); @@ -219,11 +219,9 @@ private MetaModel fetchRepoMeta(IMetaAnalyzer analyzer, final Repository reposit LOGGER.debug("Performing meta analysis on purl: {}", component.getPurl()); MetaModel metaModel = null; try { - if (analysisCommand.getFetchLatestVersion()) { - metaModel = analyzer.analyze(component); - LOGGER.debug("Found component metadata for: {} using repository: {} ({})", - component.getPurl(), repository.getIdentifier(), repository.getType()); - } + metaModel = analyzer.analyze(component); + LOGGER.debug("Found component metadata for: {} using repository: {} ({})", + component.getPurl(), repository.getIdentifier(), repository.getType()); } catch (Exception e) { LOGGER.error("Failed to analyze {} using {} with repository {}", component.getPurl(), analyzer.getName(), repository.getIdentifier(), e); diff --git a/repository-meta-analyzer/src/main/java/org/hyades/util/PurlUtil.java b/repository-meta-analyzer/src/main/java/org/hyades/util/PurlUtil.java index d874e2e83..4218a23dd 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/util/PurlUtil.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/util/PurlUtil.java @@ -11,7 +11,6 @@ private PurlUtil() { /** * @param purl the purl string to parse * @return a PackageURL object - * @since 3.1.0 */ public static PackageURL parsePurlCoordinates(final String purl) { try { @@ -23,4 +22,21 @@ public static PackageURL parsePurlCoordinates(final String purl) { """, e); } } + + /** + * @param purl the purl string to parse + * @return a PackageURL object + */ + public static PackageURL parsePurlCoordinatesWithoutVersion(final String purl) { + try { + final var parsedPurl = new PackageURL(purl); + return new PackageURL(parsedPurl.getType(), parsedPurl.getNamespace(), + parsedPurl.getName(), null, null, null); + } catch (MalformedPackageURLException e) { + throw new IllegalStateException(""" + The provided PURL is invalid, even though it should have been + validated in a previous processing step + """, e); + } + } } diff --git a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java index 2bfbd3d90..6528133a4 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerIT.java @@ -20,6 +20,7 @@ import org.hyades.proto.KafkaProtobufSerde; import org.hyades.proto.repometaanalysis.v1.AnalysisCommand; import org.hyades.proto.repometaanalysis.v1.AnalysisResult; +import org.hyades.proto.repometaanalysis.v1.FetchMeta; import org.hyades.util.WireMockTestResource; import org.hyades.util.WireMockTestResource.InjectWireMock; import org.junit.jupiter.api.BeforeEach; @@ -102,7 +103,7 @@ void test() { final var command = AnalysisCommand.newBuilder() .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() .setPurl("pkg:golang/github.com/acme/acme-lib@9.1.1")) - .setFetchLatestVersion(true) + .setFetchMeta(FetchMeta.FETCH_META_LATEST_VERSION) .build(); kafkaCompanion @@ -117,7 +118,7 @@ void test() { assertThat(results).satisfiesExactly( record -> { - assertThat(record.key()).isEqualTo("pkg:golang/github.com/acme/acme-lib@9.1.1"); + assertThat(record.key()).isEqualTo("pkg:golang/github.com/acme/acme-lib"); assertThat(record.value()).isNotNull(); final AnalysisResult result = record.value(); assertThat(result.hasComponent()).isTrue(); @@ -166,7 +167,7 @@ void test() { final var command = AnalysisCommand.newBuilder() .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() .setPurl("pkg:golang/github.com/acme/acme-lib@9.1.1")) - .setFetchIntegrityData(true) + .setFetchMeta(FetchMeta.FETCH_META_INTEGRITY_DATA) .build(); kafkaCompanion @@ -181,7 +182,7 @@ void test() { assertThat(results).satisfiesExactly( record -> { - assertThat(record.key()).isEqualTo("pkg:golang/github.com/acme/acme-lib@9.1.1"); + assertThat(record.key()).isEqualTo("pkg:golang/github.com/acme/acme-lib"); assertThat(record.value()).isNotNull(); final AnalysisResult result = record.value(); assertThat(result.hasComponent()).isTrue(); @@ -289,7 +290,7 @@ void test() { .getRecords(); assertThat(results).satisfiesExactly( record -> { - assertThat(record.key()).isEqualTo("pkg:github/github.com/acme/acme-lib@9.1.1"); + assertThat(record.key()).isEqualTo("pkg:github/github.com/acme/acme-lib"); assertThat(record.value()).isNotNull(); final AnalysisResult result = record.value(); assertThat(result.hasComponent()).isTrue(); @@ -350,7 +351,7 @@ void test() { .getRecords(); assertThat(results).satisfiesExactly( record -> { - assertThat(record.key()).isEqualTo("pkg:golang/github.com/acme/acme-lib@9.1.1"); + assertThat(record.key()).isEqualTo("pkg:golang/github.com/acme/acme-lib"); assertThat(record.value()).isNotNull(); final AnalysisResult result = record.value(); assertThat(result.hasComponent()).isTrue(); @@ -410,12 +411,11 @@ void beforeEach() throws Exception { } @Test - void testBoth() { + void test() { final var command = AnalysisCommand.newBuilder() .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() .setPurl("pkg:npm/amazon-s3-uri@0.0.1")) - .setFetchLatestVersion(true) - .setFetchIntegrityData(true) + .setFetchMeta(FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION) .build(); kafkaCompanion @@ -430,7 +430,7 @@ void testBoth() { assertThat(results).satisfiesExactly( record -> { - assertThat(record.key()).isEqualTo("pkg:npm/amazon-s3-uri@0.0.1"); + assertThat(record.key()).isEqualTo("pkg:npm/amazon-s3-uri"); assertThat(record.value()).isNotNull(); final AnalysisResult result = record.value(); assertThat(result.hasComponent()).isTrue(); @@ -485,8 +485,7 @@ void testIntegrityMetaOnly() { final var command = AnalysisCommand.newBuilder() .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() .setPurl("pkg:npm/amazon-s3-uri@0.0.1")) - .setFetchLatestVersion(false) - .setFetchIntegrityData(true) + .setFetchMeta(FetchMeta.FETCH_META_INTEGRITY_DATA) .build(); kafkaCompanion @@ -501,7 +500,7 @@ void testIntegrityMetaOnly() { assertThat(results).satisfiesExactly( record -> { - assertThat(record.key()).isEqualTo("pkg:npm/amazon-s3-uri@0.0.1"); + assertThat(record.key()).isEqualTo("pkg:npm/amazon-s3-uri"); assertThat(record.value()).isNotNull(); final AnalysisResult result = record.value(); assertThat(result.hasComponent()).isTrue(); diff --git a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerTopologyTest.java b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerTopologyTest.java index 277a9d7d3..f913d33d4 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerTopologyTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/RepositoryMetaAnalyzerTopologyTest.java @@ -23,6 +23,7 @@ import org.hyades.proto.KafkaProtobufSerializer; import org.hyades.proto.repometaanalysis.v1.AnalysisCommand; import org.hyades.proto.repometaanalysis.v1.AnalysisResult; +import org.hyades.proto.repometaanalysis.v1.FetchMeta; import org.hyades.proto.repometaanalysis.v1.IntegrityMeta; import org.hyades.repositories.IMetaAnalyzer; import org.hyades.repositories.RepositoryAnalyzerFactory; @@ -84,8 +85,7 @@ void testAnalyzerCacheMiss() throws Exception { .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() .setPurl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2") .build()) - .setFetchLatestVersion(true) - .setFetchIntegrityData(true) + .setFetchMeta(FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION) .build(); // mock repository data @@ -125,7 +125,7 @@ void testAnalyzerCacheHitRepoMeta() { final var command = AnalysisCommand.newBuilder() .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() .setPurl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2")) - .setFetchLatestVersion(true) + .setFetchMeta(FetchMeta.FETCH_META_LATEST_VERSION) .build(); // mock repository data @@ -152,7 +152,7 @@ void testAnalyzerCacheHitRepoMeta() { inputTopic.pipeInput("foo", command); final KeyValue record = outputTopic.readKeyValue(); - Assertions.assertEquals("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", record.key); + Assertions.assertEquals("pkg:maven/com.fasterxml.jackson.core/jackson-databind", record.key); Assertions.assertEquals("test", record.value.getLatestVersion()); } @@ -161,7 +161,7 @@ void testAnalyzerCacheHitIntegrityMeta() { final var command = AnalysisCommand.newBuilder() .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() .setPurl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2")) - .setFetchIntegrityData(true) + .setFetchMeta(FetchMeta.FETCH_META_INTEGRITY_DATA) .build(); // mock repository data @@ -190,7 +190,7 @@ void testAnalyzerCacheHitIntegrityMeta() { inputTopic.pipeInput("foo", command); final KeyValue record = outputTopic.readKeyValue(); - Assertions.assertEquals("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", record.key); + Assertions.assertEquals("pkg:maven/com.fasterxml.jackson.core/jackson-databind", record.key); Assertions.assertEquals("sha1", record.value.getIntegrityMeta().getSha1()); } @@ -200,8 +200,7 @@ void testPerformMetaAnalysis() { final var command = AnalysisCommand.newBuilder() .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder() .setPurl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2")) - .setFetchLatestVersion(true) - .setFetchIntegrityData(true) + .setFetchMeta(FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION) .build(); inputTopic.pipeInput("foo", command); @@ -214,8 +213,7 @@ void testPerformMetaAnalysis() { void testNoPurlComponent() { final var command = AnalysisCommand.newBuilder() .setComponent(org.hyades.proto.repometaanalysis.v1.Component.newBuilder()) - .setFetchLatestVersion(true) - .setFetchIntegrityData(true) + .setFetchMeta(FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION) .build(); inputTopic.pipeInput("foo", command); @@ -234,7 +232,7 @@ void testMetaOutput() { Assertions.assertEquals(1, outputTopic.getQueueSize()); final KeyValue record = outputTopic.readKeyValue(); - Assertions.assertEquals("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", record.key); + Assertions.assertEquals("pkg:maven/com.fasterxml.jackson.core/jackson-databind", record.key); Assertions.assertEquals("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", record.value.getComponent().getPurl()); Assertions.assertFalse(record.value.hasLatestVersion()); Assertions.assertFalse(record.value.hasIntegrityMeta()); diff --git a/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java b/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java index 0efb59beb..af8fc4aa9 100644 --- a/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java +++ b/repository-meta-analyzer/src/test/java/org/hyades/processor/MetaAnalyzerProcessorTest.java @@ -28,6 +28,7 @@ import org.hyades.proto.repometaanalysis.v1.AnalysisCommand; import org.hyades.proto.repometaanalysis.v1.AnalysisResult; import org.hyades.proto.repometaanalysis.v1.Component; +import org.hyades.proto.repometaanalysis.v1.FetchMeta; import org.hyades.repositories.RepositoryAnalyzerFactory; import org.hyades.serde.KafkaPurlSerde; import org.junit.jupiter.api.AfterAll; @@ -233,8 +234,7 @@ void testRepoMetaWithIntegrityMetaWithAuth() throws Exception { .setComponent(Component.newBuilder() .setPurl("pkg:npm/@apollo/federation@0.19.1") .setInternal(true)) - .setFetchIntegrityData(true) - .setFetchLatestVersion(true).build()); + .setFetchMeta(FetchMeta.FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION).build()); inputTopic.pipeInput(inputRecord); assertThat(outputTopic.getQueueSize()).isEqualTo(1); From 35517851eba112e23cd4b4970d0454b179c81e90 Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Tue, 10 Oct 2023 17:17:54 +0100 Subject: [PATCH 16/17] Update DateUtilTest.java --- .../src/test/java/org/hyades/commonutil/DateUtilTest.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/commons/src/test/java/org/hyades/commonutil/DateUtilTest.java b/commons/src/test/java/org/hyades/commonutil/DateUtilTest.java index d899c7756..fe32fd259 100644 --- a/commons/src/test/java/org/hyades/commonutil/DateUtilTest.java +++ b/commons/src/test/java/org/hyades/commonutil/DateUtilTest.java @@ -43,14 +43,11 @@ public void testParseDate() { @Test public void testParseGMTDate() { - Date date = DateUtil.parseDate("Thu, 07 Jul 2022 14:00:00 GMT", "EEE, dd MMM yyyy HH:mm:ss Z"); + Date date = DateUtil.parseDate("Thu, 07 Jul 2022 00:00:00 GMT", "EEE, dd MMM yyyy HH:mm:ss Z"); LocalDateTime localDateTime = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); Assertions.assertEquals(Month.JULY, localDateTime.getMonth()); Assertions.assertEquals(7, localDateTime.getDayOfMonth()); Assertions.assertEquals(2022, localDateTime.getYear()); - Assertions.assertNotNull(localDateTime.getHour()); - Assertions.assertEquals(0, localDateTime.getMinute()); - Assertions.assertEquals(0, localDateTime.getSecond()); } @Test From a661e1f266b89d9ca3df5a30c4d0ca7a1b17edd5 Mon Sep 17 00:00:00 2001 From: Sahiba Mittal Date: Wed, 11 Oct 2023 15:39:08 +0100 Subject: [PATCH 17/17] Update IMetaAnalyzer.java --- .../src/main/java/org/hyades/repositories/IMetaAnalyzer.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/repository-meta-analyzer/src/main/java/org/hyades/repositories/IMetaAnalyzer.java b/repository-meta-analyzer/src/main/java/org/hyades/repositories/IMetaAnalyzer.java index 739cd340f..15a687913 100644 --- a/repository-meta-analyzer/src/main/java/org/hyades/repositories/IMetaAnalyzer.java +++ b/repository-meta-analyzer/src/main/java/org/hyades/repositories/IMetaAnalyzer.java @@ -83,11 +83,9 @@ public interface IMetaAnalyzer { String getName(); /** - * The component meta data to analyze for integrity. + * The component integrity meta. * @param component the component to analyze * @return an IntegrityMeta object - * @throws MetaAnalyzerException in case of any issue during metadata generation - * @since 3.1.0 */ IntegrityMeta getIntegrityMeta(Component component); }