Integrate Integrity metadata in repo-meta-analyzer (#835)

commons/src/main/java/org/hyades/commonutil/DateUtil.java

@@ -30,16 +30,16 @@ private DateUtil() {
     }
 
     /**
-     * Convenience method that parses a date in yyyyMMddHHmmss format and
+     * Convenience method that parses a date in given format and
      * returns a Date object. If the parsing fails, null is returned.
-     * @param yyyyMMddHHmmss the date string to parse
+     * @param date the date string to parse
+     * @param dateFormat the date format
      * @return a Date object
-     * @since 3.1.0
      */
-    public static Date parseDate(final String yyyyMMddHHmmss) {
-        final SimpleDateFormat format = new SimpleDateFormat("yyyyMMddHHmmss");
+    public static Date parseDate(final String date, final String dateFormat) {
+        final SimpleDateFormat format = new SimpleDateFormat(dateFormat);
         try {
-            return format.parse(yyyyMMddHHmmss);
+            return format.parse(date);
         } catch (ParseException e) {
             return null;
         }

commons/src/test/java/org/hyades/commonutil/DateUtilTest.java

@@ -31,7 +31,7 @@ public class DateUtilTest {
 
     @Test
     public void testParseDate() {
-        Date date = DateUtil.parseDate("20191231153012");
+        Date date = DateUtil.parseDate("20191231153012", "yyyyMMddHHmmss");
         LocalDateTime localDateTime = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime();
         Assertions.assertEquals(Month.DECEMBER, localDateTime.getMonth());
         Assertions.assertEquals(31, localDateTime.getDayOfMonth());
@@ -41,6 +41,15 @@ public void testParseDate() {
         Assertions.assertEquals(12, localDateTime.getSecond());
     }
 
+    @Test
+    public void testParseGMTDate() {
+        Date date = DateUtil.parseDate("Thu, 07 Jul 2022 00:00:00 GMT", "EEE, dd MMM yyyy HH:mm:ss Z");
+        LocalDateTime localDateTime = date.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime();
+        Assertions.assertEquals(Month.JULY, localDateTime.getMonth());
+        Assertions.assertEquals(7, localDateTime.getDayOfMonth());
+        Assertions.assertEquals(2022, localDateTime.getYear());
+    }
+
     @Test
     public void testToISO8601() {
         Date date = Date.from(LocalDateTime.of(2019, Month.JANUARY, 31, 15, 30, 12).toInstant(ZoneOffset.UTC));

proto/src/main/proto/org/hyades/repometaanalysis/v1/repo_meta_analysis.proto

@@ -11,6 +11,14 @@ option java_package = "org.hyades.proto.repometaanalysis.v1";
 message AnalysisCommand {
   // The component that shall be analyzed.
   Component component = 1;
+  FetchMeta fetch_meta = 2;
+}
+
+enum FetchMeta{
+  FETCH_META_UNSPECIFIED = 0;
+  FETCH_META_INTEGRITY_DATA = 1;
+  FETCH_META_LATEST_VERSION = 2;
+  FETCH_META_INTEGRITY_DATA_AND_LATEST_VERSION = 3;
 }
 
 message AnalysisResult {
@@ -25,6 +33,9 @@ message AnalysisResult {
 
   // When the latest version was published.
   optional google.protobuf.Timestamp published = 4;
+
+  // Integrity metadata of the component.
+  optional IntegrityMeta integrity_meta = 5;
 }
 
 message Component {
@@ -35,3 +46,14 @@ message Component {
   // Internal components will only be looked up in internal repositories.
   optional bool internal = 2;
 }
+
+message IntegrityMeta {
+  optional string md5 = 1;
+  optional string sha1 = 2;
+  optional string sha256 = 3;
+  optional string sha512 = 4;
+  // When the component current version last modified.
+  optional google.protobuf.Timestamp current_version_last_modified = 5;
+  // Complete URL to fetch integrity metadata of the component.
+  optional string meta_source_url = 6;
+}

repository-meta-analyzer/src/main/java/org/hyades/RepositoryMetaAnalyzerTopology.java

@@ -17,12 +17,12 @@
 import org.hyades.proto.KafkaProtobufSerde;
 import org.hyades.proto.repometaanalysis.v1.AnalysisCommand;
 import org.hyades.proto.repometaanalysis.v1.AnalysisResult;
-import org.hyades.proto.repometaanalysis.v1.Component;
 import org.hyades.repositories.RepositoryAnalyzerFactory;
 import org.hyades.serde.KafkaPurlSerde;
 
 import static org.hyades.kstreams.util.KafkaStreamsUtil.processorNameConsume;
 import static org.hyades.kstreams.util.KafkaStreamsUtil.processorNameProduce;
+import static org.hyades.util.PurlUtil.parsePurlCoordinatesWithoutVersion;
 
 public class RepositoryMetaAnalyzerTopology {
 
@@ -48,7 +48,7 @@ public Topology topology(final RepositoryAnalyzerFactory analyzerFactory,
                 //
                 // Because we can't enforce this format on the keys of the input topic without causing
                 // serialization exceptions, we're left with this mandatory key change.
-                .selectKey((key, command) -> mustParsePurlCoordinatesWithoutVersion(command.getComponent().getPurl()),
+                .selectKey((key, command) -> parsePurlCoordinatesWithoutVersion(command.getComponent().getPurl()),
                         Named.as("re-key_to_purl_coordinates"))
                 // Force a repartition to ensure that the ordering guarantees we want, based on the
                 // previous re-keying operation, are effective.
@@ -57,19 +57,17 @@ public Topology topology(final RepositoryAnalyzerFactory analyzerFactory,
                         .withName("command-by-purl-coordinates"));
 
         commandStream
-                .mapValues((purl, command) -> command.getComponent(),
-                        Named.as("map_to_component"))
                 .split(Named.as("applicable_analyzer"))
-                .branch((purl, component) -> analyzerFactory.hasApplicableAnalyzer(purl), Branched
-                        .<PackageURL, Component>withConsumer(stream -> stream
+                .branch((purl, command) -> analyzerFactory.hasApplicableAnalyzer(purl), Branched
+                        .<PackageURL, AnalysisCommand>withConsumer(stream -> stream
                                 .processValues(analyzerProcessorSupplier, Named.as("analyze_component"))
                                 .to(KafkaTopic.REPO_META_ANALYSIS_RESULT.getName(), Produced
                                         .with(purlSerde, scanResultSerde)
                                         .withName(processorNameProduce(KafkaTopic.REPO_META_ANALYSIS_RESULT, "analysis_result"))))
                         .withName("-found"))
                 .defaultBranch(Branched
-                        .<PackageURL, Component>withConsumer(stream -> stream
-                                .mapValues((purl, component) -> AnalysisResult.newBuilder().setComponent(component).build(),
+                        .<PackageURL, AnalysisCommand>withConsumer(stream -> stream
+                                .mapValues((purl, command) -> AnalysisResult.newBuilder().setComponent(command.getComponent()).build(),
                                         Named.as("map_unmatched_component_to_empty_result"))
                                 .to(KafkaTopic.REPO_META_ANALYSIS_RESULT.getName(), Produced
                                         .with(purlSerde, scanResultSerde)
@@ -87,18 +85,4 @@ private boolean isValidPurl(final String purl) {
             return false;
         }
     }
-
-    private PackageURL mustParsePurlCoordinatesWithoutVersion(final String purl) {
-        try {
-            final var parsedPurl = new PackageURL(purl);
-            return new PackageURL(parsedPurl.getType(), parsedPurl.getNamespace(),
-                    parsedPurl.getName(), null, null, null);
-        } catch (MalformedPackageURLException e) {
-            throw new IllegalStateException("""
-                    The provided PURL is invalid, even though it should have been
-                    validated in a previous processing step
-                    """, e);
-        }
-    }
-
 }

repository-meta-analyzer/src/main/java/org/hyades/model/IntegrityMeta.java

@@ -0,0 +1,67 @@
+package org.hyades.model;
+
+import java.io.Serializable;
+import java.util.Date;
+
+public class IntegrityMeta implements Serializable {
+
+    private String md5;
+
+    private String sha1;
+
+    private String sha256;
+
+    private String sha512;
+
+    private Date currentVersionLastModified;
+
+    private String metaSourceUrl;
+
+    public String getMd5() {
+        return md5;
+    }
+
+    public void setMd5(String md5) {
+        this.md5 = md5;
+    }
+
+    public String getSha1() {
+        return sha1;
+    }
+
+    public void setSha1(String sha1) {
+        this.sha1 = sha1;
+    }
+
+    public String getSha256() {
+        return sha256;
+    }
+
+    public void setSha256(String sha256) {
+        this.sha256 = sha256;
+    }
+
+    public Date getCurrentVersionLastModified() {
+        return currentVersionLastModified;
+    }
+
+    public void setCurrentVersionLastModified(Date currentVersionLastModified) {
+        this.currentVersionLastModified = currentVersionLastModified;
+    }
+
+    public void setSha512(String sha512) {
+        this.sha512 = sha512;
+    }
+
+    public String getSha512() {
+        return sha512;
+    }
+
+    public String getMetaSourceUrl() {
+        return metaSourceUrl;
+    }
+
+    public void setMetaSourceUrl(String metaSourceUrl) {
+        this.metaSourceUrl = metaSourceUrl;
+    }
+}