Skip to content

Commit

Permalink
Feature/revert debug logs (#667)
Browse files Browse the repository at this point in the history
* added metaanalyzerProcessorTest

Signed-off-by: mehab <meha.bhargava@citi.com>

* improved test coverage

Signed-off-by: mehab <meha.bhargava@citi.com>

---------

Signed-off-by: mehab <meha.bhargava@citi.com>
  • Loading branch information
mehab authored Jul 14, 2023
1 parent 4dcd67e commit 5753d43
Show file tree
Hide file tree
Showing 9 changed files with 213 additions and 14 deletions.
6 changes: 5 additions & 1 deletion commons-persistence/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,11 @@
<groupId>io.quarkus</groupId>
<artifactId>quarkus-jdbc-postgresql</artifactId>
</dependency>

<dependency>
<groupId>io.quarkus</groupId>
<artifactId>quarkus-jacoco</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.github.package-url</groupId>
<artifactId>packageurl-java</artifactId>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
@TestProfile(NotificationRouterIT.TestProfile.class)
@QuarkusTestResource(KafkaCompanionResource.class)
@QuarkusTestResource(WireMockTestResource.class)
public class NotificationRouterIT {
class NotificationRouterIT {

public static class TestProfile implements QuarkusTestProfile {

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ public void process(final FixedKeyRecord<PackageURL, Component> record) {
// We do not want non-internal components being analyzed with internal repositories as
// internal repositories are not the source of truth for these components, even if the
// repository acts as a proxy to the source of truth. This cannot be assumed.
LOGGER.debug("Skipping component with purl {} ", component.getPurl());
continue;
}

Expand Down Expand Up @@ -171,7 +172,9 @@ private List<Repository> getApplicableRepositories(final RepositoryType reposito
// Quarkus has Hibernate L2 cache enabled by default, we just need to opt in to using
// it for this query: https://quarkus.io/guides/hibernate-orm#caching-of-queries
// Should be tested whether throughput can be improved this way.
return QuarkusTransaction.requiringNew()
//changed this to joinexisting because with new transaction, it is not fetching values that were inserted from
// existing previous transaction and returning empty result
return QuarkusTransaction.joiningExisting()
.call(() -> repoEntityRepository.findEnabledRepositoriesByType(repositoryType));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,6 @@ protected CloseableHttpResponse processHttpRequest(String url) throws IOExceptio
final HttpUriRequest request = new HttpGet(url);
request.addHeader("accept", "application/json");
if (username != null || password != null) {
logger.debug("Adding user name and password to request provided");
request.addHeader("Authorization", HttpUtil.basicAuthHeaderValue(username, password));
}
return httpClient.execute(request);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@

import com.github.packageurl.PackageURL;
import org.apache.http.HttpEntity;
import org.apache.http.HttpStatus;
import org.apache.http.StatusLine;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.hyades.commonutil.DateUtil;
import org.hyades.commonutil.XmlUtil;
import org.hyades.model.MetaAnalyzerException;
import org.hyades.model.MetaModel;
import org.hyades.persistence.model.Component;
import org.hyades.persistence.model.RepositoryType;
Expand Down Expand Up @@ -81,7 +81,7 @@ public MetaModel analyze(final Component component) {
final String url = String.format(baseUrl + REPO_METADATA_URL, mavenGavUrl);
try (final CloseableHttpResponse response = processHttpRequest(url)) {
final StatusLine status = response.getStatusLine();
if (status.getStatusCode() == 200) {
if (status.getStatusCode() == HttpStatus.SC_OK) {
final HttpEntity entity = response.getEntity();
if (entity != null) {
try (InputStream in = entity.getContent()) {
Expand Down Expand Up @@ -109,9 +109,6 @@ public MetaModel analyze(final Component component) {
} catch (IOException | ParserConfigurationException | SAXException | XPathExpressionException e) {
LOGGER.error("Failed to perform repo meta analysis for component with purl:{}", component.getPurl());
handleRequestException(LOGGER, e);
} catch (Exception ex) {
LOGGER.error("Failed to perform repo meta analysis for component with purl:{}", component.getPurl());
throw new MetaAnalyzerException(ex);
}
}
return meta;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,7 @@ quarkus.http.port=8091
## Logging
#
quarkus.log.console.json=false
#temporary setting to verify from logs
quarkus.log.category."org.apache.kafka".level=DEBUG
quarkus.log.category."org.apache.kafka".level=WARN

## Kafka
#
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,8 +54,8 @@ void beforeEach() throws Exception {
ConfigProvider.getConfig().getValue("quarkus.datasource.username", String.class),
ConfigProvider.getConfig().getValue("quarkus.datasource.password", String.class))) {
final PreparedStatement ps = connection.prepareStatement("""
INSERT INTO "REPOSITORY" ("ENABLED", "IDENTIFIER", "INTERNAL", "PASSWORD", "RESOLUTION_ORDER", "TYPE", "URL")
VALUES ('true', 'test', false, NULL, 1, 'GO_MODULES', 'http://localhost:%d');
INSERT INTO "REPOSITORY" ("ENABLED", "IDENTIFIER", "INTERNAL", "PASSWORD", "RESOLUTION_ORDER", "TYPE", "URL", "AUTHENTICATIONREQUIRED")
VALUES ('true', 'test', false, NULL, 1, 'GO_MODULES', 'http://localhost:%d', false);
""".formatted(wireMockServer.port()));
ps.execute();
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
package org.hyades.processor;

import com.github.packageurl.MalformedPackageURLException;
import com.github.packageurl.PackageURL;
import io.quarkus.cache.Cache;
import io.quarkus.cache.CacheName;
import io.quarkus.test.TestTransaction;
import io.quarkus.test.junit.QuarkusTest;
import io.quarkus.test.junit.QuarkusTestProfile;
import io.quarkus.test.junit.TestProfile;
import jakarta.inject.Inject;
import jakarta.persistence.EntityManager;
import org.apache.kafka.streams.StreamsBuilder;
import org.apache.kafka.streams.TestInputTopic;
import org.apache.kafka.streams.TestOutputTopic;
import org.apache.kafka.streams.TopologyTestDriver;
import org.apache.kafka.streams.errors.StreamsException;
import org.apache.kafka.streams.kstream.Consumed;
import org.apache.kafka.streams.kstream.Produced;
import org.apache.kafka.streams.test.TestRecord;
import org.hyades.common.SecretDecryptor;
import org.hyades.persistence.model.RepositoryType;
import org.hyades.persistence.repository.RepoEntityRepository;
import org.hyades.proto.KafkaProtobufSerde;
import org.hyades.proto.repometaanalysis.v1.AnalysisResult;
import org.hyades.proto.repometaanalysis.v1.Component;
import org.hyades.repositories.RepositoryAnalyzerFactory;
import org.hyades.serde.KafkaPurlSerde;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;

import java.util.Map;

import static org.assertj.core.api.Assertions.assertThat;

@QuarkusTest
@TestProfile(MetaAnalyzerProcessorTest.TestProfile.class)
class MetaAnalyzerProcessorTest {

public static class TestProfile implements QuarkusTestProfile {
@Override
public Map<String, String> getConfigOverrides() {
return Map.of(
"quarkus.kafka.snappy.enabled", "false"
);
}
}

private static final String TEST_PURL_JACKSON_BIND = "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4";

private TopologyTestDriver testDriver;
private TestInputTopic<PackageURL, Component> inputTopic;
private TestOutputTopic<PackageURL, AnalysisResult> outputTopic;
@Inject
RepoEntityRepository repoEntityRepository;

@Inject
RepositoryAnalyzerFactory analyzerFactory;

@Inject
EntityManager entityManager;

@Inject
@CacheName("metaAnalyzer")
Cache cache;

@Inject
SecretDecryptor secretDecryptor;

@BeforeEach
void beforeEach() {
final var processorSupplier = new MetaAnalyzerProcessorSupplier(repoEntityRepository, analyzerFactory, secretDecryptor, cache);

final var valueSerde = new KafkaProtobufSerde<>(Component.parser());
final var purlSerde = new KafkaPurlSerde();
final var valueSerdeResult = new KafkaProtobufSerde<>(AnalysisResult.parser());

final var streamsBuilder = new StreamsBuilder();
streamsBuilder
.stream("input-topic", Consumed.with(purlSerde, valueSerde))
.processValues(processorSupplier)
.to("output-topic", Produced.with(purlSerde, valueSerdeResult));

testDriver = new TopologyTestDriver(streamsBuilder.build());
inputTopic = testDriver.createInputTopic("input-topic", purlSerde.serializer(), valueSerde.serializer());
outputTopic = testDriver.createOutputTopic("output-topic", purlSerde.deserializer(), valueSerdeResult.deserializer());
}

@AfterEach
void afterEach() {
testDriver.close();
cache.invalidateAll().await().indefinitely();
}

@Test
void testWithNoSupportedRepositoryTypes() throws Exception {
final TestRecord<PackageURL, Component> inputRecord = new TestRecord<>(new PackageURL(TEST_PURL_JACKSON_BIND), Component.newBuilder()
.setPurl(TEST_PURL_JACKSON_BIND).build());
inputTopic.pipeInput(inputRecord);
assertThat(outputTopic.getQueueSize()).isEqualTo(1);
assertThat(outputTopic.readRecordsToList()).satisfiesExactly(
record -> {
assertThat(record.key().getType()).isEqualTo(RepositoryType.MAVEN.toString().toLowerCase());
});
}

@Test
void testMalformedPurl() throws Exception {
final TestRecord<PackageURL, Component> inputRecord = new TestRecord<>(new PackageURL(TEST_PURL_JACKSON_BIND), Component.newBuilder()
.setPurl("invalid purl").build());
Assertions.assertThrows(StreamsException.class, () -> {
inputTopic.pipeInput(inputRecord);
}, "no exception thrown");

}

@Test
void testNoAnalyzerApplicable() throws Exception {
final TestRecord<PackageURL, Component> inputRecord = new TestRecord<>(new PackageURL("pkg:test/com.fasterxml.jackson.core/jackson-databind@2.13.4"), Component.newBuilder()
.setPurl("pkg:test/com.fasterxml.jackson.core/jackson-databind@2.13.4").build());
inputTopic.pipeInput(inputRecord);
assertThat(outputTopic.getQueueSize()).isEqualTo(1);
assertThat(outputTopic.readRecordsToList()).satisfiesExactly(
record -> {
assertThat(record.key().getType()).isEqualTo("test");
});

}

@Test
@TestTransaction
void testInternalRepositoryExternalComponent() throws MalformedPackageURLException {
entityManager.createNativeQuery("""
INSERT INTO "REPOSITORY" ("TYPE", "ENABLED","IDENTIFIER", "INTERNAL", "URL", "AUTHENTICATIONREQUIRED", "RESOLUTION_ORDER") VALUES
('MAVEN',true, 'central', true, 'test.com', false,1);
""").executeUpdate();

final TestRecord<PackageURL, Component> inputRecord = new TestRecord<>(new PackageURL("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4"), Component.newBuilder()
.setPurl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4").setInternal(false).build());
inputTopic.pipeInput(inputRecord);
assertThat(outputTopic.getQueueSize()).isEqualTo(1);
assertThat(outputTopic.readRecordsToList()).satisfiesExactly(
record -> {
assertThat(record.key().getType()).isEqualTo(RepositoryType.MAVEN.toString().toLowerCase());
});

}

@Test
@TestTransaction
void testExternalRepositoryInternalComponent() throws MalformedPackageURLException {
entityManager.createNativeQuery("""
INSERT INTO "REPOSITORY" ("TYPE", "ENABLED","IDENTIFIER", "INTERNAL", "URL", "AUTHENTICATIONREQUIRED", "RESOLUTION_ORDER") VALUES
('MAVEN',true, 'central', false, 'test.com', false,1);
""").executeUpdate();

final TestRecord<PackageURL, Component> inputRecord = new TestRecord<>(new PackageURL("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4"), Component.newBuilder()
.setPurl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.4").setInternal(true).build());
inputTopic.pipeInput(inputRecord);
assertThat(outputTopic.getQueueSize()).isEqualTo(1);
assertThat(outputTopic.readRecordsToList()).satisfiesExactly(
record -> {
assertThat(record.key().getType()).isEqualTo(RepositoryType.MAVEN.toString().toLowerCase());
});

}

}
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@ void beforeEach() {
void testAnalyzer() throws Exception {
Component component = new Component();
component.setPurl(new PackageURL("pkg:maven/junit/junit@4.12"));

Assertions.assertEquals("MavenMetaAnalyzer", analyzer.getName());
Assertions.assertTrue(analyzer.isApplicable(component));
Assertions.assertEquals(RepositoryType.MAVEN, analyzer.supportedRepositoryType());
Expand All @@ -63,4 +62,32 @@ void testAnalyzerForScalaComponent() throws Exception {
Assertions.assertNotNull(metaModel.getLatestVersion());
Assertions.assertNotNull(metaModel.getPublishedTimestamp());
}

@Test
void testComponentWithNullPurl() {
Component component = new Component();
Assertions.assertFalse(analyzer.isApplicable(component));
MetaModel metaModel = analyzer.analyze(component);
Assertions.assertNull(metaModel.getComponent().getPurl());
}

@Test
void testComponentWithNonMavenPurl() {
Component component = new Component();
component.setPurl("pkg:pypi/com.typesafe.akka/package-does-not-exist@v1.2.0");
Assertions.assertFalse(analyzer.isApplicable(component));
MetaModel metaModel = analyzer.analyze(component);
Assertions.assertEquals(RepositoryType.PYPI.name(), metaModel.getComponent().getPurl().getType().toUpperCase());
}

@Test
void testIOException() {
Component component = new Component();
component.setPurl("pkg:pypi/com.typesafe.akka/package-does-not-exist@v1.2.0");
analyzer.setRepositoryBaseUrl("http://www.does.not.exist.com");
MetaModel metaModel = analyzer.analyze(component);
Assertions.assertEquals(metaModel.getComponent(), component);

}
}

0 comments on commit 5753d43

Please sign in to comment.