feat(travis): make it possible to specify the list of repos to scan (#…

…790)

* feat(travis): make it possible to specify the list of repos to scan

Signed-off-by: cmuraru <cmuraru@adobe.com>

* Implement review

Co-authored-by: cmuraru <cmuraru@adobe.com>

igor-monitor-travis/src/main/java/com/netflix/spinnaker/igor/travis/TravisBuildMonitor.java

@@ -108,9 +108,10 @@ protected BuildPollingDelta generateDelta(PollContext ctx) {
             .collect(Collectors.toList());
 
     log.info(
-        "({}) generateDelta: Took {}ms to generate polling delta",
+        "({}) generateDelta: Took {}ms to generate polling delta. Polled {} builds.",
         kv("master", master),
-        Duration.between(startTime, Instant.now()).toMillis());
+        Duration.between(startTime, Instant.now()).toMillis(),
+        builds.size());
 
     return BuildPollingDelta.builder().master(master).items(builds).build();
   }

igor-monitor-travis/src/main/java/com/netflix/spinnaker/igor/travis/config/TravisConfig.java

@@ -100,6 +100,7 @@ public Map<String, TravisService> travisMasters(
                           host.getGithubToken(),
                           host.getNumberOfJobs(),
                           host.getBuildResultLimit(),
+                          host.getFilteredRepositories(),
                           client,
                           travisCache,
                           artifactDecorator,

igor-monitor-travis/src/main/java/com/netflix/spinnaker/igor/travis/config/TravisProperties.java

@@ -19,6 +19,8 @@
 
 import com.netflix.spinnaker.fiat.model.resources.Permissions;
 import com.netflix.spinnaker.igor.config.BuildServerProperties;
+import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import javax.validation.Valid;
 import javax.validation.constraints.NotEmpty;
@@ -68,6 +70,8 @@ public static class TravisHost implements BuildServerProperties.Host {
      */
     private int buildResultLimit = 10;
 
+    private Collection<String> filteredRepositories = Collections.emptySet();
+
     private Integer itemUpperThreshold;
     private Permissions.Builder permissions = new Permissions.Builder();
 

igor-monitor-travis/src/main/java/com/netflix/spinnaker/igor/travis/service/TravisService.java

@@ -17,6 +17,7 @@
 
 package com.netflix.spinnaker.igor.travis.service;
 
+import static java.util.Collections.emptyList;
 import static net.logstash.logback.argument.StructuredArguments.kv;
 
 import com.netflix.spinnaker.fiat.model.resources.Permissions;
@@ -52,6 +53,7 @@
 import io.github.resilience4j.circuitbreaker.CircuitBreaker;
 import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry;
 import io.github.resilience4j.core.SupplierUtils;
+import io.vavr.control.Either;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -82,6 +84,7 @@ public class TravisService implements BuildOperations, BuildProperties {
   private final GithubAuth gitHubAuth;
   private final int numberOfJobs;
   private final int buildResultLimit;
+  private Collection<String> filteredRepositories;
   private final TravisClient travisClient;
   private final TravisCache travisCache;
   private final Collection<String> artifactRegexes;
@@ -98,6 +101,7 @@ public TravisService(
       String githubToken,
       int numberOfJobs,
       int buildResultLimit,
+      Collection<String> filteredRepositories,
       TravisClient travisClient,
       TravisCache travisCache,
       Optional<ArtifactDecorator> artifactDecorator,
@@ -110,6 +114,7 @@ public TravisService(
     this.buildResultLimit = buildResultLimit;
     this.groupKey = travisHostId;
     this.gitHubAuth = new GithubAuth(githubToken);
+    this.filteredRepositories = filteredRepositories;
     this.travisClient = travisClient;
     this.baseUrl = baseUrl;
     this.travisCache = travisCache;
@@ -309,13 +314,18 @@ private List<V3Job> getJobs(int limit, TravisBuildState... buildStatesFilter) {
                     .collect(Collectors.toList()),
             error -> {
               log.warn("An error occurred while fetching new jobs from Travis.", error);
-              return Collections.emptyList();
+              return emptyList();
             });
 
     return breaker.executeSupplier(supplier);
   }
 
   public List<V3Build> getLatestBuilds() {
+
+    if (!filteredRepositories.isEmpty()) {
+      return getBuildsForSpecificRepos(filteredRepositories);
+    }
+
     Map<V3Build, List<V3Job>> jobs =
         getJobs(
                 numberOfJobs,
@@ -338,6 +348,28 @@ public List<V3Build> getLatestBuilds() {
         .collect(Collectors.toList());
   }
 
+  private List<V3Build> getBuildsForSpecificRepos(Collection<String> repositories) {
+    return repositories
+        .parallelStream()
+        .map(this::fetchBuilds)
+        .filter(Either::isLeft)
+        .map(Either::getLeft)
+        .flatMap(item -> item.getBuilds().stream())
+        .collect(Collectors.toList());
+  }
+
+  private Either<V3Builds, Exception> fetchBuilds(String repoSlug) {
+    try {
+      log.info("fetching builds for repo: {}", repoSlug);
+      return Either.left(
+          travisClient.v3builds(
+              getAccessToken(), repoSlug, buildResultLimit, addLogCompleteIfApplicable()));
+    } catch (Exception e) {
+      log.error("Failed to fetch builds for repo {}. Skipping. {}.", repoSlug, e.getMessage());
+      return Either.right(e);
+    }
+  }
+
   private String addLogCompleteIfApplicable(String... initialParams) {
     List<String> queryParams = new ArrayList<>(Arrays.asList(initialParams));
     if (!legacyLogFetching) {

igor-monitor-travis/src/test/groovy/com/netflix/spinnaker/igor/travis/service/TravisServiceSpec.groovy

@@ -38,12 +38,15 @@ import com.netflix.spinnaker.igor.travis.client.model.v3.V3Jobs
 import com.netflix.spinnaker.igor.travis.client.model.v3.V3Log
 import com.netflix.spinnaker.igor.travis.client.model.v3.V3Repository
 import io.github.resilience4j.circuitbreaker.CircuitBreakerRegistry
+import org.assertj.core.util.Lists
 import spock.lang.Shared
 import spock.lang.Specification
 import spock.lang.Unroll
 
 import java.time.Instant
 
+import static java.util.Collections.emptyList
+
 class TravisServiceSpec extends Specification {
     @Shared
     TravisClient client
@@ -62,7 +65,7 @@ class TravisServiceSpec extends Specification {
         client = Mock()
         travisCache = Mock()
         artifactDecorator = Optional.of(new ArtifactDecorator([new DebDetailsDecorator(), new RpmDetailsDecorator()], null))
-        service = new TravisService('travis-ci', 'http://my.travis.ci', 'someToken', TravisService.TRAVIS_JOB_RESULT_LIMIT, TRAVIS_BUILD_RESULT_LIMIT, client, travisCache, artifactDecorator, [], "travis.buildMessage", Permissions.EMPTY, false, CircuitBreakerRegistry.ofDefaults())
+        service = new TravisService('travis-ci', 'http://my.travis.ci', 'someToken', TravisService.TRAVIS_JOB_RESULT_LIMIT, TRAVIS_BUILD_RESULT_LIMIT, emptyList(), client, travisCache, artifactDecorator, [], "travis.buildMessage", Permissions.EMPTY, false, CircuitBreakerRegistry.ofDefaults())
 
         AccessToken accessToken = new AccessToken()
         accessToken.accessToken = "someToken"
@@ -102,7 +105,7 @@ class TravisServiceSpec extends Specification {
     @Unroll
     def "getLatestBuilds() with #numberOfJobs jobs should query Travis #expectedNumberOfPages time(s)"() {
         given:
-        service = new TravisService('travis-ci', 'http://my.travis.ci', 'someToken', numberOfJobs, TRAVIS_BUILD_RESULT_LIMIT, client, travisCache, artifactDecorator, [], "travis.buildMessage", Permissions.EMPTY, false, CircuitBreakerRegistry.ofDefaults())
+        service = new TravisService('travis-ci', 'http://my.travis.ci', 'someToken', numberOfJobs, TRAVIS_BUILD_RESULT_LIMIT, emptyList(), client, travisCache, artifactDecorator, [], "travis.buildMessage", Permissions.EMPTY, false, CircuitBreakerRegistry.ofDefaults())
         AccessToken accessToken = new AccessToken()
         accessToken.accessToken = "someToken"
         service.accessToken = accessToken
@@ -150,6 +153,36 @@ class TravisServiceSpec extends Specification {
         }
     }
 
+    def "getLatestBuilds() with filteredRepositories should fetch builds only for those repos"() {
+      given:
+      List<String> filteredRepos = Lists.newArrayList("myorg/myrepo")
+      service = new TravisService('travis-ci', 'http://my.travis.ci', 'someToken', 100, TRAVIS_BUILD_RESULT_LIMIT, filteredRepos, client, travisCache, artifactDecorator, [], "travis.buildMessage", Permissions.EMPTY, false, CircuitBreakerRegistry.ofDefaults())
+      AccessToken accessToken = new AccessToken()
+      accessToken.accessToken = "someToken"
+      service.accessToken = accessToken
+
+      def job = new V3Job().with { v3job ->
+        v3job.id = 2
+        return v3job
+      }
+
+      def build = new V3Build([
+        id: 1,
+        jobs: [job],
+        state: TravisBuildState.passed,
+        repository: new V3Repository([slug: "myorg/myrepo"])
+      ])
+
+      when:
+      def builds = service.getLatestBuilds()
+
+      then:
+      1 * client.v3builds("token someToken", "myorg/myrepo", TRAVIS_BUILD_RESULT_LIMIT,
+        "build.log_complete") >> new V3Builds([builds: [build]])
+      0 * client.jobs(*_)
+      builds == [build]
+    }
+
     @Unroll
     def "cleanRepoSlug(repoSlug)"() {
         expect:
@@ -306,7 +339,7 @@ class TravisServiceSpec extends Specification {
                 ])],
             content: "log"
         ])
-        service = new TravisService('travis-ci', 'http://my.travis.ci', 'someToken', 25, TRAVIS_BUILD_RESULT_LIMIT, client, travisCache, artifactDecorator, [], "travis.buildMessage", Permissions.EMPTY, legacyLogFetching, CircuitBreakerRegistry.ofDefaults())
+        service = new TravisService('travis-ci', 'http://my.travis.ci', 'someToken', 25, TRAVIS_BUILD_RESULT_LIMIT, emptyList(), client, travisCache, artifactDecorator, [], "travis.buildMessage", Permissions.EMPTY, legacyLogFetching, CircuitBreakerRegistry.ofDefaults())
         AccessToken accessToken = new AccessToken()
         accessToken.accessToken = "someToken"
         service.accessToken = accessToken

igor-web/config/igor.yml

@@ -1,3 +1,5 @@
+services.fiat.baseUrl: https://fiat.net
+
 netflix:
   appinfo:
     name: 'igor'
@@ -9,7 +11,7 @@ spinnaker:
     # A lower value can help prevent accidental re-triggers of pipelines, but may require hands-on operations if
     # set low.
     # See code documentation for more details. Class [PollingSafeguardProperties] in
-    # https://github.com/spinnaker/igor/blob/master/igor-web/src/main/groovy/com/netflix/spinnaker/igor/IgorConfigurationProperties.groovy#L47
+    # https://github.com/spinnaker/igor/blob/master/igor-core/src/main/java/com/netflix/spinnaker/igor/IgorConfigurationProperties.java#L92
     itemUpperThreshold: 1000
 
 endpoints.health.sensitive: false

igor-web/src/test/groovy/com/netflix/spinnaker/igor/build/InfoControllerSpec.groovy

@@ -39,6 +39,7 @@ import spock.lang.Shared
 import spock.lang.Specification
 import spock.lang.Unroll
 
+import static java.util.Collections.emptyList
 import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get
 /**
  * tests for the info controller
@@ -154,7 +155,7 @@ class InfoControllerSpec extends Specification {
                 .add(Authorization.READ, ['group-1', 'group-2'])
                 .add(Authorization.WRITE, 'group-2').build(),
           circuitBreakerRegistry)
-        TravisService travisService = new TravisService('travis-baz', null, null, 100, 10, null, null, Optional.empty(), [], null,
+        TravisService travisService = new TravisService('travis-baz', null, null, 100, 10, emptyList(), null, null, Optional.empty(), [], null,
             new Permissions.Builder()
                 .add(Authorization.READ, ['group-3', 'group-4'])
                 .add(Authorization.WRITE, 'group-3').build(), false, CircuitBreakerRegistry.ofDefaults())