Implemented Gitlab CI polling and build property support (#986)

* feat(gitlabci): Completed GitLab CI polling and added build properties implementation

* chore(gitlabci): removed unused class

* chore(gitlabci): removed unused test code

* fix(gitlabci): Reverted config from gitlabci to gitlab-ci

* fix(gitlabci): Reverted build service provider to GITLAB_CI

* chore(gitlabci): Fixing typo in README

* chore(gitlabci): Adjust wording of integration test for accuracy

Co-authored-by: David Byron <82477955+dbyron-sf@users.noreply.github.com>

* feat(gitlabci): Additional gitlab-ci config settings, README docs

* chore(gitlabci): Updated code comments for additional clarity

* fix(gitlabci): Fixing false positive matches in build log property parser (#2)

* feat(gitlabci): Added additional GitlabCI configuration options

* chore(gitlabci): Additional unit tests and configuration validation

Co-authored-by: David Byron <82477955+dbyron-sf@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

README.md

@@ -164,6 +164,33 @@ that will match on output from the `jfrog rt`/`art` CLI tool.  Different regexes
 default may be configured using the `regexes` list.
 
 
+## Configuring Gitlab CI Masters
+
+In your configuration block (either in igor.yml, igor-local.yml, spinnaker.yml or spinnaker-local.yml), you can define multiple masters blocks by using the list format.
+
+To authenticate with Gitlab CI use a [Personal Access Token](https://docs.gitlab.com/ee/security/token_overview.html#personal-access-tokens) with permissions `read_api`.
+
+```
+gitlab-ci:
+  enabled: true
+  itemUpperThreshold: 1000 # Optional, default 1000.  Determines max new pipeline count before a cache cycle is rejected
+  masters:
+    - address: "https://git.mycompany.com"
+      name: mygitlab
+      privateToken: kjsdf023ofku209823
+      # Optional:
+      defaultHttpPageLength: 100 # defaults 100, page length when querying paginated Gitlab API endpoints (100 is max per Gitlab docs)
+      limitByOwnership: false # defaults false, limits API results to projects/groups owned by the token creator
+      limitByMembership: true # defaults true, limits API results to projects/groups the token creator is a member in
+      httpRetryMaxAttempts: 5 # defaults 5, # default max number of retries when hitting Gitlab APIs and errors occur
+      httpRetryWaitSeconds: 2 # defaults 2, # of seconds to wait between retries
+      httpRetryExponentialBackoff: false # deafults false, if true retries to Gitlab will increase exponentially using the httpRetryWaitSeconds option's value
+```
+
+Build properties are automatically read from successful Gitlab CI Pipelines using the pattern `SPINNAKER_PROPERTY_*=value`.  For example a log containing a line
+`SPINNAKER_PROPERTY_HELLO=world` will create a build property item `hello=world`.  Gitlab CI artifacts are not yet supported.
+
+
 ## Integration with Docker Registry
 
 Clouddriver can be [configured to poll your registries](http://www.spinnaker.io/v1.0/docs/target-deployment-configuration#section-docker-registry). When that is the case, igor can then create a poller that will list the registries indexed by clouddriver, check each one for new images and submit events to echo (hence allowing Docker triggers)

igor-monitor-travis/src/main/java/com/netflix/spinnaker/igor/travis/client/logparser/PropertyParser.java

@@ -30,7 +30,8 @@ public class PropertyParser {
 
   private static final Logger log = LoggerFactory.getLogger(PropertyParser.class);
   private static final String MAGIC_SEARCH_STRING = "SPINNAKER_PROPERTY_";
-  private static final Pattern MAGIC_SEARCH_PATTERN = Pattern.compile(MAGIC_SEARCH_STRING);
+  private static final Pattern MAGIC_SEARCH_PATTERN =
+      Pattern.compile("^\\s*" + MAGIC_SEARCH_STRING);
   private static final String MAGIC_JSON_SEARCH_STRING = "SPINNAKER_CONFIG_JSON=";
   private static final Pattern MAGIC_JSON_SEARCH_PATTERN =
       Pattern.compile("^\\s*" + MAGIC_JSON_SEARCH_STRING);

igor-monitor-travis/src/test/groovy/com/netflix/spinnaker/igor/travis/client/logparser/PropertyParserTest.groovy

@@ -72,12 +72,47 @@ class PropertyParserTest extends Specification {
     }
 
     def "Do not detect json magic string if it is not first non-whitespace substring in the line"() {
-        String buildLog = "some log SPINNAKER_CONFIG_JSON={\"key1\":\"value1\"}\r"
+      String buildLog = "some log SPINNAKER_CONFIG_JSON={\"key1\":\"value1\"}\r\n"
+      buildLog += "\u001B[32;1m\$ echo \"SPINNAKER_PROPERTY_HELLO_WORLD=ello world\"\u001B[0;m\n"
 
-        when:
-        Map<String, Object> properties = PropertyParser.extractPropertiesFromLog(buildLog)
+      when:
+      Map<String, Object> properties = PropertyParser.extractPropertiesFromLog(buildLog)
 
-        then:
-        properties.size() == 0
+      then:
+      properties.size() == 0
+    }
+
+    def "Do not ExtractPropertiesFromLog if build logs have script steps before output"() {
+      String testValue = "hello world"
+      String testKey = "HELLO_WORLD"
+      String buildLog = "\$ echo \"SPINNAKER_PROPERTY_${testKey}=i should not appear\"\n"
+      buildLog += "SPINNAKER_PROPERTY_${testKey}=${testValue}"
+
+      when:
+      Map<String, Object> properties = PropertyParser.extractPropertiesFromLog(buildLog)
+
+      then:
+      properties.size() == 1
+      properties.getOrDefault(testKey.toLowerCase(), "").toString() == testValue
+    }
+
+    def "Do not ExtractPropertiesFromLog if key is not at start of file"() {
+      String buildLog = "????SPINNAKER_PROPERTY_HELLO_WORLD=hello world"
+
+      when:
+      Map<String, Object> properties = PropertyParser.extractPropertiesFromLog(buildLog)
+
+      then:
+      properties.size() == 0
+    }
+
+    def "Do ExtractPropertiesFromLog if key is not at start of file but it is only whitespace"() {
+      String buildLog = "     SPINNAKER_PROPERTY_HELLO_WORLD=hello world"
+
+      when:
+      Map<String, Object> properties = PropertyParser.extractPropertiesFromLog(buildLog)
+
+      then:
+      properties.size() == 1
     }
 }

igor-web/src/main/groovy/com/netflix/spinnaker/igor/config/GitlabCiConfig.java

@@ -1,5 +1,6 @@
 /*
  * Copyright 2017 Netflix, Inc.
+ * Copyright 2022 Redbox Entertainment, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -46,7 +47,7 @@ public class GitlabCiConfig {
   private static final Logger log = LoggerFactory.getLogger(GitlabCiConfig.class);
 
   @Bean
-  public Map<String, GitlabCiService> gitlabCiMasters(
+  public Map<String, GitlabCiService> masters(
       BuildServices buildServices,
       final IgorConfigurationProperties igorConfigurationProperties,
       GitlabCiProperties gitlabCiProperties,
@@ -58,7 +59,7 @@ public Map<String, GitlabCiService> gitlabCiMasters(
                 gitlabCiHost ->
                     gitlabCiService(
                         igorConfigurationProperties,
-                        "gitlab-ci-" + gitlabCiHost.getName(),
+                        gitlabCiHost.getName(),
                         gitlabCiHost,
                         objectMapper))
             .collect(Collectors.toMap(GitlabCiService::getName, Function.identity()));
@@ -78,9 +79,7 @@ private static GitlabCiService gitlabCiService(
             igorConfigurationProperties.getClient().getTimeout(),
             objectMapper),
         name,
-        host.getAddress(),
-        host.getLimitByMembership(),
-        host.getLimitByOwnership(),
+        host,
         host.getPermissions().build());
   }
 

igor-web/src/main/groovy/com/netflix/spinnaker/igor/config/GitlabCiProperties.java

@@ -50,9 +50,13 @@ public static class GitlabCiHost implements BuildServerProperties.Host {
     @NotEmpty private String name;
     @NotEmpty private String address;
     private String privateToken;
-    private boolean limitByMembership = false;
-    private boolean limitByOwnership = true;
-    private Integer itemUpperThreshold;
+    private boolean limitByMembership = true;
+    private boolean limitByOwnership = false;
+    private Integer defaultHttpPageLength = 100;
+    private Integer itemUpperThreshold = 1000;
+    private Integer httpRetryMaxAttempts = 5;
+    private Integer httpRetryWaitSeconds = 2;
+    private Boolean httpRetryExponentialBackoff = false;
     private Permissions.Builder permissions = new Permissions.Builder();
 
     public String getName() {
@@ -118,5 +122,43 @@ public Permissions.Builder getPermissions() {
     public void setPermissions(Permissions.Builder permissions) {
       this.permissions = permissions;
     }
+
+    public Integer getHttpRetryWaitSeconds() {
+      return httpRetryWaitSeconds;
+    }
+
+    public void setHttpRetryWaitSeconds(Integer httpRetryWaitSeconds) {
+      this.httpRetryWaitSeconds = httpRetryWaitSeconds;
+    }
+
+    public Integer getHttpRetryMaxAttempts() {
+      return httpRetryMaxAttempts;
+    }
+
+    public void setHttpRetryMaxAttempts(Integer httpRetryMaxAttempts) {
+      this.httpRetryMaxAttempts = httpRetryMaxAttempts;
+    }
+
+    public Boolean getHttpRetryExponentialBackoff() {
+      return httpRetryExponentialBackoff;
+    }
+
+    public void setHttpRetryExponentialBackoff(Boolean httpRetryExponentialBackoff) {
+      this.httpRetryExponentialBackoff = httpRetryExponentialBackoff;
+    }
+
+    public Integer getDefaultHttpPageLength() {
+      return defaultHttpPageLength;
+    }
+
+    public void setDefaultHttpPageLength(Integer defaultHttpPageLength) {
+      if (defaultHttpPageLength == null
+          || defaultHttpPageLength < 1
+          || defaultHttpPageLength > 100) {
+        throw new IllegalArgumentException(
+            "Invalid Gitlab CI config.  defaultHttpPageLength must be a valid number between 1-100");
+      }
+      this.defaultHttpPageLength = defaultHttpPageLength;
+    }
   }
 }