feat(helm): Add Helm repo polling and caching (#829)

- Helm polling/caching/triggering
- Models and etc. for Helm repository information

Co-authored-by: Gal Yardeni <55253849+gal-yardeni@users.noreply.github.com>

igor-web/igor-web.gradle

@@ -36,7 +36,8 @@ dependencies {
     implementation "com.sun.xml.bind:jaxb-core:2.3.0.1"
     implementation "com.sun.xml.bind:jaxb-impl:2.3.2"
 
-    implementation "com.vdurmont:semver4j:3.1.0"
+    implementation "com.vdurmont:semver4j"
+    implementation "commons-io:commons-io"
 
     testImplementation "org.springframework.boot:spring-boot-starter-test"
     testImplementation "org.spockframework:spock-core"

igor-web/src/main/java/com/netflix/spinnaker/igor/config/HelmConfig.java

@@ -0,0 +1,95 @@
+/*
+ * Copyright 2020 Apple, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.igor.config;
+
+import com.amazonaws.util.IOUtils;
+import com.google.gson.Gson;
+import com.netflix.spinnaker.config.OkHttpClientConfiguration;
+import com.netflix.spinnaker.igor.IgorConfigurationProperties;
+import com.netflix.spinnaker.igor.helm.accounts.HelmAccounts;
+import com.netflix.spinnaker.igor.helm.accounts.HelmAccountsService;
+import com.netflix.spinnaker.retrofit.Slf4jRetrofitLogger;
+import java.io.IOException;
+import java.lang.reflect.Type;
+import lombok.extern.slf4j.Slf4j;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.util.StringUtils;
+import retrofit.Endpoints;
+import retrofit.RestAdapter;
+import retrofit.client.OkClient;
+import retrofit.converter.ConversionException;
+import retrofit.converter.Converter;
+import retrofit.converter.GsonConverter;
+import retrofit.mime.TypedInput;
+import retrofit.mime.TypedOutput;
+
+@Configuration
+@ConditionalOnProperty("helm.enabled")
+@Slf4j
+public class HelmConfig {
+  @Bean
+  HelmAccounts helmAccounts() {
+    return new HelmAccounts();
+  }
+
+  // Custom converter to deal with index file raw string responses
+  class StringConverter implements Converter {
+    private GsonConverter gson = new GsonConverter(new Gson());
+
+    @Override
+    public Object fromBody(TypedInput body, Type type) throws ConversionException {
+      // If the return type is a String, provide it as such
+      if (type.getTypeName().equals("java.lang.String")) {
+        try {
+          return IOUtils.toString(body.in());
+        } catch (IOException e) {
+          throw new ConversionException("Cannot convert response to string");
+        }
+      } else {
+        return gson.fromBody(body, type);
+      }
+    }
+
+    @Override
+    public TypedOutput toBody(Object object) {
+      return gson.toBody(object);
+    }
+  }
+
+  @Bean
+  HelmAccountsService helmAccountsService(
+      OkHttpClientConfiguration okHttpClientConfig,
+      IgorConfigurationProperties igorConfigurationProperties) {
+    String address = igorConfigurationProperties.getServices().getClouddriver().getBaseUrl();
+
+    if (StringUtils.isEmpty(address)) {
+      log.warn(
+          "No Clouddriver URL is configured - Igor will be unable to fetch Helm charts and repository indexes");
+    }
+
+    return new RestAdapter.Builder()
+        .setEndpoint(Endpoints.newFixedEndpoint(address))
+        .setClient(new OkClient(okHttpClientConfig.create()))
+        .setConverter(new StringConverter())
+        .setLogLevel(RestAdapter.LogLevel.BASIC)
+        .setLog(new Slf4jRetrofitLogger(HelmAccountsService.class))
+        .build()
+        .create(HelmAccountsService.class);
+  }
+}

igor-web/src/main/java/com/netflix/spinnaker/igor/helm/HelmMonitor.java

@@ -0,0 +1,207 @@
+/*
+ * Copyright 2020 Apple, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.igor.helm;
+
+import static net.logstash.logback.argument.StructuredArguments.kv;
+
+import com.netflix.spectator.api.Registry;
+import com.netflix.spinnaker.igor.IgorConfigurationProperties;
+import com.netflix.spinnaker.igor.build.model.GenericArtifact;
+import com.netflix.spinnaker.igor.helm.accounts.HelmAccount;
+import com.netflix.spinnaker.igor.helm.accounts.HelmAccounts;
+import com.netflix.spinnaker.igor.helm.cache.HelmCache;
+import com.netflix.spinnaker.igor.helm.model.HelmIndex;
+import com.netflix.spinnaker.igor.history.EchoService;
+import com.netflix.spinnaker.igor.history.model.HelmEvent;
+import com.netflix.spinnaker.igor.polling.*;
+import com.netflix.spinnaker.kork.discovery.DiscoveryStatusListener;
+import com.netflix.spinnaker.kork.dynamicconfig.DynamicConfigService;
+import com.netflix.spinnaker.kork.exceptions.ConstraintViolationException;
+import com.netflix.spinnaker.security.AuthenticatedRequest;
+import java.util.*;
+import java.util.stream.Collectors;
+import lombok.AllArgsConstructor;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.scheduling.TaskScheduler;
+import org.springframework.stereotype.Service;
+import org.springframework.util.StringUtils;
+
+@Service
+@ConditionalOnProperty("helm.enabled")
+public class HelmMonitor
+    extends CommonPollingMonitor<HelmMonitor.HelmDelta, HelmMonitor.HelmPollingDelta> {
+  private final HelmCache cache;
+  private final HelmAccounts helmAccounts;
+  private final Optional<EchoService> echoService;
+
+  @Autowired
+  public HelmMonitor(
+      IgorConfigurationProperties properties,
+      Registry registry,
+      DynamicConfigService dynamicConfigService,
+      DiscoveryStatusListener discoveryStatusListener,
+      Optional<LockService> lockService,
+      HelmCache cache,
+      HelmAccounts helmAccounts,
+      Optional<EchoService> echoService,
+      TaskScheduler scheduler) {
+    super(
+        properties,
+        registry,
+        dynamicConfigService,
+        discoveryStatusListener,
+        lockService,
+        scheduler);
+    this.cache = cache;
+    this.helmAccounts = helmAccounts;
+    this.echoService = echoService;
+  }
+
+  @Override
+  public void poll(boolean sendEvents) {
+    helmAccounts.updateAccounts();
+    helmAccounts.accounts.forEach(
+        account -> pollSingle(new PollContext(account.name, account.toMap(), !sendEvents)));
+  }
+
+  @Override
+  public PollContext getPollContext(String partition) {
+    Optional<HelmAccount> account =
+        helmAccounts.accounts.stream().filter(it -> it.name.equals(partition)).findFirst();
+    if (!account.isPresent()) {
+      throw new ConstraintViolationException(
+          String.format("Cannot find Helm account named '%s'", partition));
+    }
+    return new PollContext(account.get().name, account.get().toMap());
+  }
+
+  @Override
+  protected HelmPollingDelta generateDelta(PollContext ctx) {
+    final String account = ctx.partitionName;
+    log.info("Checking for new Helm Charts {}", kv("account", account));
+
+    List<HelmDelta> deltas = new ArrayList<>();
+
+    HelmIndex index = helmAccounts.getIndex(account);
+    if (index == null) {
+      log.error("Failed to fetch Helm index {}", kv("account", account));
+    } else {
+      Set<String> cachedCharts = cache.getChartDigests(account);
+
+      // If we have no cache at all, do not fire any trigger events
+      // This is so we don't trigger every chart version if Redis dies
+      boolean eventable = !cachedCharts.isEmpty();
+
+      index.entries.forEach(
+          (key, charts) ->
+              charts.forEach(
+                  chartEntry -> {
+                    if (!StringUtils.isEmpty(chartEntry.digest)
+                        && !cachedCharts.contains(chartEntry.digest)) {
+                      deltas.add(
+                          new HelmDelta(
+                              chartEntry.name, chartEntry.version, chartEntry.digest, eventable));
+                    }
+                  }));
+    }
+
+    if (!deltas.isEmpty()) {
+      List<HelmDelta> eventableDeltas =
+          deltas.stream().filter(it -> it.eventable).collect(Collectors.toList());
+      log.info(
+          "Found Helm charts: {} {} {}",
+          kv("account", account),
+          kv("total", deltas.size()),
+          kv("eventable", eventableDeltas.size()));
+    }
+
+    return new HelmPollingDelta(account, deltas);
+  }
+
+  @Override
+  protected void commitDelta(HelmPollingDelta deltas, boolean sendEvents) {
+    List<String> digests = deltas.items.stream().map(i -> i.digest).collect(Collectors.toList());
+
+    // Cache results
+    cache.cacheChartDigests(deltas.account, digests);
+
+    // Send events, if needed
+    deltas.items.forEach(
+        item -> {
+          if (sendEvents && item.eventable) {
+            sendEvent(deltas.account, item);
+          }
+        });
+
+    log.info(
+        "Last Helm poll took {} ms {}",
+        System.currentTimeMillis() - deltas.startTime,
+        kv("account", deltas.account));
+  }
+
+  @Override
+  public String getName() {
+    return "helmMonitor";
+  }
+
+  private void sendEvent(String account, HelmDelta delta) {
+    if (!echoService.isPresent()) {
+      log.warn("Cannot send Helm notification: Echo is not enabled");
+      registry
+          .counter(missedNotificationId.withTag("monitor", getClass().getSimpleName()))
+          .increment();
+      return;
+    }
+
+    log.info(
+        "Sending trigger event for {}:{} {}", delta.name, delta.version, kv("account", account));
+    GenericArtifact helmArtifact =
+        new GenericArtifact("helm/chart", delta.name, delta.version, account);
+
+    HelmEvent.Content helmContent =
+        new HelmEvent.Content(account, delta.name, delta.version, delta.digest);
+
+    AuthenticatedRequest.allowAnonymous(
+        () -> echoService.get().postEvent(new HelmEvent(helmContent, helmArtifact)));
+  }
+
+  @AllArgsConstructor
+  protected static class HelmDelta implements DeltaItem {
+    final String name;
+    final String version;
+    final String digest;
+    final boolean eventable;
+  }
+
+  protected static class HelmPollingDelta implements PollingDelta<HelmDelta> {
+    final String account;
+    final List<HelmDelta> items;
+    final Long startTime;
+
+    public HelmPollingDelta(String account, List<HelmDelta> items) {
+      this.account = account;
+      this.items = items;
+      startTime = System.currentTimeMillis();
+    }
+
+    @Override
+    public List<HelmDelta> getItems() {
+      return items;
+    }
+  }
+}

igor-web/src/main/java/com/netflix/spinnaker/igor/helm/accounts/ArtifactAccount.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2020 Apple, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.igor.helm.accounts;
+
+import java.util.List;
+
+public class ArtifactAccount {
+  public String name;
+  public List<String> types;
+}

igor-web/src/main/java/com/netflix/spinnaker/igor/helm/accounts/HelmAccount.java

@@ -0,0 +1,35 @@
+/*
+ * Copyright 2020 Apple, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.netflix.spinnaker.igor.helm.accounts;
+
+import com.google.common.collect.ImmutableMap;
+import java.util.Map;
+
+public class HelmAccount {
+  public String cloudProvider = "helm";
+  public String name;
+
+  public HelmAccount(String name) {
+    this.name = name;
+  }
+
+  public Map<String, Object> toMap() {
+    return ImmutableMap.of(
+        "cloudProvider", cloudProvider,
+        "name", name);
+  }
+}