feat: QuarksJob to rotate all secrets

Makefile

@@ -13,11 +13,14 @@ version:
 update-subcharts:
 	@./scripts/update-subcharts.sh
 
-lint: shellcheck yamllint helmlint httplint
+lint: shellcheck yamllint helmlint httplint rubocop
 
 helmlint:
 	@./scripts/helmlint.sh
 
+rubocop:
+	@./scripts/rubocop.sh
+
 shellcheck:
 	@./scripts/shellcheck.sh
 
@@ -26,6 +29,7 @@ yamllint:
 
 test:
 	./tests/config.sh
+	./tests/rspec.sh
 
 .PHONY: httplint
 httplint:

chart/assets/operations/quarks-restart-on-update.yaml

@@ -0,0 +1,54 @@
+{{- /*
+  This ops file sets Quarks-specific annotations on instance groups to ensure
+  they are correctly restarted when their dependent secrets change (so that we
+  can rotate the various generated secrets).
+
+  See: https://github.com/cloudfoundry-incubator/quarks-operator/issues/1136
+*/ -}}
+{{- include "_config.load" $ }}
+
+{{- $instance_groups := list }}
+
+{{- $instance_groups = append $instance_groups "api" }}
+{{- $instance_groups = append $instance_groups "cc-worker" }}
+{{- $instance_groups = append $instance_groups "diego-api" }}
+{{- $instance_groups = append $instance_groups "doppler" }}
+{{- $instance_groups = append $instance_groups "log-api" }}
+{{- $instance_groups = append $instance_groups "log-cache" }}
+{{- $instance_groups = append $instance_groups "nats" }}
+{{- $instance_groups = append $instance_groups "router" }}
+{{- $instance_groups = append $instance_groups "scheduler" }}
+{{- $instance_groups = append $instance_groups "uaa" }}
+
+{{- if .Values.features.autoscaler.enabled }}
+  {{- $instance_groups = append $instance_groups "asactors" }}
+  {{- $instance_groups = append $instance_groups "asapi" }}
+  {{- $instance_groups = append $instance_groups "asmetrics" }}
+  {{- $instance_groups = append $instance_groups "asnozzle" }}
+{{- end }}
+
+{{- if .Values.features.credhub.enabled }}
+  {{- $instance_groups = append $instance_groups "credhub" }}
+{{- end }}
+
+{{- if .Values.features.routing_api.enabled }}
+  {{- $instance_groups = append $instance_groups "routing-api" }}
+  {{- $instance_groups = append $instance_groups "tcp-router" }}
+{{- end }}
+
+{{- if not $.Values.features.eirini.enabled }}
+  {{- $instance_groups = append $instance_groups "auctioneer" }}
+  {{- if not .Values.features.multiple_cluster_mode.control_plane.enabled }}
+    {{- $instance_groups = append $instance_groups "diego-cell" }}
+  {{- end }}
+{{- end }}
+
+{{- if eq .Values.features.blobstore.provider "singleton" }}
+  {{- $instance_groups = append $instance_groups "singleton-blobstore" }}
+{{- end }}
+
+{{- range $instance_groups }}
+- type: replace
+  path: /instance_groups/name={{ . }}/env?/bosh/agent/settings/annotations/quarks.cloudfoundry.org~1restart-on-update
+  value: "true"
+{{- end }}

chart/assets/operations/sequencing.yaml

@@ -28,6 +28,7 @@
 #            --> tcp-router                (feature: routing-api)
 #            --> credhub                   (feature: credhub)
 #            --> diego-cell                (feature: not eirini)
+# apps-dns   --> diego-cell                (feature: not eirini)
 # deigo-api  --> auctioneer                (feature: not eirini)
 # asdatabase --> asactors                  (feature: autoscaler)
 #            --> asapi                     (feature: autoscaler)
@@ -50,7 +51,7 @@
 
 {{- if not .Values.features.eirini.enabled }}
   {{- template "wait-for" list "auctioneer" "diego-api" }}
-  {{- template "wait-for" list "diego-cell" "uaa" }}
+  {{- template "wait-for" list "diego-cell" "uaa" "apps-dns" }}
 {{- end }}
 
 {{- if eq .Values.features.blobstore.provider "singleton" }}

chart/assets/scripts/helm/secret-rotation/rotate-secrets.rb

@@ -0,0 +1,125 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# This script is used to generate a ConfigMap to rotate all secrets in the
+# deployment.
+
+require 'English'
+require 'kubeclient'
+require 'json'
+require 'open3'
+require 'time'
+require 'yaml'
+
+# SecretRotator will rotate all quarks secrets for the current deployment
+class SecretRotator
+  # Path to the Kubernetes API server CA certificate
+  CA_CERT_PATH = '/var/run/secrets/kubernetes.io/serviceaccount/ca.crt'
+
+  # Path to the Kubernetes service account token
+  TOKEN_PATH = '/var/run/secrets/kubernetes.io/serviceaccount/token'
+
+  # HTTP status code when a resource already exists
+  HTTP_STATUS_CONFLICT = 409
+
+  # The namespace the QuarksSecrets is in, and the same to create the ConfigMap
+  def namespace
+    @namespace ||= ENV['NAMESPACE'] || raise('NAMESPACE not set')
+  end
+
+  # The BOSHDeployment name
+  def deployment
+    @deployment ||= ENV['DEPLOYMENT'] || raise('DEPLOYMENT not set')
+  end
+
+  # Authentication options to create a Kubernetes client
+  def auth_options
+    { bearer_token_file: TOKEN_PATH }
+  end
+
+  # SSL options to create a Kubernetes client
+  def ssl_options
+    @ssl_options ||= {}.tap do |options|
+      options[:ca_file] = CA_CERT_PATH if File.exist? CA_CERT_PATH
+    end
+  end
+
+  # Kubernetes client to access default APIs
+  def client
+    @client ||= Kubeclient::Client.new(
+      'https://kubernetes.default.svc',
+      'v1',
+      auth_options: auth_options,
+      ssl_options: ssl_options
+    )
+  end
+
+  # Kubernetes client to access Quarks APIs
+  def quarks_client
+    @quarks_client ||= Kubeclient::Client.new(
+      'https://kubernetes.default.svc/apis/quarks.cloudfoundry.org',
+      'v1alpha1',
+      auth_options: auth_options,
+      ssl_options: ssl_options
+    )
+  end
+
+  # The selector used to find interesting secrets
+  def secret_selector
+    "quarks.cloudfoundry.org/deployment-name=#{deployment}"
+  end
+
+  # The names of the QuarksSecrets to rotate
+  def all_secrets
+    quarks_client
+      .get_quarks_secrets(namespace: namespace, selector: secret_selector)
+      .map { |secret| secret.metadata.name }
+  end
+
+  def excluded_secrets
+    [
+      # Do not rotate the various CC DB encryption related secrets; that has
+      # effects on the data in the databases.  These need to be rotated manually.
+      /^var-ccdb-key-label/,
+      'var-cc-db-encryption-key',
+      # Do not rotate the PXC root password: we can't restart the database pod
+      # afterwards if we do (because the database root password isn't actually
+      # updated).
+      'var-pxc-root-password',
+      # Since we don't restart the PXC container, we can't update its CA cert.
+      'var-pxc-ca'
+    ]
+  end
+
+  def secrets
+    all_secrets.sort.reject do |secret|
+      excluded_secrets.any? { |excluded| excluded === secret } # rubocop:disable Style/CaseEquality
+    end
+  end
+
+  def configmap_name
+    @configmap_name ||= "rotate.all-secrets.#{Time.now.to_i}"
+  end
+
+  # The ConfigMap resource to be created
+  def configmap
+    @configmap ||= Kubeclient::Resource.new(
+      metadata: {
+        namespace: namespace,
+        # Set a unique-ish name to help running this multiple times
+        name: configmap_name,
+        labels: { 'quarks.cloudfoundry.org/secret-rotation': 'true' }
+      },
+      data: { secrets: JSON.pretty_generate(secrets) }
+    )
+  end
+
+  # Trigger rotation of all QuarksSecrets
+  def rotate
+    client.create_config_map configmap
+    puts "Created secret rotation configmap #{configmap_name} with #{secrets.length} secrets:"
+    secrets.each { |secret| puts "    #{secret}" }
+  end
+end
+
+SecretRotator.new.rotate if $PROGRAM_NAME == __FILE__

chart/assets/scripts/helm/secret-rotation/rotate-secrets_spec.rb

@@ -0,0 +1,131 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require_relative 'rotate-secrets'
+
+RSpec.describe(SecretRotator) do
+  def instance
+    @instance ||= described_class.new
+  end
+
+  before :each do
+    # Default to having a good setup
+    allow(ENV).to receive(:[]).with('NAMESPACE').and_return 'namespace'
+    allow(ENV).to receive(:[]).with('DEPLOYMENT').and_return 'deployment'
+    allow(File).to receive(:exist?).with(described_class::CA_CERT_PATH).and_return true
+  end
+
+  describe '#namespace' do
+    it 'returns the namespace' do
+      expect(ENV).to receive(:[]).with('NAMESPACE').and_return 'ns'
+      expect(instance.namespace).to eq 'ns'
+    end
+
+    it 'errors out if namespace is not set' do
+      expect(ENV).to receive(:[]).with('NAMESPACE').and_return nil
+      expect { instance.namespace }.to raise_error(/NAMESPACE not set/)
+    end
+  end
+
+  describe '#deployment' do
+    it 'returns the deployment name' do
+      expect(ENV).to receive(:[]).with('DEPLOYMENT').and_return 'dp'
+      expect(instance.deployment).to eq 'dp'
+    end
+
+    it 'errors out if deployment name is not set' do
+      expect(ENV).to receive(:[]).with('DEPLOYMENT').and_return nil
+      expect { instance.deployment }.to raise_error(/DEPLOYMENT not set/)
+    end
+  end
+
+  describe '#ssl_options' do
+    it 'returns empty options if the CA cert is missing' do
+      allow(File).to receive(:exist?).with(described_class::CA_CERT_PATH).and_return false
+      expect(instance.ssl_options).to be_empty
+    end
+
+    it 'returns options with the CA cert path' do
+      expect(instance.ssl_options).to eq(ca_file: described_class::CA_CERT_PATH)
+    end
+  end
+
+  describe '#client' do
+    it 'creates a Kubernetes client' do
+      expected_client = {}
+      expect(Kubeclient::Client).to receive(:new).with(
+        'https://kubernetes.default.svc',
+        'v1',
+        auth_options: { bearer_token_file: described_class::TOKEN_PATH },
+        ssl_options: { ca_file: described_class::CA_CERT_PATH }
+      ).once.and_return expected_client
+      expect(instance.client).to be expected_client
+
+      # call it again should not ask for a new client
+      expect(instance.client).to be expected_client
+    end
+  end
+
+  describe '#quarks_client' do
+    it 'creates a Kubernetes client' do
+      expected_client = {}
+      expect(Kubeclient::Client).to receive(:new).with(
+        'https://kubernetes.default.svc/apis/quarks.cloudfoundry.org',
+        'v1alpha1',
+        auth_options: { bearer_token_file: described_class::TOKEN_PATH },
+        ssl_options: { ca_file: described_class::CA_CERT_PATH }
+      ).once.and_return expected_client
+      expect(instance.quarks_client).to be expected_client
+
+      # call it again should not ask for a new client
+      expect(instance.quarks_client).to be expected_client
+    end
+  end
+
+  describe '#secrets' do
+    it 'returns the QuarksSecret names' do
+      expected_names = %w[one two three]
+      secrets = expected_names.map do |name|
+        double('QuarksSecret').tap do |secret|
+          expect(secret).to receive_message_chain(:metadata, :name).and_return name
+        end
+      end
+
+      expect(instance)
+        .to receive_message_chain(:quarks_client, :get_quarks_secrets)
+        .with(
+          namespace: 'namespace',
+          selector: 'quarks.cloudfoundry.org/deployment-name=deployment'
+        )
+        .and_return(secrets)
+
+      expect(instance.secrets).to eq expected_names
+    end
+  end
+
+  describe '#configmap' do
+    it 'returns the desired config map' do
+      expect(instance).to receive(:secrets).and_return %w[one two]
+      allow(Time).to receive(:now).and_return 123
+      expect(instance.configmap.to_h).to eq(
+        metadata: {
+          namespace: 'namespace',
+          name: 'rotate.all-secrets-123',
+          labels: { 'quarks.cloudfoundry.org/secret-rotation': 'true' }
+        },
+        data: { secrets: %w[one two].to_json }
+      )
+    end
+  end
+
+  describe '#rotate' do
+    it 'attempts to create a config map' do
+      configmap = {}
+      expect(instance).to receive(:configmap).and_return configmap
+      expect(instance)
+        .to receive_message_chain(:client, :create_config_map)
+        .with(be(configmap))
+      instance.rotate
+    end
+  end
+end

chart/assets/scripts/jobs/pxc/seeder.sh

@@ -42,6 +42,7 @@ mysql < <(
 
       GRANT ALL ON \`${database}\`.* TO '${database}'@'%';
     "
+    echo "  ${database}" >&2 # For status
   done
   echo "COMMIT;"
 )

chart/config/releases.yaml

@@ -48,6 +48,11 @@ releases:
   postgres:
     condition: features.autoscaler.enabled
     version: "39"
+  secret-rotation:
+    # secret-rotation is not a BOSH release.  It requires ruby & kubectl.
+    image:
+      repository: splatform/fissile-stemcell-sle
+      tag: SLE_15_SP1-27.4
   sync-integration-tests:
     # XXX SITS only makes sense when using Diego; add error check somewhere?
     condition: testing.sync_integration_tests.enabled