Merge pull request #18 from omgnetwork/infra_v2

Infra v2

.circleci/config.yml

@@ -10,23 +10,41 @@ jobs:
           command: |
             make docker-build
           no_output_timeout: 2400s
+      - run:
+          name: Save Docker image
+          command: |
+            mkdir -p /tmp/workspace
+            docker save -o /tmp/workspace/omgnetwork_vault-latest.tar omgnetwork/vault:latest
+      - persist_to_workspace:
+          root: /tmp/workspace
+          paths:
+            - "omgnetwork_vault-latest.tar"
 
   publish:
-    docker:
-      - image: google/cloud-sdk
+    machine:
+      image: ubuntu-1604:202004-01
     environment:
       GCR_PREFIX: gcr.io/omisego-development
       IMMUTABILITY_IMAGE: omgnetwork/vault
     steps:
+      - attach_workspace:
+          at: /tmp/workspace
       - checkout
-      - run: |
-          export SEMVER=$(cat ./VERSION)
-          echo $GCLOUD_SERVICE_KEY | gcloud auth activate-service-account --key-file=-
-          gcloud -q auth configure-docker
-          docker tag $IMMUTABILITY_IMAGE:latest $GCR_PREFIX/$IMMUTABILITY_IMAGE:latest
-          docker tag $IMMUTABILITY_IMAGE:latest $GCR_PREFIX/$IMMUTABILITY_IMAGE:$SEMVER
-          docker push $GCR_PREFIX/$IMMUTABILITY_IMAGE:latest
-          docker push $GCR_PREFIX/$IMMUTABILITY_IMAGE:$SEMVER
+      - run:
+          name: Install GCloud SDK
+          command: |
+            echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+            curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+            sudo apt-get update && sudo apt-get install google-cloud-sdk
+      - run:
+          name: Publish to GCR
+          command: |
+            export SEMVER=$(cat ./VERSION)
+            echo $GCLOUD_SERVICE_KEY | gcloud auth activate-service-account --key-file=-
+            gcloud -q auth configure-docker
+            docker load -i /tmp/workspace/omgnetwork_vault-latest.tar
+            docker tag $IMMUTABILITY_IMAGE:latest $GCR_PREFIX/$IMMUTABILITY_IMAGE:$SEMVER
+            docker push $GCR_PREFIX/$IMMUTABILITY_IMAGE:$SEMVER
 
 workflows:
   version: 2

.gitignore

@@ -29,6 +29,9 @@ docker/config/*.csr
 docker/config/*.srl
 *.key.json
 credentials.json
+cluster-keys.json
 *.pem
 *.crt
 *.key
+.srl
+.certs/

infrastructure/README.md

@@ -131,7 +131,7 @@ This script will activate the KMS service account in the `gcloud` tool using the
 
 ### Helm / Deployment
 
-Deploying services to a Kubernetes cluster typically require the use of (helm)[https://helm.sh] to manage the cluster configuration and dependencies. This guide shows how to use the official (Hashicorp)[https://www.hashicorp.com] _helm chart_ to deploy a Vault cluster.
+Deploying services to a Kubernetes cluster typically require the use of [helm](https://helm.sh) to manage the cluster configuration and dependencies. This guide shows how to use the official [Hashicorp](https://www.hashicorp.com) _helm chart_ to deploy a Vault cluster.
 
 #### Installing Helm and Supporting Tools
 
@@ -141,17 +141,34 @@ If you are running on MacOS, you can install helm by executing:
 brew install helm yq
 ```
 
-If you are running on Linux or Windows, see the (Helm Download Page)[https://github.com/helm/helm/releases/latest]. You'll also want to install the yq utility.
+If you are running on Linux or Windows, see the [Helm Download Page](https://github.com/helm/helm/releases/latest). You'll also want to install the yq utility.
 
-#### Establish hashicorp registry
+#### Establish Remote Registries
 
-In order to use the official Hashicorp helm chart, we need to add it to the local helm registry by executing:
+In order to use the official Hashicorp and Datadog Helm repositories, we need to add it to the local helm registry by executing:
 
 ```bash
-helm repo add hashicorp http://helm.releases.hashicorp.com
+helm repo add hashicorp https://helm.releases.hashicorp.com
+helm repo add datadog https://helm.datadoghq.com
+helm repo add stable https://kubernetes-charts.storage.googleapis.com
+helm repo update
 ```
 
-#### Generate Self-Signed Certs
+#### Datadog
+
+In the [Datadog overrides file](./k8s/datadog-overrides.yaml), insert your Datadog API and App key into the YAML file at `.datadog.apiKey` and `.datadog.appKey` respectively.
+
+From the `infrastructure` folder, execute the following command to deploy the Datadog Helm chart:
+
+```sh
+helm upgrade --atomic --cleanup-on-fail --install --values ./k8s/datadog-overrides.yaml datadog datadog/datadog
+```
+
+With the existing overrides (in addition to your API and app keys), this Helm chart instantiates a DaemonSet for the Datadog agent pods that will be responsible for collecting and forwarding Vault server and audit logs into your Datadog dashboards.
+
+#### Vault
+
+##### Generate Self-Signed Certs
 
 In `./infrastructure`, execute:
 
@@ -170,17 +187,25 @@ For Minikube, use: `-d vault-internal`
 
 ---
 
-#### Create a kubernetes secret with the cert
+##### Create a kubernetes secret with the cert
 
 The `gen-certs.sh` script updates `k8s/vault-overrides.yaml` with the name of the secret that was generated with the new certs material. To see the created secret, execute:
 
 ```bash
-kubectl list secrets
+kubectl get secrets
 ```
 
 and look for "omgnetwork-certs-"
 
-#### Update value overrides
+##### Generate Storage Classes
+
+In `infrastructure`, execute:
+
+```bash
+./scripts/gen_storage.sh
+```
+
+##### Update Value Overrides
 
 In `infrastructure`, execute:
 
@@ -196,7 +221,10 @@ $GCP_PROJECT
 $GKE_CLUSTER_NAME
 ```
 
-#### Start the Pods using the Helm Chart
+<<<<<<< HEAD
+##### Start the Pods using the Helm Chart
+=======
+>>>>>>> 1526b2980e828e9057bfe4cbaf0a629887648fc5
 
 Execute:
 
@@ -221,20 +249,12 @@ Before you initialize vault, you'll see errors like this:
 2020-08-21T03:26:41.684Z [WARN]  failed to unseal core: error="fetching stored unseal keys failed: failed to decrypt encrypted stored keys: failed to decrypt envelope: rpc error: code = InvalidArgument desc = Decryption failed: verify that 'name' refers to the correct CryptoKey."
 ```
 
-#### Set up k8s Port Forwarding
-
-In one terminal, execute:
-
-```bash
-kubectl port-forward vault-0 8200:8200
-```
-
 #### Access Vault using the CLI
 
 In another terminal, execute:
 
 ```bash
-export VAULT_ADDR=https://localhost:8200
+export VAULT_ADDR=https://<load-balancer>:8200
 export VAULT_CACERT=$K8S/certs/ca-chain.cert.pem
 
 vault status
@@ -255,6 +275,39 @@ vault status
 vault audit enable file file_path=/vault/audit/audit.log
 ```
 
+#### Backup Vault RAFT Data to a Snapshot File
+
+Determining how many backup files you want to keep is a business decision. There are different strategies for maintaining a set of backup snapshots that can be employed.
+
+*Time-based strategy*. The snapshot filename is derived from the formatted timestamp. In this strategy, you'll have to determine how many snapshots to maintain and how to rotate them out when they're no longer appropriate.
+
+```bash
+vault operator raft snapshot save snapshot-$(date +%Y%m%d-%H%M%S).raft
+```
+
+*Rotational strategy*. In this example, a maximum of 5 snapshots are maintained at any given time.
+
+```bash
+rm -f snapshot-4.raft
+
+for i in 3 2 1; do
+  let NEXT=$i+1
+  mv -f snapshot-${i}.raft snapshot-${NEXT}.raft 2> /dev/null
+done
+
+mv -f snapshot.raft snapshot-1.raft 2> /dev/null
+
+vault operator raft snapshot save snapshot.raft
+```
+
+#### Restore Vault RAFT Data from a Snapshot File
+
+When you need to restore your Vault cluster back to a known-good state, identify the snapshot-file you want to restore and execute this command:
+
+```bash
+vault operator raft snapshot restore snapshot-file.raft
+```
+
 ### Uninstalling Vault
 
 When you're done, you can uninstall vault.

infrastructure/k8s/certs/kustomization.yaml

@@ -1,6 +1,6 @@
 secretGenerator:
-- name: omgnetwork-certs
-  files:
-  - ca.crt
-  - vault.crt
-  - vault.key
+  - name: omgnetwork-certs
+    files:
+      - ca.crt
+      - vault.crt
+      - vault.key

infrastructure/k8s/datadog-overrides.yaml

@@ -0,0 +1,13 @@
+datadog:
+  apiKey: ""
+  appKey: ""
+
+  kubeStateMetricsEnabled: false
+
+  logs:
+    enabled: true
+    containerCollectAll: true
+
+  env:
+    - name: DD_CONTAINER_EXCLUDE
+      value: "image:datadog/cluster-agent"

infrastructure/k8s/storage/storage-class-audit.yaml

@@ -0,0 +1,11 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: vault-audit
+provisioner: kubernetes.io/gce-pd
+parameters:
+  type: pd-standard
+  fstype: ext4
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true

infrastructure/k8s/storage/storage-class-data.yaml

@@ -0,0 +1,12 @@
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: vault-data
+provisioner: kubernetes.io/gce-pd
+parameters:
+  type: pd-ssd
+  fstype: ext4
+  replication-type: regional-pd
+reclaimPolicy: Delete
+volumeBindingMode: WaitForFirstConsumer
+allowVolumeExpansion: true

infrastructure/k8s/vault-overrides.yaml

@@ -1,14 +1,24 @@
 global:
   tlsDisable: false
-  certSecretName: omgnetwork-certs-k7hc7k9k42
+  certSecretName: omgnetwork-certs-d67f2d4m49
 injector:
   enabled: false
 server:
+  dataStorage:
+    enabled: true
+    size: 200Gi
+    storageClassName: vault-data
+    accessMode: ReadWriteOnce
   auditStorage:
     enabled: true
-    size: 10Gi
-    storageClass: default
+    size: 100Gi
+    storageClass: vault-audit
     accessMode: ReadWriteOnce
+  service:
+    enabled: true
+    type: LoadBalancer
+    annotations:
+      cloud.google.com/load-balancer-type: Internal
   ha:
     enabled: true
     replicas: 5
@@ -30,8 +40,8 @@ server:
         }
 
         seal "gcpckms" {
-            region      = "us-west4"
-            project     = "vault-cluster-0829"
+            region      = "us-central1"
+            project     = "omgnetwork-test-cluster"
             key_ring    = "omgnetwork-vault-keyring"
             crypto_key  = "omgnetwork-vault-unseal-key"
         }
@@ -76,21 +86,28 @@ server:
         }
   image:
     repository: vault # TODO: will change to the omgnetwork/vault image
-    tag: "1.5.2" # TODO: will change to the appropriate tag for omgnetwork/vault
+    tag: "1.5.3" # TODO: will change to the appropriate tag for omgnetwork/vault
     pullPolicy: IfNotPresent
   extraEnvironmentVars:
-    GOOGLE_REGION: us-west4
-    GOOGLE_PROJECT: vault-cluster-0829
+    GOOGLE_REGION: us-central1
+    GOOGLE_PROJECT: omgnetwork-test-cluster
     GOOGLE_APPLICATION_CREDENTIALS: /vault/userconfig/kms-creds/kms_account.key.json
   extraVolumes:
     - type: secret
       name: "kms-creds"
     - type: secret
-      name: omgnetwork-certs-k7hc7k9k42
+      name: omgnetwork-certs-d67f2d4m49
   affinity: null
   dev:
     enabled: false
   standalone:
     enabled: false
+  resources:
+    requests:
+      memory: 256Mi
+      cpu: 250m
+    limits:
+      memory: 256Mi
+      cpu: 250m
 ui:
   enabled: true

infrastructure/scripts/gcp_services.sh

@@ -6,7 +6,6 @@ SERVICES=("compute.googleapis.com" "cloudkms.googleapis.com" "containerregistry.
 SECONDARY=("container.googleapis.com")
 
 MODE="enable"
-GCP_PROJECT=""
 
 while getopts "hd" opt; do
   case "$opt" in

infrastructure/scripts/gen_certs.sh

@@ -215,9 +215,3 @@ echo ""
 echo "> In your shell, execute this command:"
 echo "export VAULT_CACERT=$(PWD)/k8s/certs/ca.crt"
 echo ""
-echo "> If you are going to port-forward from your localhost, execute:"
-echo "export VAULT_ADDR=https://localhost:8200"
-echo ""
-echo "> If you have DNS set up to route to your EKS cluster, execute:"
-echo "export VAULT_ADDR=https://vault-0.${DOMAIN}:8200"
-echo ""