From 11e5c9b59ed67d9c780f594569c55069be58678c Mon Sep 17 00:00:00 2001 From: polatengin Date: Fri, 1 May 2020 15:33:18 +0300 Subject: [PATCH 01/23] initial commit for deploying feast to AKS --- .github/workflows/deploy-to-aks.yaml | 17 ++ act/docker-build-push-action@v1 | 1 + .../.helmsman-tmp/tmp326240254/feast.yaml | 165 ++++++++++++++++++ .../tmp553497568/feast-kafka.yaml | 14 ++ .../tmp622116287/feast-zookeeper.yaml | 10 ++ .../tmp717124421/feast-redis-jobstore.yaml | 40 +++++ .../chart-values/feast-kafka.yaml | 14 ++ .../chart-values/feast-redis-jobstore.yaml | 40 +++++ .../chart-values/feast-zookeeper.yaml | 10 ++ infra/desired-state/chart-values/feast.yaml | 165 ++++++++++++++++++ infra/desired-state/desired-state.yaml | 49 ++++++ infra/scripts/deploy-to-aks.sh | 36 ++++ 12 files changed, 561 insertions(+) create mode 100644 .github/workflows/deploy-to-aks.yaml create mode 160000 act/docker-build-push-action@v1 create mode 100644 infra/desired-state/.helmsman-tmp/tmp326240254/feast.yaml create mode 100644 infra/desired-state/.helmsman-tmp/tmp553497568/feast-kafka.yaml create mode 100644 infra/desired-state/.helmsman-tmp/tmp622116287/feast-zookeeper.yaml create mode 100644 infra/desired-state/.helmsman-tmp/tmp717124421/feast-redis-jobstore.yaml create mode 100644 infra/desired-state/chart-values/feast-kafka.yaml create mode 100644 infra/desired-state/chart-values/feast-redis-jobstore.yaml create mode 100644 infra/desired-state/chart-values/feast-zookeeper.yaml create mode 100644 infra/desired-state/chart-values/feast.yaml create mode 100644 infra/desired-state/desired-state.yaml create mode 100644 infra/scripts/deploy-to-aks.sh diff --git a/.github/workflows/deploy-to-aks.yaml b/.github/workflows/deploy-to-aks.yaml new file mode 100644 index 0000000000..4a4a055d6f --- /dev/null +++ b/.github/workflows/deploy-to-aks.yaml @@ -0,0 +1,17 @@ +name: deploy-to-aks + +on: push + +jobs: + deploy: + runs-on: ubuntu-latest + name: deploy + steps: + - uses: actions/checkout@v1 + + - uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: deploy + run: ./infra/scripts/deploy-to-aks.sh diff --git a/act/docker-build-push-action@v1 b/act/docker-build-push-action@v1 new file mode 160000 index 0000000000..e5e36c4a4e --- /dev/null +++ b/act/docker-build-push-action@v1 @@ -0,0 +1 @@ +Subproject commit e5e36c4a4e2085c523c6fb6a553b8d00c30c1716 diff --git a/infra/desired-state/.helmsman-tmp/tmp326240254/feast.yaml b/infra/desired-state/.helmsman-tmp/tmp326240254/feast.yaml new file mode 100644 index 0000000000..608706a1c1 --- /dev/null +++ b/infra/desired-state/.helmsman-tmp/tmp326240254/feast.yaml @@ -0,0 +1,165 @@ +# Feast deployment installs the following components: +# - Feast Core +# - Feast Serving Online +# - Feast Serving Batch +# - Prometheus StatsD Exporter +# +# The configuration for different components can be referenced from: +# - charts/feast-core/values.yaml +# - charts/feast-serving/values.yaml +# - charts/prometheus-statsd-exporter/values.yaml +# +# Note that "feast-serving-online" and "feast-serving-batch" are +# aliases to "feast-serving" chart since in typical scenario two instances +# of Feast Serving: online and batch will be deployed. Both described +# using the same chart "feast-serving". +# +# Note that the import job by default uses DirectRunner +# https://beam.apache.org/documentation/runners/direct/ +# in this configuration since it allows Feast to run in more environments +# (unlike DataflowRunner which requires Google Cloud services). +# +# A secret containing Google Cloud service account JSON key is required +# in this configuration. +# https://cloud.google.com/iam/docs/creating-managing-service-accounts +# +# The Google Cloud service account must have the following roles: +# - bigquery.dataEditor +# - bigquery.jobUser +# +# Assuming a service account JSON key file has been downloaded to +# (please name the file key.json): +# /home/user/key.json +# +# Run the following command to create the secret in your Kubernetes cluster: +# +# kubectl create secret generic feast-gcp-service-account \ +# --from-file=/home/user/key.json +# +# ============================================================ +# Feast Core +# ============================================================ +kafka: + enabled: false +postgresql: + enabled: true + +feast-core: + # If enabled specifies whether to install Feast Core component. + # + # Normally, this is set to "false" when Feast users need access to low latency + # Feast Serving, by deploying multiple instances of Feast Serving closest + # to the client. These instances of Feast Serving however can still use + # the same shared Feast Core. + enabled: true + # Specify which image tag to use. Keep this consistent for all components + image: + tag: "0.4.4" + # jvmOptions are options that will be passed to the Java Virtual Machine (JVM) + # running Feast Core. + # + # resources that should be allocated to Feast Core. + resources: + limits: + cpu: 2 + memory: 2G + requests: + cpu: 100m + memory: 500Mi + gcpProjectId: dev-konnekt-data-deep-1 + # gcpServiceAccount is the Google service account that Feast Core will use. + gcpServiceAccount: + # useExistingSecret specifies Feast to use an existing secret containing + # Google Cloud service account JSON key file. + # + # This is the only supported option for now to use a service account JSON. + # Feast admin is expected to create this secret before deploying Feast. + useExistingSecret: true + existingSecret: + # name is the secret name of the existing secret for the service account. + name: feast-gcp-service-account + # key is the secret key of the existing secret for the service account. + # key is normally derived from the file name of the JSON key file. + key: dev-konnekt-data-deep-1_feast-server-azure-cluster.json + # Setting service.type to NodePort exposes feast-core service at a static port + service: + type: ClusterIP + http: + nodePort: false + grpc: + nodePort: false + # Make kafka externally accessible using NodePort + kafka: + enabled: + false + postgresql: + enabled: true + application.yaml: + feast: + stream: + options: + # Point to one of your Kafka brokers + bootstrapServers: feast-kafka:9092 +# ============================================================ +# Feast Serving Online +# ============================================================ +feast-serving-online: + enabled: false +# ============================================================ +# Feast Serving Batch +# ============================================================ +feast-serving-batch: + enabled: true + redis: + enabled: false + image: + tag: "0.4.4" + service: + type: ClusterIP + http: + nodePort: false + grpc: + nodePort: false + ingress: + http: + enabled: false + grpc: + enabled: false + resources: + limits: + cpu: 1 + memory: 2G + requests: + cpu: 50m + memory: 256M + livenessProbe: + enabled: false + readinessProbe: + enabled: false + gcpServiceAccount: + useExistingSecret: true + existingSecret: + name: feast-gcp-service-account + key: dev-konnekt-data-deep-1_feast-server-azure-cluster.json + gcpProjectId: dev-konnekt-data-deep-1 + logLevel: info + application.yaml: + feast: + jobs: + staging-location: gs://dev-konnekt-data-deep-1-dataflow-workspace/batch-staging/ + bigquery-initial-retry-delay-secs: 1 + bigquery-total-timeout-secs: 21600 + store-type: REDIS + store-options: + host: feast-redis-jobstore-master + port: 6379 + store.yaml: + name: bigquery + type: BIGQUERY + bigquery_config: + project_id: dev-konnekt-data-deep-1 + dataset_id: feast_warehouse + subscriptions: + - name: "*" + project: "*" + version: "*" diff --git a/infra/desired-state/.helmsman-tmp/tmp553497568/feast-kafka.yaml b/infra/desired-state/.helmsman-tmp/tmp553497568/feast-kafka.yaml new file mode 100644 index 0000000000..bfe681c14f --- /dev/null +++ b/infra/desired-state/.helmsman-tmp/tmp553497568/feast-kafka.yaml @@ -0,0 +1,14 @@ +heapOpts: -Xmx1024m -Xms1024m +persistence: + enabled: false +resources: + limits: + cpu: 1200m + memory: 1280Mi + requests: + cpu: 50m + memory: 1024Mi +zookeeper: + enabled: false +externalZookeeper: + servers: feast-zookeeper:2181 diff --git a/infra/desired-state/.helmsman-tmp/tmp622116287/feast-zookeeper.yaml b/infra/desired-state/.helmsman-tmp/tmp622116287/feast-zookeeper.yaml new file mode 100644 index 0000000000..d4409e400d --- /dev/null +++ b/infra/desired-state/.helmsman-tmp/tmp622116287/feast-zookeeper.yaml @@ -0,0 +1,10 @@ +heapSize: 256 +persistence: + enabled: false +resources: + limits: + cpu: 100m + memory: 512Mi + requests: + cpu: 10m + memory: 256Mi diff --git a/infra/desired-state/.helmsman-tmp/tmp717124421/feast-redis-jobstore.yaml b/infra/desired-state/.helmsman-tmp/tmp717124421/feast-redis-jobstore.yaml new file mode 100644 index 0000000000..5b4f228e56 --- /dev/null +++ b/infra/desired-state/.helmsman-tmp/tmp717124421/feast-redis-jobstore.yaml @@ -0,0 +1,40 @@ +cluster: + enabled: false +sentinel: + enabled: false +usePassword: false +master: + resources: + limits: + cpu: 2 + memory: 300Mi + requests: + cpu: 100m + memory: 220Mi + persistence: + enabled: false +metrics: + enabled: true + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 25m + memory: 32Mi +sysctlImage: + enabled: true + mountHostSys: true + command: + - /bin/sh + - -c + - |- + echo never > /host-sys/kernel/mm/transparent_hugepage/enabled +configmap: |- + save "" + appendonly yes + appendfsync everysec + dir /data + appendfilename "appendonly.aof" + maxmemory 200mb + maxmemory-policy noeviction diff --git a/infra/desired-state/chart-values/feast-kafka.yaml b/infra/desired-state/chart-values/feast-kafka.yaml new file mode 100644 index 0000000000..bfe681c14f --- /dev/null +++ b/infra/desired-state/chart-values/feast-kafka.yaml @@ -0,0 +1,14 @@ +heapOpts: -Xmx1024m -Xms1024m +persistence: + enabled: false +resources: + limits: + cpu: 1200m + memory: 1280Mi + requests: + cpu: 50m + memory: 1024Mi +zookeeper: + enabled: false +externalZookeeper: + servers: feast-zookeeper:2181 diff --git a/infra/desired-state/chart-values/feast-redis-jobstore.yaml b/infra/desired-state/chart-values/feast-redis-jobstore.yaml new file mode 100644 index 0000000000..5b4f228e56 --- /dev/null +++ b/infra/desired-state/chart-values/feast-redis-jobstore.yaml @@ -0,0 +1,40 @@ +cluster: + enabled: false +sentinel: + enabled: false +usePassword: false +master: + resources: + limits: + cpu: 2 + memory: 300Mi + requests: + cpu: 100m + memory: 220Mi + persistence: + enabled: false +metrics: + enabled: true + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 25m + memory: 32Mi +sysctlImage: + enabled: true + mountHostSys: true + command: + - /bin/sh + - -c + - |- + echo never > /host-sys/kernel/mm/transparent_hugepage/enabled +configmap: |- + save "" + appendonly yes + appendfsync everysec + dir /data + appendfilename "appendonly.aof" + maxmemory 200mb + maxmemory-policy noeviction diff --git a/infra/desired-state/chart-values/feast-zookeeper.yaml b/infra/desired-state/chart-values/feast-zookeeper.yaml new file mode 100644 index 0000000000..d4409e400d --- /dev/null +++ b/infra/desired-state/chart-values/feast-zookeeper.yaml @@ -0,0 +1,10 @@ +heapSize: 256 +persistence: + enabled: false +resources: + limits: + cpu: 100m + memory: 512Mi + requests: + cpu: 10m + memory: 256Mi diff --git a/infra/desired-state/chart-values/feast.yaml b/infra/desired-state/chart-values/feast.yaml new file mode 100644 index 0000000000..608706a1c1 --- /dev/null +++ b/infra/desired-state/chart-values/feast.yaml @@ -0,0 +1,165 @@ +# Feast deployment installs the following components: +# - Feast Core +# - Feast Serving Online +# - Feast Serving Batch +# - Prometheus StatsD Exporter +# +# The configuration for different components can be referenced from: +# - charts/feast-core/values.yaml +# - charts/feast-serving/values.yaml +# - charts/prometheus-statsd-exporter/values.yaml +# +# Note that "feast-serving-online" and "feast-serving-batch" are +# aliases to "feast-serving" chart since in typical scenario two instances +# of Feast Serving: online and batch will be deployed. Both described +# using the same chart "feast-serving". +# +# Note that the import job by default uses DirectRunner +# https://beam.apache.org/documentation/runners/direct/ +# in this configuration since it allows Feast to run in more environments +# (unlike DataflowRunner which requires Google Cloud services). +# +# A secret containing Google Cloud service account JSON key is required +# in this configuration. +# https://cloud.google.com/iam/docs/creating-managing-service-accounts +# +# The Google Cloud service account must have the following roles: +# - bigquery.dataEditor +# - bigquery.jobUser +# +# Assuming a service account JSON key file has been downloaded to +# (please name the file key.json): +# /home/user/key.json +# +# Run the following command to create the secret in your Kubernetes cluster: +# +# kubectl create secret generic feast-gcp-service-account \ +# --from-file=/home/user/key.json +# +# ============================================================ +# Feast Core +# ============================================================ +kafka: + enabled: false +postgresql: + enabled: true + +feast-core: + # If enabled specifies whether to install Feast Core component. + # + # Normally, this is set to "false" when Feast users need access to low latency + # Feast Serving, by deploying multiple instances of Feast Serving closest + # to the client. These instances of Feast Serving however can still use + # the same shared Feast Core. + enabled: true + # Specify which image tag to use. Keep this consistent for all components + image: + tag: "0.4.4" + # jvmOptions are options that will be passed to the Java Virtual Machine (JVM) + # running Feast Core. + # + # resources that should be allocated to Feast Core. + resources: + limits: + cpu: 2 + memory: 2G + requests: + cpu: 100m + memory: 500Mi + gcpProjectId: dev-konnekt-data-deep-1 + # gcpServiceAccount is the Google service account that Feast Core will use. + gcpServiceAccount: + # useExistingSecret specifies Feast to use an existing secret containing + # Google Cloud service account JSON key file. + # + # This is the only supported option for now to use a service account JSON. + # Feast admin is expected to create this secret before deploying Feast. + useExistingSecret: true + existingSecret: + # name is the secret name of the existing secret for the service account. + name: feast-gcp-service-account + # key is the secret key of the existing secret for the service account. + # key is normally derived from the file name of the JSON key file. + key: dev-konnekt-data-deep-1_feast-server-azure-cluster.json + # Setting service.type to NodePort exposes feast-core service at a static port + service: + type: ClusterIP + http: + nodePort: false + grpc: + nodePort: false + # Make kafka externally accessible using NodePort + kafka: + enabled: + false + postgresql: + enabled: true + application.yaml: + feast: + stream: + options: + # Point to one of your Kafka brokers + bootstrapServers: feast-kafka:9092 +# ============================================================ +# Feast Serving Online +# ============================================================ +feast-serving-online: + enabled: false +# ============================================================ +# Feast Serving Batch +# ============================================================ +feast-serving-batch: + enabled: true + redis: + enabled: false + image: + tag: "0.4.4" + service: + type: ClusterIP + http: + nodePort: false + grpc: + nodePort: false + ingress: + http: + enabled: false + grpc: + enabled: false + resources: + limits: + cpu: 1 + memory: 2G + requests: + cpu: 50m + memory: 256M + livenessProbe: + enabled: false + readinessProbe: + enabled: false + gcpServiceAccount: + useExistingSecret: true + existingSecret: + name: feast-gcp-service-account + key: dev-konnekt-data-deep-1_feast-server-azure-cluster.json + gcpProjectId: dev-konnekt-data-deep-1 + logLevel: info + application.yaml: + feast: + jobs: + staging-location: gs://dev-konnekt-data-deep-1-dataflow-workspace/batch-staging/ + bigquery-initial-retry-delay-secs: 1 + bigquery-total-timeout-secs: 21600 + store-type: REDIS + store-options: + host: feast-redis-jobstore-master + port: 6379 + store.yaml: + name: bigquery + type: BIGQUERY + bigquery_config: + project_id: dev-konnekt-data-deep-1 + dataset_id: feast_warehouse + subscriptions: + - name: "*" + project: "*" + version: "*" diff --git a/infra/desired-state/desired-state.yaml b/infra/desired-state/desired-state.yaml new file mode 100644 index 0000000000..2699aa2391 --- /dev/null +++ b/infra/desired-state/desired-state.yaml @@ -0,0 +1,49 @@ +settings: + kubeContext: feast-temp + +helmRepos: + stable: "https://kubernetes-charts.storage.googleapis.com" + incubator: "https://kubernetes-charts-incubator.storage.googleapis.com" + feast-charts: "https://feast-charts.storage.googleapis.com" + bitnami: "https://charts.bitnami.com/bitnami" + +namespaces: + feast: + +apps: + feast: + name: feast + description: Feast + namespace: feast + enabled: true + chart: feast-charts/feast + version: 0.4.7 + valuesFiles: + - chart-values/feast.yaml + feast-redis-jobstore: + name: feast-redis-jobstore + description: Feast Redis Database for Job Store + namespace: feast + enabled: true + chart: stable/redis + version: 10.2.1 + valuesFiles: + - chart-values/feast-redis-jobstore.yaml + feast-kafka: + name: feast-kafka + description: Feast Kafka instance + namespace: feast + enabled: true + chart: bitnami/kafka + version: 5.2.4 + valuesFiles: + - chart-values/feast-kafka.yaml + feast-zookeeper: + name: feast-zookeeper + description: Feast Zookeeper instance + namespace: feast + enabled: true + chart: bitnami/zookeeper + version: 4.3.4 + valuesFiles: + - chart-values/feast-zookeeper.yaml diff --git a/infra/scripts/deploy-to-aks.sh b/infra/scripts/deploy-to-aks.sh new file mode 100644 index 0000000000..984f04938a --- /dev/null +++ b/infra/scripts/deploy-to-aks.sh @@ -0,0 +1,36 @@ +# check kubectl + +if ! [ -x "$(command -v kubectl)" ]; then + echo 'Error: kubectl is not installed.' >&2 + + curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl +fi + +az aks get-credentials --name feast-temp --resource-group farfetch_rg + +# check helm + +if ! [ -x "$(command -v helm)" ]; then + echo 'Error: helm is not installed.' >&2 + + curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash +fi + +# update helm repos + +helm repo update + +# check helmsman + +if ! [ -x "$(command -v helmsman)" ]; then + echo 'Error: helmsman is not installed.' >&2 + + curl -L https://github.com/Praqma/helmsman/releases/download/v3.2.0/helmsman_3.2.0_linux_amd64.tar.gz | tar zx + mv helmsman /usr/local/bin/helmsman +fi + +# run helmsman + +helmsman -apply -f ../desired-state/desired-state.yaml -debug -no-ns -no-env-subst From 195560323121b9081da0b0b63bd60c5a34f01195 Mon Sep 17 00:00:00 2001 From: polatengin Date: Fri, 1 May 2020 15:36:49 +0300 Subject: [PATCH 02/23] make deploy-to-aks script executable --- .github/workflows/deploy-to-aks.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/deploy-to-aks.yaml b/.github/workflows/deploy-to-aks.yaml index 4a4a055d6f..869a741410 100644 --- a/.github/workflows/deploy-to-aks.yaml +++ b/.github/workflows/deploy-to-aks.yaml @@ -13,5 +13,8 @@ jobs: with: creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: make deploy-to-azure script executable + run: chmod +x ./infra/scripts/deploy-to-aks.sh + - name: deploy run: ./infra/scripts/deploy-to-aks.sh From 2577f3fd33a8769d5d22ea60b9e6a7381bb91dec Mon Sep 17 00:00:00 2001 From: polatengin Date: Fri, 1 May 2020 15:39:58 +0300 Subject: [PATCH 03/23] helmsman cli stay where we download it --- infra/scripts/deploy-to-aks.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/infra/scripts/deploy-to-aks.sh b/infra/scripts/deploy-to-aks.sh index 984f04938a..3e673276e4 100644 --- a/infra/scripts/deploy-to-aks.sh +++ b/infra/scripts/deploy-to-aks.sh @@ -28,9 +28,10 @@ if ! [ -x "$(command -v helmsman)" ]; then echo 'Error: helmsman is not installed.' >&2 curl -L https://github.com/Praqma/helmsman/releases/download/v3.2.0/helmsman_3.2.0_linux_amd64.tar.gz | tar zx - mv helmsman /usr/local/bin/helmsman + + chmod +x helmsman fi # run helmsman -helmsman -apply -f ../desired-state/desired-state.yaml -debug -no-ns -no-env-subst +./helmsman -apply -f ../desired-state/desired-state.yaml -debug -no-ns -no-env-subst From 0b5d0cc2c822f1b2063db9f56dfb1a264515346a Mon Sep 17 00:00:00 2001 From: polatengin Date: Fri, 1 May 2020 15:42:10 +0300 Subject: [PATCH 04/23] missing helm diff plugin installed --- infra/scripts/deploy-to-aks.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infra/scripts/deploy-to-aks.sh b/infra/scripts/deploy-to-aks.sh index 3e673276e4..8162c24b71 100644 --- a/infra/scripts/deploy-to-aks.sh +++ b/infra/scripts/deploy-to-aks.sh @@ -22,6 +22,8 @@ fi helm repo update +helm plugin install https://github.com/databus23/helm-diff --version master + # check helmsman if ! [ -x "$(command -v helmsman)" ]; then From 5eb824152b8c86a4389a0cec6559d1a9e318e7de Mon Sep 17 00:00:00 2001 From: polatengin Date: Fri, 1 May 2020 15:44:57 +0300 Subject: [PATCH 05/23] fixing desired state yaml file location --- infra/scripts/deploy-to-aks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/scripts/deploy-to-aks.sh b/infra/scripts/deploy-to-aks.sh index 8162c24b71..52733922f0 100644 --- a/infra/scripts/deploy-to-aks.sh +++ b/infra/scripts/deploy-to-aks.sh @@ -36,4 +36,4 @@ fi # run helmsman -./helmsman -apply -f ../desired-state/desired-state.yaml -debug -no-ns -no-env-subst +./helmsman -apply -f ./infra/desired-state/desired-state.yaml -debug -no-ns -no-env-subst From b2542f090b8662199096066d8e64388ab4a3be6b Mon Sep 17 00:00:00 2001 From: polatengin Date: Fri, 1 May 2020 15:52:16 +0300 Subject: [PATCH 06/23] comment added to trigger github action --- infra/scripts/deploy-to-aks.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/infra/scripts/deploy-to-aks.sh b/infra/scripts/deploy-to-aks.sh index 52733922f0..ba0aa9df38 100644 --- a/infra/scripts/deploy-to-aks.sh +++ b/infra/scripts/deploy-to-aks.sh @@ -22,6 +22,8 @@ fi helm repo update +# install required plugins + helm plugin install https://github.com/databus23/helm-diff --version master # check helmsman From 9d9bc0ac84fbd0871fc4289ff6e513646f759b49 Mon Sep 17 00:00:00 2001 From: Ben Vozza Date: Mon, 4 May 2020 11:57:14 +0100 Subject: [PATCH 07/23] update to use latest feast chart version + clean feast chart values --- infra/desired-state/chart-values/feast.yaml | 237 ++++++-------------- infra/desired-state/desired-state.yaml | 4 +- 2 files changed, 74 insertions(+), 167 deletions(-) diff --git a/infra/desired-state/chart-values/feast.yaml b/infra/desired-state/chart-values/feast.yaml index 608706a1c1..8b348506a6 100644 --- a/infra/desired-state/chart-values/feast.yaml +++ b/infra/desired-state/chart-values/feast.yaml @@ -1,165 +1,72 @@ -# Feast deployment installs the following components: -# - Feast Core -# - Feast Serving Online -# - Feast Serving Batch -# - Prometheus StatsD Exporter -# -# The configuration for different components can be referenced from: -# - charts/feast-core/values.yaml -# - charts/feast-serving/values.yaml -# - charts/prometheus-statsd-exporter/values.yaml -# -# Note that "feast-serving-online" and "feast-serving-batch" are -# aliases to "feast-serving" chart since in typical scenario two instances -# of Feast Serving: online and batch will be deployed. Both described -# using the same chart "feast-serving". -# -# Note that the import job by default uses DirectRunner -# https://beam.apache.org/documentation/runners/direct/ -# in this configuration since it allows Feast to run in more environments -# (unlike DataflowRunner which requires Google Cloud services). -# -# A secret containing Google Cloud service account JSON key is required -# in this configuration. -# https://cloud.google.com/iam/docs/creating-managing-service-accounts -# -# The Google Cloud service account must have the following roles: -# - bigquery.dataEditor -# - bigquery.jobUser -# -# Assuming a service account JSON key file has been downloaded to -# (please name the file key.json): -# /home/user/key.json -# -# Run the following command to create the secret in your Kubernetes cluster: -# -# kubectl create secret generic feast-gcp-service-account \ -# --from-file=/home/user/key.json -# -# ============================================================ -# Feast Core -# ============================================================ -kafka: - enabled: false -postgresql: - enabled: true - -feast-core: - # If enabled specifies whether to install Feast Core component. - # - # Normally, this is set to "false" when Feast users need access to low latency - # Feast Serving, by deploying multiple instances of Feast Serving closest - # to the client. These instances of Feast Serving however can still use - # the same shared Feast Core. - enabled: true - # Specify which image tag to use. Keep this consistent for all components - image: - tag: "0.4.4" - # jvmOptions are options that will be passed to the Java Virtual Machine (JVM) - # running Feast Core. - # - # resources that should be allocated to Feast Core. - resources: - limits: - cpu: 2 - memory: 2G - requests: - cpu: 100m - memory: 500Mi - gcpProjectId: dev-konnekt-data-deep-1 - # gcpServiceAccount is the Google service account that Feast Core will use. - gcpServiceAccount: - # useExistingSecret specifies Feast to use an existing secret containing - # Google Cloud service account JSON key file. - # - # This is the only supported option for now to use a service account JSON. - # Feast admin is expected to create this secret before deploying Feast. - useExistingSecret: true - existingSecret: - # name is the secret name of the existing secret for the service account. - name: feast-gcp-service-account - # key is the secret key of the existing secret for the service account. - # key is normally derived from the file name of the JSON key file. - key: dev-konnekt-data-deep-1_feast-server-azure-cluster.json - # Setting service.type to NodePort exposes feast-core service at a static port - service: - type: ClusterIP - http: - nodePort: false - grpc: - nodePort: false - # Make kafka externally accessible using NodePort - kafka: - enabled: - false - postgresql: - enabled: true - application.yaml: - feast: - stream: - options: - # Point to one of your Kafka brokers - bootstrapServers: feast-kafka:9092 -# ============================================================ -# Feast Serving Online -# ============================================================ -feast-serving-online: - enabled: false -# ============================================================ -# Feast Serving Batch -# ============================================================ -feast-serving-batch: - enabled: true - redis: - enabled: false - image: - tag: "0.4.4" - service: - type: ClusterIP - http: - nodePort: false - grpc: - nodePort: false - ingress: - http: - enabled: false - grpc: - enabled: false - resources: - limits: - cpu: 1 - memory: 2G - requests: - cpu: 50m - memory: 256M - livenessProbe: - enabled: false - readinessProbe: - enabled: false - gcpServiceAccount: - useExistingSecret: true - existingSecret: - name: feast-gcp-service-account - key: dev-konnekt-data-deep-1_feast-server-azure-cluster.json - gcpProjectId: dev-konnekt-data-deep-1 - logLevel: info - application.yaml: - feast: - jobs: - staging-location: gs://dev-konnekt-data-deep-1-dataflow-workspace/batch-staging/ - bigquery-initial-retry-delay-secs: 1 - bigquery-total-timeout-secs: 21600 - store-type: REDIS - store-options: - host: feast-redis-jobstore-master - port: 6379 - store.yaml: - name: bigquery - type: BIGQUERY - bigquery_config: - project_id: dev-konnekt-data-deep-1 - dataset_id: feast_warehouse - subscriptions: - - name: "*" - project: "*" - version: "*" +kafka: + enabled: false +redis: + enabled: false + +feast-core: + enabled: + true + gcpServiceAccount: + enabled: true + existingSecret: + name: feast-gcp-service-account + key: dev-konnekt-data-deep-1_feast-server-azure-cluster.json + postgresql: + existingSecret: feast-postgresql + kafka: + enabled: + false + application-override.yaml: + feast: + stream: + type: kafka + options: + topic: feast-features + bootstrapServers: feast-kafka:9092 + +feast-online-serving: + enabled: false + +feast-batch-serving: + enabled: + true + redis: + enabled: false + livenessProbe: + enabled: true + readinessProbe: + enabled: true + gcpServiceAccount: + enabled: true + existingSecret: + name: feast-gcp-service-account + key: dev-konnekt-data-deep-1_feast-server-azure-cluster.json + application-override.yaml: + feast: + active_store: historical + stores: + - name: historical + type: BIGQUERY + config: + project_id: dev-konnekt-data-deep-1 + dataset_id: feast_warehouse + staging_location: gs://dev-konnekt-data-deep-1-dataflow-workspace/batch-staging/ + initial_retry_delay_seconds: 3 + total_timeout_seconds: 21600 + subscriptions: + - name: "*" + project: "*" + version: "*" + stream: + type: kafka + options: + topic: feast-features + bootstrapServers: feast-kafka:9092 + job-store: + redis_host: feast-redis-jobstore-master + redis_port: 6379 + + + +postgresql: + existingSecret: feast-postgresql diff --git a/infra/desired-state/desired-state.yaml b/infra/desired-state/desired-state.yaml index 2699aa2391..614542c8e6 100644 --- a/infra/desired-state/desired-state.yaml +++ b/infra/desired-state/desired-state.yaml @@ -16,8 +16,8 @@ apps: description: Feast namespace: feast enabled: true - chart: feast-charts/feast - version: 0.4.7 + chart: farfetch-public/feast-beta + version: 0.5.1 valuesFiles: - chart-values/feast.yaml feast-redis-jobstore: From 8d84e5a2cc64705f3e38554b4a63eb43af015185 Mon Sep 17 00:00:00 2001 From: Ben Vozza Date: Mon, 4 May 2020 12:04:27 +0100 Subject: [PATCH 08/23] add missing farfetch chart repo --- infra/desired-state/desired-state.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/infra/desired-state/desired-state.yaml b/infra/desired-state/desired-state.yaml index 614542c8e6..af6ee99005 100644 --- a/infra/desired-state/desired-state.yaml +++ b/infra/desired-state/desired-state.yaml @@ -6,6 +6,7 @@ helmRepos: incubator: "https://kubernetes-charts-incubator.storage.googleapis.com" feast-charts: "https://feast-charts.storage.googleapis.com" bitnami: "https://charts.bitnami.com/bitnami" + farfetch-public: "https://konnekt-public-charts.storage.googleapis.com" namespaces: feast: From c6a734558deb766c4ace234946cc4397c17b7d09 Mon Sep 17 00:00:00 2001 From: Ben Vozza Date: Mon, 4 May 2020 12:18:40 +0100 Subject: [PATCH 09/23] disable prometheus + grafana --- infra/desired-state/chart-values/feast.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/infra/desired-state/chart-values/feast.yaml b/infra/desired-state/chart-values/feast.yaml index 8b348506a6..df448824ab 100644 --- a/infra/desired-state/chart-values/feast.yaml +++ b/infra/desired-state/chart-values/feast.yaml @@ -2,6 +2,12 @@ kafka: enabled: false redis: enabled: false +grafana: + enabled: false +prometheus-statsd-exporter: + enabled: false +prometheus: + enabled: faslse feast-core: enabled: From 49025fea49387a3cca7ad5bc56fd6466e2c78527 Mon Sep 17 00:00:00 2001 From: Ben Vozza Date: Mon, 4 May 2020 12:19:06 +0100 Subject: [PATCH 10/23] correct prometheus typo --- infra/desired-state/chart-values/feast.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/desired-state/chart-values/feast.yaml b/infra/desired-state/chart-values/feast.yaml index df448824ab..f042aabcf4 100644 --- a/infra/desired-state/chart-values/feast.yaml +++ b/infra/desired-state/chart-values/feast.yaml @@ -7,7 +7,7 @@ grafana: prometheus-statsd-exporter: enabled: false prometheus: - enabled: faslse + enabled: false feast-core: enabled: From 01c18667d6af3686415c0cff570eed1ca8fb122d Mon Sep 17 00:00:00 2001 From: Ben Vozza Date: Mon, 4 May 2020 13:10:22 +0100 Subject: [PATCH 11/23] disable prometheus in core --- infra/desired-state/chart-values/feast.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/infra/desired-state/chart-values/feast.yaml b/infra/desired-state/chart-values/feast.yaml index f042aabcf4..7b23ccba56 100644 --- a/infra/desired-state/chart-values/feast.yaml +++ b/infra/desired-state/chart-values/feast.yaml @@ -12,6 +12,9 @@ prometheus: feast-core: enabled: true + prometheus: + enabled: + false gcpServiceAccount: enabled: true existingSecret: From fd7139b62f34ac63263b5914f09380b523b5b207 Mon Sep 17 00:00:00 2001 From: Ben Vozza Date: Mon, 4 May 2020 14:34:29 +0100 Subject: [PATCH 12/23] enable prometheus --- infra/desired-state/chart-values/feast.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/desired-state/chart-values/feast.yaml b/infra/desired-state/chart-values/feast.yaml index 7b23ccba56..c4cbbc9246 100644 --- a/infra/desired-state/chart-values/feast.yaml +++ b/infra/desired-state/chart-values/feast.yaml @@ -3,18 +3,18 @@ kafka: redis: enabled: false grafana: - enabled: false + enabled: true prometheus-statsd-exporter: - enabled: false + enabled: true prometheus: - enabled: false + enabled: true feast-core: enabled: true prometheus: enabled: - false + true gcpServiceAccount: enabled: true existingSecret: From bf94a47eae265bad482e6ffc62fe827ad65b5e1d Mon Sep 17 00:00:00 2001 From: Ben Vozza Date: Tue, 5 May 2020 07:56:17 +0100 Subject: [PATCH 13/23] update feast chart version --- infra/desired-state/desired-state.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/desired-state/desired-state.yaml b/infra/desired-state/desired-state.yaml index af6ee99005..3e0493ccac 100644 --- a/infra/desired-state/desired-state.yaml +++ b/infra/desired-state/desired-state.yaml @@ -17,8 +17,8 @@ apps: description: Feast namespace: feast enabled: true - chart: farfetch-public/feast-beta - version: 0.5.1 + chart: farfetch-public/feast + version: 0.5.0 valuesFiles: - chart-values/feast.yaml feast-redis-jobstore: From 262d35f23d779d6b2bd4757a43d680292655c89c Mon Sep 17 00:00:00 2001 From: polatengin Date: Tue, 5 May 2020 12:04:55 +0300 Subject: [PATCH 14/23] temp folder removed --- .../.helmsman-tmp/tmp326240254/feast.yaml | 165 ------------------ .../tmp553497568/feast-kafka.yaml | 14 -- .../tmp622116287/feast-zookeeper.yaml | 10 -- .../tmp717124421/feast-redis-jobstore.yaml | 40 ----- 4 files changed, 229 deletions(-) delete mode 100644 infra/desired-state/.helmsman-tmp/tmp326240254/feast.yaml delete mode 100644 infra/desired-state/.helmsman-tmp/tmp553497568/feast-kafka.yaml delete mode 100644 infra/desired-state/.helmsman-tmp/tmp622116287/feast-zookeeper.yaml delete mode 100644 infra/desired-state/.helmsman-tmp/tmp717124421/feast-redis-jobstore.yaml diff --git a/infra/desired-state/.helmsman-tmp/tmp326240254/feast.yaml b/infra/desired-state/.helmsman-tmp/tmp326240254/feast.yaml deleted file mode 100644 index 608706a1c1..0000000000 --- a/infra/desired-state/.helmsman-tmp/tmp326240254/feast.yaml +++ /dev/null @@ -1,165 +0,0 @@ -# Feast deployment installs the following components: -# - Feast Core -# - Feast Serving Online -# - Feast Serving Batch -# - Prometheus StatsD Exporter -# -# The configuration for different components can be referenced from: -# - charts/feast-core/values.yaml -# - charts/feast-serving/values.yaml -# - charts/prometheus-statsd-exporter/values.yaml -# -# Note that "feast-serving-online" and "feast-serving-batch" are -# aliases to "feast-serving" chart since in typical scenario two instances -# of Feast Serving: online and batch will be deployed. Both described -# using the same chart "feast-serving". -# -# Note that the import job by default uses DirectRunner -# https://beam.apache.org/documentation/runners/direct/ -# in this configuration since it allows Feast to run in more environments -# (unlike DataflowRunner which requires Google Cloud services). -# -# A secret containing Google Cloud service account JSON key is required -# in this configuration. -# https://cloud.google.com/iam/docs/creating-managing-service-accounts -# -# The Google Cloud service account must have the following roles: -# - bigquery.dataEditor -# - bigquery.jobUser -# -# Assuming a service account JSON key file has been downloaded to -# (please name the file key.json): -# /home/user/key.json -# -# Run the following command to create the secret in your Kubernetes cluster: -# -# kubectl create secret generic feast-gcp-service-account \ -# --from-file=/home/user/key.json -# -# ============================================================ -# Feast Core -# ============================================================ -kafka: - enabled: false -postgresql: - enabled: true - -feast-core: - # If enabled specifies whether to install Feast Core component. - # - # Normally, this is set to "false" when Feast users need access to low latency - # Feast Serving, by deploying multiple instances of Feast Serving closest - # to the client. These instances of Feast Serving however can still use - # the same shared Feast Core. - enabled: true - # Specify which image tag to use. Keep this consistent for all components - image: - tag: "0.4.4" - # jvmOptions are options that will be passed to the Java Virtual Machine (JVM) - # running Feast Core. - # - # resources that should be allocated to Feast Core. - resources: - limits: - cpu: 2 - memory: 2G - requests: - cpu: 100m - memory: 500Mi - gcpProjectId: dev-konnekt-data-deep-1 - # gcpServiceAccount is the Google service account that Feast Core will use. - gcpServiceAccount: - # useExistingSecret specifies Feast to use an existing secret containing - # Google Cloud service account JSON key file. - # - # This is the only supported option for now to use a service account JSON. - # Feast admin is expected to create this secret before deploying Feast. - useExistingSecret: true - existingSecret: - # name is the secret name of the existing secret for the service account. - name: feast-gcp-service-account - # key is the secret key of the existing secret for the service account. - # key is normally derived from the file name of the JSON key file. - key: dev-konnekt-data-deep-1_feast-server-azure-cluster.json - # Setting service.type to NodePort exposes feast-core service at a static port - service: - type: ClusterIP - http: - nodePort: false - grpc: - nodePort: false - # Make kafka externally accessible using NodePort - kafka: - enabled: - false - postgresql: - enabled: true - application.yaml: - feast: - stream: - options: - # Point to one of your Kafka brokers - bootstrapServers: feast-kafka:9092 -# ============================================================ -# Feast Serving Online -# ============================================================ -feast-serving-online: - enabled: false -# ============================================================ -# Feast Serving Batch -# ============================================================ -feast-serving-batch: - enabled: true - redis: - enabled: false - image: - tag: "0.4.4" - service: - type: ClusterIP - http: - nodePort: false - grpc: - nodePort: false - ingress: - http: - enabled: false - grpc: - enabled: false - resources: - limits: - cpu: 1 - memory: 2G - requests: - cpu: 50m - memory: 256M - livenessProbe: - enabled: false - readinessProbe: - enabled: false - gcpServiceAccount: - useExistingSecret: true - existingSecret: - name: feast-gcp-service-account - key: dev-konnekt-data-deep-1_feast-server-azure-cluster.json - gcpProjectId: dev-konnekt-data-deep-1 - logLevel: info - application.yaml: - feast: - jobs: - staging-location: gs://dev-konnekt-data-deep-1-dataflow-workspace/batch-staging/ - bigquery-initial-retry-delay-secs: 1 - bigquery-total-timeout-secs: 21600 - store-type: REDIS - store-options: - host: feast-redis-jobstore-master - port: 6379 - store.yaml: - name: bigquery - type: BIGQUERY - bigquery_config: - project_id: dev-konnekt-data-deep-1 - dataset_id: feast_warehouse - subscriptions: - - name: "*" - project: "*" - version: "*" diff --git a/infra/desired-state/.helmsman-tmp/tmp553497568/feast-kafka.yaml b/infra/desired-state/.helmsman-tmp/tmp553497568/feast-kafka.yaml deleted file mode 100644 index bfe681c14f..0000000000 --- a/infra/desired-state/.helmsman-tmp/tmp553497568/feast-kafka.yaml +++ /dev/null @@ -1,14 +0,0 @@ -heapOpts: -Xmx1024m -Xms1024m -persistence: - enabled: false -resources: - limits: - cpu: 1200m - memory: 1280Mi - requests: - cpu: 50m - memory: 1024Mi -zookeeper: - enabled: false -externalZookeeper: - servers: feast-zookeeper:2181 diff --git a/infra/desired-state/.helmsman-tmp/tmp622116287/feast-zookeeper.yaml b/infra/desired-state/.helmsman-tmp/tmp622116287/feast-zookeeper.yaml deleted file mode 100644 index d4409e400d..0000000000 --- a/infra/desired-state/.helmsman-tmp/tmp622116287/feast-zookeeper.yaml +++ /dev/null @@ -1,10 +0,0 @@ -heapSize: 256 -persistence: - enabled: false -resources: - limits: - cpu: 100m - memory: 512Mi - requests: - cpu: 10m - memory: 256Mi diff --git a/infra/desired-state/.helmsman-tmp/tmp717124421/feast-redis-jobstore.yaml b/infra/desired-state/.helmsman-tmp/tmp717124421/feast-redis-jobstore.yaml deleted file mode 100644 index 5b4f228e56..0000000000 --- a/infra/desired-state/.helmsman-tmp/tmp717124421/feast-redis-jobstore.yaml +++ /dev/null @@ -1,40 +0,0 @@ -cluster: - enabled: false -sentinel: - enabled: false -usePassword: false -master: - resources: - limits: - cpu: 2 - memory: 300Mi - requests: - cpu: 100m - memory: 220Mi - persistence: - enabled: false -metrics: - enabled: true - resources: - limits: - cpu: 100m - memory: 64Mi - requests: - cpu: 25m - memory: 32Mi -sysctlImage: - enabled: true - mountHostSys: true - command: - - /bin/sh - - -c - - |- - echo never > /host-sys/kernel/mm/transparent_hugepage/enabled -configmap: |- - save "" - appendonly yes - appendfsync everysec - dir /data - appendfilename "appendonly.aof" - maxmemory 200mb - maxmemory-policy noeviction From cbbab3c555809a1327a331a57ca9919c998daba6 Mon Sep 17 00:00:00 2001 From: polatengin Date: Tue, 5 May 2020 12:05:35 +0300 Subject: [PATCH 15/23] helmsman temp folder ignored --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5aefa03593..8bbf1f9d20 100644 --- a/.gitignore +++ b/.gitignore @@ -193,4 +193,5 @@ pom-ff-deploy.xml secrets/ # Dumping ground -tmp/ \ No newline at end of file +tmp/ +.helmsman-tmp/ From 6d6798fd52a8ebdd38d01426525e27420aacf514 Mon Sep 17 00:00:00 2001 From: polatengin Date: Tue, 5 May 2020 15:45:47 +0300 Subject: [PATCH 16/23] workflow trigger has branch and path filters now --- .github/workflows/deploy-to-aks.yaml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy-to-aks.yaml b/.github/workflows/deploy-to-aks.yaml index 869a741410..7be7229699 100644 --- a/.github/workflows/deploy-to-aks.yaml +++ b/.github/workflows/deploy-to-aks.yaml @@ -1,6 +1,13 @@ name: deploy-to-aks -on: push +on: + push: + branches: + - master + paths: + - 'infra/charts/**' + - 'infra/desired-state/**' + - 'infra/scripts/deploy-to-aks.sh' jobs: deploy: From 9775a43a24e38726b3a242989182c9ef3f27d88d Mon Sep 17 00:00:00 2001 From: Ben Vozza Date: Tue, 5 May 2020 14:33:15 +0100 Subject: [PATCH 17/23] remove redis persistance config --- .../chart-values/feast-redis-jobstore.yaml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/infra/desired-state/chart-values/feast-redis-jobstore.yaml b/infra/desired-state/chart-values/feast-redis-jobstore.yaml index 5b4f228e56..c45e0e2b30 100644 --- a/infra/desired-state/chart-values/feast-redis-jobstore.yaml +++ b/infra/desired-state/chart-values/feast-redis-jobstore.yaml @@ -29,12 +29,4 @@ sysctlImage: - /bin/sh - -c - |- - echo never > /host-sys/kernel/mm/transparent_hugepage/enabled -configmap: |- - save "" - appendonly yes - appendfsync everysec - dir /data - appendfilename "appendonly.aof" - maxmemory 200mb - maxmemory-policy noeviction + echo never > /host-sys/kernel/mm/transparent_hugepage/enabled \ No newline at end of file From 75d23f7a457c67389ed7695239d0644b7ae4a073 Mon Sep 17 00:00:00 2001 From: polatengin Date: Wed, 13 May 2020 18:49:03 +0300 Subject: [PATCH 18/23] shebang and a description added --- infra/scripts/deploy-to-aks.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/infra/scripts/deploy-to-aks.sh b/infra/scripts/deploy-to-aks.sh index ba0aa9df38..555b7d9dae 100644 --- a/infra/scripts/deploy-to-aks.sh +++ b/infra/scripts/deploy-to-aks.sh @@ -1,3 +1,11 @@ +#!/bin/bash + +# This script automates deploy Feast to Azure Kubernetes Service (a.k.a AKS) +# Checks dependencies first, if there is a missing dependency, install it silently + +# Stop on error +set -e + # check kubectl if ! [ -x "$(command -v kubectl)" ]; then From ee48fbfb089e996b181cbad94c7d1509475da820 Mon Sep 17 00:00:00 2001 From: polatengin Date: Wed, 13 May 2020 18:51:31 +0300 Subject: [PATCH 19/23] curl commands silent now --- infra/scripts/deploy-to-aks.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/infra/scripts/deploy-to-aks.sh b/infra/scripts/deploy-to-aks.sh index 555b7d9dae..5e713a1285 100644 --- a/infra/scripts/deploy-to-aks.sh +++ b/infra/scripts/deploy-to-aks.sh @@ -23,7 +23,7 @@ az aks get-credentials --name feast-temp --resource-group farfetch_rg if ! [ -x "$(command -v helm)" ]; then echo 'Error: helm is not installed.' >&2 - curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash + curl -Ls https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | DESIRED_VERSION=v3.2.1 bash fi # update helm repos @@ -39,7 +39,7 @@ helm plugin install https://github.com/databus23/helm-diff --version master if ! [ -x "$(command -v helmsman)" ]; then echo 'Error: helmsman is not installed.' >&2 - curl -L https://github.com/Praqma/helmsman/releases/download/v3.2.0/helmsman_3.2.0_linux_amd64.tar.gz | tar zx + curl -Ls https://github.com/Praqma/helmsman/releases/download/v3.2.0/helmsman_3.2.0_linux_amd64.tar.gz | tar zx chmod +x helmsman fi From 402bdce0cab66f076cd17574f38452bc0f8c9cc0 Mon Sep 17 00:00:00 2001 From: polatengin Date: Wed, 13 May 2020 18:53:05 +0300 Subject: [PATCH 20/23] install pinned helm-diff plugin if it's not installed already --- infra/scripts/deploy-to-aks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/scripts/deploy-to-aks.sh b/infra/scripts/deploy-to-aks.sh index 5e713a1285..7f9ed12aa6 100644 --- a/infra/scripts/deploy-to-aks.sh +++ b/infra/scripts/deploy-to-aks.sh @@ -32,7 +32,7 @@ helm repo update # install required plugins -helm plugin install https://github.com/databus23/helm-diff --version master +helm plugin list | grep -cE ^diff>/dev/null || helm plugin install "https://github.com/databus23/helm-diff" --version 3.1.1 # check helmsman From 16156c65037d0e4f3c9e3f1fac0c00aedd4e42ee Mon Sep 17 00:00:00 2001 From: polatengin Date: Wed, 13 May 2020 18:55:23 +0300 Subject: [PATCH 21/23] deploy to aks script is executable now, unneccessary step removed from workflow --- .github/workflows/deploy-to-aks.yaml | 3 --- infra/scripts/deploy-to-aks.sh | 0 2 files changed, 3 deletions(-) mode change 100644 => 100755 infra/scripts/deploy-to-aks.sh diff --git a/.github/workflows/deploy-to-aks.yaml b/.github/workflows/deploy-to-aks.yaml index 7be7229699..476eb2b548 100644 --- a/.github/workflows/deploy-to-aks.yaml +++ b/.github/workflows/deploy-to-aks.yaml @@ -20,8 +20,5 @@ jobs: with: creds: ${{ secrets.AZURE_CREDENTIALS }} - - name: make deploy-to-azure script executable - run: chmod +x ./infra/scripts/deploy-to-aks.sh - - name: deploy run: ./infra/scripts/deploy-to-aks.sh diff --git a/infra/scripts/deploy-to-aks.sh b/infra/scripts/deploy-to-aks.sh old mode 100644 new mode 100755 From f772a214244b4812ffc03ad73f48b2ed64f31e4e Mon Sep 17 00:00:00 2001 From: polatengin Date: Wed, 13 May 2020 18:57:15 +0300 Subject: [PATCH 22/23] no-env-subst switch removed, 'cause it's the default behavior --- infra/scripts/deploy-to-aks.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/infra/scripts/deploy-to-aks.sh b/infra/scripts/deploy-to-aks.sh index 7f9ed12aa6..c62dfbb4ab 100755 --- a/infra/scripts/deploy-to-aks.sh +++ b/infra/scripts/deploy-to-aks.sh @@ -46,4 +46,4 @@ fi # run helmsman -./helmsman -apply -f ./infra/desired-state/desired-state.yaml -debug -no-ns -no-env-subst +./helmsman -apply -f ./infra/desired-state/desired-state.yaml -debug -no-ns From d3bdee7fdf774bcf4e538f55a3a1741bd3980111 Mon Sep 17 00:00:00 2001 From: polatengin Date: Wed, 13 May 2020 18:58:49 +0300 Subject: [PATCH 23/23] helmsman automatically updates helm lists, no need to do it implicitly --- infra/scripts/deploy-to-aks.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/infra/scripts/deploy-to-aks.sh b/infra/scripts/deploy-to-aks.sh index c62dfbb4ab..fccabf6359 100755 --- a/infra/scripts/deploy-to-aks.sh +++ b/infra/scripts/deploy-to-aks.sh @@ -26,10 +26,6 @@ if ! [ -x "$(command -v helm)" ]; then curl -Ls https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | DESIRED_VERSION=v3.2.1 bash fi -# update helm repos - -helm repo update - # install required plugins helm plugin list | grep -cE ^diff>/dev/null || helm plugin install "https://github.com/databus23/helm-diff" --version 3.1.1