diff --git a/.github/workflows/deploy-to-aks.yaml b/.github/workflows/deploy-to-aks.yaml new file mode 100644 index 0000000000..476eb2b548 --- /dev/null +++ b/.github/workflows/deploy-to-aks.yaml @@ -0,0 +1,24 @@ +name: deploy-to-aks + +on: + push: + branches: + - master + paths: + - 'infra/charts/**' + - 'infra/desired-state/**' + - 'infra/scripts/deploy-to-aks.sh' + +jobs: + deploy: + runs-on: ubuntu-latest + name: deploy + steps: + - uses: actions/checkout@v1 + + - uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + + - name: deploy + run: ./infra/scripts/deploy-to-aks.sh diff --git a/.gitignore b/.gitignore index 5aefa03593..8bbf1f9d20 100644 --- a/.gitignore +++ b/.gitignore @@ -193,4 +193,5 @@ pom-ff-deploy.xml secrets/ # Dumping ground -tmp/ \ No newline at end of file +tmp/ +.helmsman-tmp/ diff --git a/infra/desired-state/chart-values/feast-kafka.yaml b/infra/desired-state/chart-values/feast-kafka.yaml new file mode 100644 index 0000000000..bfe681c14f --- /dev/null +++ b/infra/desired-state/chart-values/feast-kafka.yaml @@ -0,0 +1,14 @@ +heapOpts: -Xmx1024m -Xms1024m +persistence: + enabled: false +resources: + limits: + cpu: 1200m + memory: 1280Mi + requests: + cpu: 50m + memory: 1024Mi +zookeeper: + enabled: false +externalZookeeper: + servers: feast-zookeeper:2181 diff --git a/infra/desired-state/chart-values/feast-redis-jobstore.yaml b/infra/desired-state/chart-values/feast-redis-jobstore.yaml new file mode 100644 index 0000000000..c45e0e2b30 --- /dev/null +++ b/infra/desired-state/chart-values/feast-redis-jobstore.yaml @@ -0,0 +1,32 @@ +cluster: + enabled: false +sentinel: + enabled: false +usePassword: false +master: + resources: + limits: + cpu: 2 + memory: 300Mi + requests: + cpu: 100m + memory: 220Mi + persistence: + enabled: false +metrics: + enabled: true + resources: + limits: + cpu: 100m + memory: 64Mi + requests: + cpu: 25m + memory: 32Mi +sysctlImage: + enabled: true + mountHostSys: true + command: + - /bin/sh + - -c + - |- + echo never > /host-sys/kernel/mm/transparent_hugepage/enabled \ No newline at end of file diff --git a/infra/desired-state/chart-values/feast-zookeeper.yaml b/infra/desired-state/chart-values/feast-zookeeper.yaml new file mode 100644 index 0000000000..d4409e400d --- /dev/null +++ b/infra/desired-state/chart-values/feast-zookeeper.yaml @@ -0,0 +1,10 @@ +heapSize: 256 +persistence: + enabled: false +resources: + limits: + cpu: 100m + memory: 512Mi + requests: + cpu: 10m + memory: 256Mi diff --git a/infra/desired-state/chart-values/feast.yaml b/infra/desired-state/chart-values/feast.yaml new file mode 100644 index 0000000000..c4cbbc9246 --- /dev/null +++ b/infra/desired-state/chart-values/feast.yaml @@ -0,0 +1,81 @@ +kafka: + enabled: false +redis: + enabled: false +grafana: + enabled: true +prometheus-statsd-exporter: + enabled: true +prometheus: + enabled: true + +feast-core: + enabled: + true + prometheus: + enabled: + true + gcpServiceAccount: + enabled: true + existingSecret: + name: feast-gcp-service-account + key: dev-konnekt-data-deep-1_feast-server-azure-cluster.json + postgresql: + existingSecret: feast-postgresql + kafka: + enabled: + false + application-override.yaml: + feast: + stream: + type: kafka + options: + topic: feast-features + bootstrapServers: feast-kafka:9092 + +feast-online-serving: + enabled: false + +feast-batch-serving: + enabled: + true + redis: + enabled: false + livenessProbe: + enabled: true + readinessProbe: + enabled: true + gcpServiceAccount: + enabled: true + existingSecret: + name: feast-gcp-service-account + key: dev-konnekt-data-deep-1_feast-server-azure-cluster.json + application-override.yaml: + feast: + active_store: historical + stores: + - name: historical + type: BIGQUERY + config: + project_id: dev-konnekt-data-deep-1 + dataset_id: feast_warehouse + staging_location: gs://dev-konnekt-data-deep-1-dataflow-workspace/batch-staging/ + initial_retry_delay_seconds: 3 + total_timeout_seconds: 21600 + subscriptions: + - name: "*" + project: "*" + version: "*" + stream: + type: kafka + options: + topic: feast-features + bootstrapServers: feast-kafka:9092 + job-store: + redis_host: feast-redis-jobstore-master + redis_port: 6379 + + + +postgresql: + existingSecret: feast-postgresql diff --git a/infra/desired-state/desired-state.yaml b/infra/desired-state/desired-state.yaml new file mode 100644 index 0000000000..3e0493ccac --- /dev/null +++ b/infra/desired-state/desired-state.yaml @@ -0,0 +1,50 @@ +settings: + kubeContext: feast-temp + +helmRepos: + stable: "https://kubernetes-charts.storage.googleapis.com" + incubator: "https://kubernetes-charts-incubator.storage.googleapis.com" + feast-charts: "https://feast-charts.storage.googleapis.com" + bitnami: "https://charts.bitnami.com/bitnami" + farfetch-public: "https://konnekt-public-charts.storage.googleapis.com" + +namespaces: + feast: + +apps: + feast: + name: feast + description: Feast + namespace: feast + enabled: true + chart: farfetch-public/feast + version: 0.5.0 + valuesFiles: + - chart-values/feast.yaml + feast-redis-jobstore: + name: feast-redis-jobstore + description: Feast Redis Database for Job Store + namespace: feast + enabled: true + chart: stable/redis + version: 10.2.1 + valuesFiles: + - chart-values/feast-redis-jobstore.yaml + feast-kafka: + name: feast-kafka + description: Feast Kafka instance + namespace: feast + enabled: true + chart: bitnami/kafka + version: 5.2.4 + valuesFiles: + - chart-values/feast-kafka.yaml + feast-zookeeper: + name: feast-zookeeper + description: Feast Zookeeper instance + namespace: feast + enabled: true + chart: bitnami/zookeeper + version: 4.3.4 + valuesFiles: + - chart-values/feast-zookeeper.yaml diff --git a/infra/scripts/deploy-to-aks.sh b/infra/scripts/deploy-to-aks.sh new file mode 100755 index 0000000000..fccabf6359 --- /dev/null +++ b/infra/scripts/deploy-to-aks.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# This script automates deploy Feast to Azure Kubernetes Service (a.k.a AKS) +# Checks dependencies first, if there is a missing dependency, install it silently + +# Stop on error +set -e + +# check kubectl + +if ! [ -x "$(command -v kubectl)" ]; then + echo 'Error: kubectl is not installed.' >&2 + + curl -LO https://storage.googleapis.com/kubernetes-release/release/`curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt`/bin/linux/amd64/kubectl + chmod +x ./kubectl + sudo mv ./kubectl /usr/local/bin/kubectl +fi + +az aks get-credentials --name feast-temp --resource-group farfetch_rg + +# check helm + +if ! [ -x "$(command -v helm)" ]; then + echo 'Error: helm is not installed.' >&2 + + curl -Ls https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | DESIRED_VERSION=v3.2.1 bash +fi + +# install required plugins + +helm plugin list | grep -cE ^diff>/dev/null || helm plugin install "https://github.com/databus23/helm-diff" --version 3.1.1 + +# check helmsman + +if ! [ -x "$(command -v helmsman)" ]; then + echo 'Error: helmsman is not installed.' >&2 + + curl -Ls https://github.com/Praqma/helmsman/releases/download/v3.2.0/helmsman_3.2.0_linux_amd64.tar.gz | tar zx + + chmod +x helmsman +fi + +# run helmsman + +./helmsman -apply -f ./infra/desired-state/desired-state.yaml -debug -no-ns