diff --git a/.prow/scripts/sync-helm-charts.sh b/.prow/scripts/sync-helm-charts.sh new file mode 100755 index 0000000000..918979dd3f --- /dev/null +++ b/.prow/scripts/sync-helm-charts.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# Script to sync local charts to remote helm repository in Google Cloud Storage +# Copied from: https://github.com/helm/charts/blob/master/test/repo-sync.sh + +set -o errexit +set -o nounset +set -o pipefail + +log_error() { + printf '\e[31mERROR: %s\n\e[39m' "$1" >&2 +} + +# Assume working directory is Feast repository root folder +repo_dir=infra/charts +bucket=gs://feast-charts +repo_url=https://feast-charts.storage.googleapis.com/ +sync_dir=/tmp/syncdir +index_dir=/tmp/indexdir + +rm -rf $sync_dir $index_dir + +echo "Syncing repo '$repo_dir'..." + +mkdir -p "$sync_dir" +if ! gsutil cp "$bucket/index.yaml" "$index_dir/index.yaml"; then + log_error "Exiting because unable to copy index locally. Not safe to proceed." + exit 1 +fi + +exit_code=0 + +for dir in "$repo_dir"/*; do + if helm dependency build "$dir"; then + helm package --destination "$sync_dir" "$dir" + else + log_error "Problem building dependencies. Skipping packaging of '$dir'." + exit_code=1 + fi +done + +if helm repo index --url "$repo_url" --merge "$index_dir/index.yaml" "$sync_dir"; then + # Move updated index.yaml to sync folder so we don't push the old one again + mv -f "$sync_dir/index.yaml" "$index_dir/index.yaml" + + gsutil -m rsync "$sync_dir" "$bucket" + + # Make sure index.yaml is synced last + gsutil cp "$index_dir/index.yaml" "$bucket" +else + log_error "Exiting because unable to update index. Not safe to push update." + exit 1 +fi + +ls -l "$sync_dir" + +exit "$exit_code" \ No newline at end of file diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 15881b50e6..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM openjdk:8-jre-alpine -ARG REVISION=0.3.0-dev -COPY ./serving/target/feast-serving-$REVISION.jar /usr/share/feast/feast-serving.jar -ENTRYPOINT ["java", \ - "-XX:+UseG1GC", \ - "-XX:+UseStringDeduplication", \ - "-XX:+UnlockExperimentalVMOptions", \ - "-XX:+UseCGroupMemoryLimitForHeap", \ - "-jar", "/usr/share/feast/feast-serving.jar"] \ No newline at end of file diff --git a/infra/charts/README.md b/infra/charts/README.md deleted file mode 100644 index 88edc6f4ae..0000000000 --- a/infra/charts/README.md +++ /dev/null @@ -1,119 +0,0 @@ -# Helm charts -This chart adds all the components required to run feast, sans the stores to which you might want to ingest your features. Those will have to be deployed seperately. - -## Installing the chart - -``` -helm dep update -helm install --name feast . -``` - -## Requirements - -### External Permissions - -The feast deployment requires access to the following components: - -- `gcs`: read/write - -As well as the following, depending on your use case: - -- `bigquery`: read/write -- `bigtable`: read/write/tableAdmin - -The recommended way to give your deployment these permissions is to deploy Feast on a cluster with the required permissions. - -## Components - -The components included in this chart are: - -- `feast-core`: the main API controller of feast -- `feast-metadata`: postgres db that persists feast's metadata - - The stable postgres chart is being used for this deployment - any of the parameters described [here](https://github.com/helm/charts/tree/master/stable/postgresql) can be overridden to customise your postgres deployment. -- `feast-serving`: service that serves up features from the various serving stores - -Components that Feast supports, but this installation will not include are: - -- Serving Storage - - `Redis` or `Bigtable` dbs for feature serving storage -- [TICK stack](https://www.influxdata.com/time-series-platform/) for metrics monitoring - - Set `statsd.host` and `statsd.port` to direct job metrics to your metrics store. - - Note that if you do not provision a metrics store, feast will only retain the latest metrics from your jobs. -- [Jaeger tracing](www.jaegertracing.io) for serving performance. - - Set `serving.jaeger.enabled` to `true`, and configure the following parameters: - - `serving.jaeger.host` - - `serving.jaeger.port` - - `serving.jaeger.options.samplerType` - - `serving.jaeger.options.samplerParam` - -## Uninstalling Feast - -To uninstall the `feast` deployment: -``` -helm del feast -kubectl delete persistentvolumeclaim feast-postgresql -``` - -## Configuration -The following table lists the configurable parameters of the Feast chart and their default values. - -| var | desc | default | -| --- | --- | --- | -| `core.image.registry` | core docker image registry | feast | -| `core.image.repository` | core docker image repository | feast-core | -| `core.image.tag` | core docker image version | 0.1.0 | -| `core.jobs.monitoring.initialDelay` | delay before a job starts to be monitored in ms | 60000 | -| `core.jobs.monitoring.period` | polling interval for jobs monitoring in ms | 5000 | -| `core.jobs.options` | additional options to be provided to the beam job. Should be a char escaped json k-v object | {} | -| `core.jobs.runner` | beam job runner - one of `DirectRunner`, `FlinkRunner` or `DataflowRunner` | DirectRunner | -| `core.jobs.workspace` | workspace path for ingestion jobs, used for separate job workspaces to share importJobSpecs.yaml with ingestion and for writing errors to if no default errors store is configured | nil | -| `core.replicaCount` | core deployment replica count | 3 | -| `core.resources.limits.cpu` | core cpu limits | 1 | -| `core.resources.limits.memory` | core memory limits | 2G | -| `core.resources.requests.cpu` | core cpu requested | 1 | -| `core.resources.requests.memory` | core memory requested | 2G | -| `core.service.extIPAdr` | Internal load balancer IP Address for core, required so jobs on external runners can connect to core | nil | -| `core.service.loadBalancerSourceRanges` | IP source ranges that will have access to core. If not set, will default to 0.0.0.0/0 | nil | -| `core.service.grpc.port` | core service exposed grpc port | 8433 | -| `core.service.grpc.targetPort` | core service target grpc port | 8433 | -| `core.service.http.port` | core service exposed http port | 80 | -| `core.service.http.targetPort` | core service target http port | 8080 | -| `core.projectId` | GCP project ID core service resides at | gcp-project-id | -| `core.trainingDatasetPrefix` | prefix for training datasets created in bq | fs | -| `dataflow.location` | desired dataflow's region | nil | -| `dataflow.projectID` | desired dataflow's project id | nil | -| `serving.config.maxEntityPerBatch` | max entities that can be requested at a time | 2000 | -| `serving.config.maxNumberOfThread` | max number of threads per instance of serving | 256 | -| `serving.config.redisPool.maxIdle` | max idle connections to redis | 16 | -| `serving.config.redisPool.maxSize` | max number of connections to redis | 256 | -| `serving.config.timeout` | request timeout in seconds | 5 | -| `serving.image.registry` | serving docker image registry | feast | -| `serving.image.repository` | serving docker image repository | feast-serving | -| `serving.image.tag` | serving docker image version | 0.1.0 | -| `serving.replicaCount` | serving replica count | 4 | -| `serving.resources.limits.cpu` | serving cpu limits | 1 | -| `serving.resources.limits.memory` | serving memory limits | 2G | -| `serving.resources.requests.cpu` | serving cpu requested | 1 | -| `serving.resources.requests.memory` | serving memory requested | 2G | -| `serving.service.grpc.port` | serving service exposed grpc port | 8433 | -| `serving.service.grpc.targetPort` | serving service target grpc port | 8433 | -| `serving.service.http.port` | serving service exposed http port | 80 | -| `serving.service.http.targetPort` | serving service target http port | 8080 | -| `serving.service.extIPAdr` | Internal load balancer IP Address for serving, required so jobs on external runners can connect to the service | nil | -| `serving.service.loadBalancerSourceRanges` | IP source ranges that will have access to serving. If not set, will default to 0.0.0.0/0 | nil | -| `serviceAccount.name` | service account secret name to mount to deployments | nil | -| `serviceAccount.key` | service account secret key to mount to deployments | nil | -| `statsd.host` | host of statsd daemon for job metrics to be sent to | nil | -| `statsd.port` | port of statsd daemon for job metrics to be sent to | nil | -| `store.errors.type` | type of default errors store to write errors to. One of `stdout`, `stderr`, `file.json` | nil | -| `store.errors.options` | additional options for the default error store in json string format | `{}` | -| `store.serving.type` | type of default serving store to write errors to. One of `redis`, `bigtable` | nil | -| `store.serving.options` | additional options for the default serving store in json string format | `{}` | -| `store.warehouse.type` | type of default warehouse store to write errors to. One of `bigquery`, `file.json` | nil | -| `store.warehouse.options` | additional options for the default warehouse store in json string format | `{}` | -| `postgresql.provision` | Provision PostgreSQL | true | -| `postgresql.postgresPassword` | specify password if you want the postgres password secret to be generated | nil | -| `postgresql.resources.requests.cpu` | postgres requested cpu | 100m | -| `postgresql.resources.requests.memory` | postgres requested memory | 256Mi | -| `redis.provision` | Provision Redis instance | true | -| `redis.name` | Helm release name for the Redis instance | feast-redis | \ No newline at end of file diff --git a/infra/charts/feast-core/Chart.yaml b/infra/charts/feast-core/Chart.yaml deleted file mode 100644 index d1b4a798bc..0000000000 --- a/infra/charts/feast-core/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -appVersion: "0.3.0" -description: A Helm chart for feast core -name: feast-core -version: 0.3.0 diff --git a/infra/charts/feast-core/charts/postgresql-3.17.0.tgz b/infra/charts/feast-core/charts/postgresql-3.17.0.tgz deleted file mode 100644 index 3fe96f84b2..0000000000 Binary files a/infra/charts/feast-core/charts/postgresql-3.17.0.tgz and /dev/null differ diff --git a/infra/charts/feast-core/requirements.lock b/infra/charts/feast-core/requirements.lock deleted file mode 100644 index d3574f1306..0000000000 --- a/infra/charts/feast-core/requirements.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: postgresql - repository: https://kubernetes-charts.storage.googleapis.com - version: 3.17.0 -digest: sha256:b771dedc573e1e14de59b6c0ad2297fbfa9f70b72a36628b6ba0dc0fd6c1a262 -generated: "2019-10-09T20:05:27.227845+08:00" diff --git a/infra/charts/feast-core/requirements.yaml b/infra/charts/feast-core/requirements.yaml deleted file mode 100644 index de2a8310a8..0000000000 --- a/infra/charts/feast-core/requirements.yaml +++ /dev/null @@ -1,5 +0,0 @@ -dependencies: -- name: postgresql - version: 3.17.0 - repository: "@stable" - condition: postgres.provision \ No newline at end of file diff --git a/infra/charts/feast-core/templates/core-configmap.yaml b/infra/charts/feast-core/templates/core-configmap.yaml deleted file mode 100644 index 3c48260683..0000000000 --- a/infra/charts/feast-core/templates/core-configmap.yaml +++ /dev/null @@ -1,48 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "feast-core.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "feast-core.name" . }} - component: core - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -data: - application.yml: | - - # Please see gojek/feast/core/src/main/resources/application.yml for an explanation of config - server: - port: {{ .Values.service.http.targetPort }} - - grpc: - port: {{ .Values.service.grpc.targetPort }} - - feast: - jobs: - runner: {{ .Values.jobs.runner }} - options: -{{- if .Values.jobs.options }} -{{ toYaml .Values.jobs.options | indent 10 }} -{{- end }} - metrics: -{{- if .Values.jobs.metrics.enabled }} -{{ toYaml .Values.jobs.metrics | indent 10 }} -{{ end }} - stream: - # Feature stream type. Only kafka is supported. - type: {{ .Values.stream.type }} - # Feature stream options. - options: -{{- if eq .Values.stream.type "kafka" }} - bootstrapServers: {{ .Values.stream.options.bootstrapServers }} - replicationFactor: {{ .Values.stream.options.replicationFactor }} - partitions: {{ .Values.stream.options.partitions }} -{{- end }} - - spring: - datasource: - url: jdbc:postgresql://{{ template "postgresql.host" . }}:{{ .Values.postgresql.service.port }}/{{ .Values.postgresql.postgresqlDatabase }} - username: ${DB_USERNAME:{{ .Values.postgresql.postgresqlUsername }}} - password: ${DB_PASSWORD:{{ .Values.postgresql.postgresqlPassword }}} diff --git a/infra/charts/feast-core/templates/core-deploy.yaml b/infra/charts/feast-core/templates/core-deploy.yaml deleted file mode 100644 index 9686e4e4c9..0000000000 --- a/infra/charts/feast-core/templates/core-deploy.yaml +++ /dev/null @@ -1,110 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "feast-core.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "feast-core.name" . }} - component: core - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - app: {{ template "feast-core.name" . }} - component: core - release: {{ .Release.Name }} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: {{ .Values.rollingUpdate.maxSurge }} - maxUnavailable: {{ .Values.rollingUpdate.maxUnavailable }} - template: - metadata: - labels: - app: {{ template "feast-core.name" . }} - component: core - release: {{ .Release.Name }} - spec: -{{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} - containers: - - name: {{ template "feast-core.name" . }} - image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: - - java - - "-Dspring.config.additional-location=/usr/share/feast/config/" - - "-XX:+UnlockExperimentalVMOptions" - - "-XX:+UseCGroupMemoryLimitForHeap" - - "-jar" - - "/usr/share/feast/feast-core.jar" - ports: - - containerPort: {{ .Values.service.http.targetPort }} - name: "http" - - containerPort: {{ .Values.service.grpc.targetPort }} - name: "grpc" - {{- if .Values.livenessProbe.enabled }} - livenessProbe: - httpGet: - path: /ping - port: {{ .Values.service.http.targetPort }} - scheme: HTTP - initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - failureThreshold: {{ .Values.livenessProbe.failureThreshold }} - {{ end }} - readinessProbe: - httpGet: - path: /healthz - port: {{ .Values.service.http.targetPort }} - scheme: HTTP - initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 3 - failureThreshold: {{ .Values.readinessProbe.failureThreshold }} - resources: - requests: - cpu: {{ .Values.resources.requests.cpu }} - memory: {{ .Values.resources.requests.memory }} - {{- if .Values.resources.limits }} - limits: - cpu: {{ .Values.resources.limits.cpu }} - memory: {{ .Values.resources.limits.memory }} - {{- end }} - volumeMounts: - - name: feast-core-config - mountPath: /usr/share/feast/config - {{- if .Values.serviceAccount }} - - name: "{{ .Values.serviceAccount.name }}" - mountPath: "/etc/gcloud/service-accounts" - readOnly: true - {{- end }} - env: - - name: DB_HOST - value: {{ template "postgresql.host" . }} - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ template "postgresql.secretName" . }} - key: postgresql-password - {{- if .Values.serviceAccount }} - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/gcloud/service-accounts/{{ .Values.serviceAccount.key }} - {{- end }} - volumes: - - name: feast-core-config - configMap: - name: {{ template "feast-core.fullname" . }} - {{- if .Values.serviceAccount }} - - name: "{{ .Values.serviceAccount.name }}" - secret: - secretName: "{{ .Values.serviceAccount.name }}" - {{- end }} \ No newline at end of file diff --git a/infra/charts/feast-core/values.yaml b/infra/charts/feast-core/values.yaml deleted file mode 100644 index fbf91ddf02..0000000000 --- a/infra/charts/feast-core/values.yaml +++ /dev/null @@ -1,101 +0,0 @@ -global: - postgresql: {} - -replicaCount: 1 - -# nameOverride: "" -# fullnameOverride: "" - -image: - registry: gcr.io/kf-feast - repository: feast-core - tag: 0.3.0 - pullPolicy: IfNotPresent - -jobs: - runner: DirectRunner - options: {} - ## If using DataflowRunner, the following options are necessary: - # project: my-gcp-project - # tempLocation: gs://my-feast-bucket/temp - # region: us-central1 - # subnetwork: default - metrics: - enabled: false - ## Type of metrics sink. Only prometheus is currently supported. - # type: prometheus - ## Host of the metrics sink. In the case of prometheus, this is the host of the prometheus - ## pushGateway to sink metrics to. - # host: localhost - ## Port of the metrics sink. In the case of prometheus, this is the port of the prometheus - ## pushGateway to sink metrics to. - # port: 9091 - -stream: - type: kafka - options: - bootstrapServers: kafka:9092 - replicationFactor: 2 - partitions: 4 - -service: - # annotations: [] - http: - port: 80 - targetPort: 8080 - grpc: - port: 6565 - targetPort: 6565 - type: ClusterIP - # loadBalancerIP: - # loadBalancerSourceRanges: - # - 10.0.0.0/8 - port: 80 - -rollingUpdate: - maxSurge: 2 - maxUnavailable: 1 - -livenessProbe: - enabled: false - # initialDelaySeconds: 30 - # failureThreshold: 3 -readinessProbe: - initialDelaySeconds: 30 - failureThreshold: 3 - -## A service account if using the dataflow runner or BigQuery. -# serviceAccount: - # name: my-service-account - -postgresql: - # provision a postgres instance for feast metadata - provision: false - ## These opts will have to be set regardless of whether you use helm to provision the - ## postgres instance. If using an existing postgres instance, they should contain the - ## credentials necessary to connect to that instance. - postgresqlUsername: postgres - # existingSecret: postgresql-password - ## If unset, feast will default to some postgres instance in the same namespace as the - ## feast core deployment. - # host: - postgresqlDatabase: postgres - -ingress: - enabled: false - annotations: {} - # kubernetes.io/ingress.class: nginx - hosts: - # - host: chart-example.local - # port: http - -resources: {} - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -nodeSelector: {} - diff --git a/infra/charts/feast-serving/Chart.yaml b/infra/charts/feast-serving/Chart.yaml deleted file mode 100644 index 8c620ff039..0000000000 --- a/infra/charts/feast-serving/Chart.yaml +++ /dev/null @@ -1,5 +0,0 @@ -apiVersion: v1 -appVersion: "0.3.0" -description: A Helm chart for feast serving -name: feast-serving -version: 0.3.0 diff --git a/infra/charts/feast-serving/templates/serving-configmap.yaml b/infra/charts/feast-serving/templates/serving-configmap.yaml deleted file mode 100644 index d6dabfb433..0000000000 --- a/infra/charts/feast-serving/templates/serving-configmap.yaml +++ /dev/null @@ -1,69 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "feast-serving.fullname" . }}-application-config - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "feast-serving.name" . }} - component: serving - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -data: - application.yml: | - feast: - # GRPC service address for Feast Core - # Feast Serving requires connection to Feast Core to retrieve and reload Feast metadata (e.g. FeatureSpecs, Store information) - core-host: {{ .Values.core.host }} - core-port: {{ .Values.core.port }} - - tracing: - # If true, Feast will provide tracing data (using OpenTracing API) for various RPC method calls - # which can be useful to debug performance issues and perform benchmarking - enabled: {{ .Values.tracing.enabled }} - {{- if .Values.tracing.enabled }} - # Only Jaeger tracer is supported currently - # https://opentracing.io/docs/supported-tracers/ - tracer-name: {{ .Values.tracing.tracerName }} - # The service name identifier for the tracing data - service-name: {{ .Values.tracing.serviceName }} - {{ end }} - - store: - # Path containing the store configuration for this serving store. - config-path: /usr/share/feast/config/store.yml - {{- if eq .Values.store.config.type "REDIS" }} - # If serving redis, the redis pool max size - redis-pool-max-size: {{ .Values.store.redisPool.maxSize }} - # If serving redis, the redis pool max idle conns - redis-pool-max-idle: {{ .Values.store.redisPool.maxIdle }} - {{ end }} - - {{- if eq .Values.store.config.type "BIGQUERY" }} - jobs: - # job-staging-location specifies the URI to store intermediate files for batch serving. - # Feast Serving client is expected to have read access to this staging location - # to download the batch features. - # - # For example: gs://mybucket/myprefix - # Please omit the trailing slash in the URI. - staging-location: {{ .Values.store.jobs.stagingLocation }} - # Type of store to store job metadata. This only needs to be set if the - # serving store type is Bigquery. - store-type: {{ .Values.store.jobs.storeType }} - # Job store connection options. If the job store is redis, the redis instance host and port are required. - store-options: -{{ toYaml .Values.store.jobs.storeOptions | indent 10 }} - {{ end }} - - grpc: - # The port number Feast Serving GRPC service should listen on - # It is set default to 6566 so it does not conflict with the GRPC server on Feast Core - # which defaults to port 6565 - port: {{ .Values.service.grpc.targetPort }} - - server: - # The port number on which the Tomcat webserver that serves REST API endpoints should listen - # It is set by default to 8080 so it does not conflict with Tomcat webserver on Feast Core - # if both Feast Core and Serving are running on the same machine - port: {{ .Values.service.http.targetPort }} \ No newline at end of file diff --git a/infra/charts/feast-serving/templates/serving-deploy.yaml b/infra/charts/feast-serving/templates/serving-deploy.yaml deleted file mode 100644 index 6cdb2a422a..0000000000 --- a/infra/charts/feast-serving/templates/serving-deploy.yaml +++ /dev/null @@ -1,107 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ template "feast-serving.fullname" . }} - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "feast-serving.name" . }} - component: serving - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -spec: - replicas: {{ .Values.replicaCount }} - selector: - matchLabels: - app: {{ template "feast-serving.name" . }} - component: serving - release: {{ .Release.Name }} - strategy: - type: RollingUpdate - rollingUpdate: - maxSurge: {{ .Values.rollingUpdate.maxSurge }} - maxUnavailable: {{ .Values.rollingUpdate.maxUnavailable }} - template: - metadata: - creationTimestamp: null - labels: - app: {{ template "feast-serving.name" . }} - component: serving - release: {{ .Release.Name }} - spec: -{{- if .Values.nodeSelector }} - nodeSelector: -{{ toYaml .Values.nodeSelector | indent 8 }} -{{- end }} - containers: - - name: {{ template "feast-serving.name" . }} - image: "{{ .Values.image.registry }}/{{ .Values.image.repository }}:{{ .Values.image.tag }}" - imagePullPolicy: {{ .Values.image.pullPolicy }} - command: - - java - - "-Dspring.config.additional-location=/usr/share/feast/config/" - - "-XX:+UnlockExperimentalVMOptions" - - "-XX:+UseCGroupMemoryLimitForHeap" - - "-jar" - - "/usr/share/feast/feast-serving.jar" - ports: - - containerPort: {{ .Values.service.grpc.targetPort }} - name: grpc - - containerPort: {{ .Values.service.http.targetPort }} - name: http - {{- if .Values.livenessProbe }} - livenessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:{{ .Values.service.grpc.targetPort }}"] - initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} - periodSeconds: 10 - successThreshold: 1 - timeoutSeconds: 5 - failureThreshold: {{ .Values.livenessProbe.failureThreshold }} - {{ end }} - readinessProbe: - exec: - command: ["/bin/grpc_health_probe", "-addr=:{{ .Values.service.grpc.targetPort }}"] - initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} - periodSeconds: 5 - successThreshold: 1 - timeoutSeconds: 3 - failureThreshold: {{ .Values.readinessProbe.failureThreshold }} - resources: - requests: - cpu: "{{ .Values.resources.requests.cpu }}" - memory: "{{ .Values.resources.requests.memory }}" - {{- if .Values.resources.limits }} - limits: - cpu: "{{ .Values.resources.limits.cpu }}" - memory: "{{ .Values.resources.limits.memory }}" - {{- end }} - env: - {{- if .Values.serviceAccount }} - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /etc/gcloud/service-accounts/{{ .Values.serviceAccount.key }} - {{- end }} - volumeMounts: - - name: feast-serving-application-config - mountPath: /usr/share/feast/config/application.yml - subPath: application.yml - - name: feast-serving-store-config - mountPath: /usr/share/feast/config/store.yml - subPath: store.yml - {{- if .Values.serviceAccount }} - - name: "{{ .Values.serviceAccount.name }}" - mountPath: "/etc/gcloud/service-accounts" - readOnly: true - {{- end }} - volumes: - - name: feast-serving-application-config - configMap: - name: {{ template "feast-serving.fullname" . }}-application-config - - name: feast-serving-store-config - configMap: - name: {{ template "feast-serving.fullname" . }}-store-config - {{- if .Values.serviceAccount }} - - name: "{{ .Values.serviceAccount.name }}" - secret: - secretName: "{{ .Values.serviceAccount.name }}" - {{- end }} diff --git a/infra/charts/feast-serving/templates/store-configmap.yaml b/infra/charts/feast-serving/templates/store-configmap.yaml deleted file mode 100644 index 5de7ab29aa..0000000000 --- a/infra/charts/feast-serving/templates/store-configmap.yaml +++ /dev/null @@ -1,14 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: {{ template "feast-serving.fullname" . }}-store-config - namespace: {{ .Release.Namespace }} - labels: - app: {{ template "feast-serving.name" . }} - component: serving - chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} - release: {{ .Release.Name }} - heritage: {{ .Release.Service }} -data: - store.yml: | -{{ toYaml .Values.store.config | indent 4 }} \ No newline at end of file diff --git a/infra/charts/feast-serving/values.yaml b/infra/charts/feast-serving/values.yaml deleted file mode 100644 index 379a704534..0000000000 --- a/infra/charts/feast-serving/values.yaml +++ /dev/null @@ -1,98 +0,0 @@ -replicaCount: 1 - -image: - registry: gcr.io/kf-feast - repository: feast-serving - tag: 0.3.0 - pullPolicy: IfNotPresent - -# nameOverride: "" -# fullnameOverride: "" -core: - host: feast-core - port: 6565 - -store: - config: {} - # name: serving - # type: REDIS - # redis_config: - # host: redis.svc.local - # port: 6379 - # subscriptions: - # - name: .* - # version: ">0" - - ## For redis serving. Configuration for the redis connection pool. - # redisPool: - # maxSize: 128 - # maxIdle: 8 - - ## Job configuration is necessary if this serving store is used to - ## facilitate batch retrieval of features, which involve async jobs. - jobs: {} - # stagingLocation: gs://feast-bucket/bq/staging - # storeType: REDIS - # storeOptions: - # host: redis.svc.local - # port: 6379 - -tracing: - enabled: false - # tracer-name: jaeger - # service-name: feast-serving - -rollingUpdate: - maxSurge: 2 - maxUnavailable: 1 - -service: - # annotations: [] - http: - port: 80 - targetPort: 8080 - grpc: - port: 6566 - targetPort: 6566 - type: ClusterIP - # loadBalancerIP: - # loadBalancerSourceRanges: - # - 10.0.0.0/8 - port: 80 - -rollingUpdate: - maxSurge: 2 - maxUnavailable: 1 - -livenessProbe: - # initialDelaySeconds: 30 - # failureThreshold: 3 -readinessProbe: - initialDelaySeconds: 30 - failureThreshold: 3 - -## A service account if using the dataflow runner or BigQuery. -# serviceAccount: - # name: my-service-account - -ingress: - enabled: false - annotations: {} - # kubernetes.io/ingress.class: nginx - hosts: - # - host: chart-example.local - # port: http - -resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - -nodeSelector: {} diff --git a/infra/charts/feast/.helmignore b/infra/charts/feast/.helmignore index f0c1319444..50af031725 100644 --- a/infra/charts/feast/.helmignore +++ b/infra/charts/feast/.helmignore @@ -19,3 +19,4 @@ .project .idea/ *.tmproj +.vscode/ diff --git a/infra/charts/feast/Chart.yaml b/infra/charts/feast/Chart.yaml index 9eb45964a6..30ccf81091 100644 --- a/infra/charts/feast/Chart.yaml +++ b/infra/charts/feast/Chart.yaml @@ -1,5 +1,4 @@ apiVersion: v1 -appVersion: "0.1.0" description: A Helm chart to install Feast on kubernetes name: feast -version: 0.1.0 +version: 0.3.0-alpha.1 diff --git a/infra/charts/feast/README.md b/infra/charts/feast/README.md new file mode 100644 index 0000000000..03ba53a574 --- /dev/null +++ b/infra/charts/feast/README.md @@ -0,0 +1,179 @@ +# Feast Chart + +This directory provides the Helm chart for Feast installation. + +This chart installs Feast Core and Feast Serving components of Feast, along with +the required and optional dependencies. Components and dependencies can be +enabled or disabled by changing the corresponding `enabled` flag. Feast Core and +Feast Serving are subcharts of this parent Feast chart. The structure of the charts +are as follows: + +``` +feast // top level feast chart +│ +├── feast-core // feast-core subchart +│ ├── postgresql // Postgresql dependency for feast-core (Feast database) +│ └── kafka // Kafka dependency for feast-core (default stream source) +│ +├── feast-serving-online // feast-serving subchart +│ └── redis // Redis dependency for installation of store together with feast-serving +│ +└── feast-serving-batch // feast-serving subchart +``` + +## Prerequisites +- Kubernetes 1.13 or newer cluster +- Helm 2.15.2 or newer + +## Resources Required +The chart deploys pods that consume minimum resources as specified in the resources configuration parameter. + +## Installing the Chart + +Add repository for Feast chart: +```bash +helm repo add feast-charts https://feast-charts.storage.googleapis.com +helm repo update +``` + +Install Feast release with minimal features, without batch serving and persistency: +```bash +RELEASE_NAME=demo +helm install feast-charts/feast --name $RELEASE_NAME --version 0.3.0-alpha.1 -f values-demo.yaml +``` + +Install Feast release for typical use cases, with batch and online serving: +```bash +# To install Feast Batch serving, BigQuery and Google Cloud service account +# is required. The service account needs to have these roles: +# - bigquery.dataEditor +# - bigquery.jobUser +# +# Assuming a service account JSON file has been downloaded to /home/user/key.json, +# run the following command to create a secret in Kubernetes +# (make sure the file name is called key.json): +kubectl create secret generic feast-gcp-service-account --from-file=/home/user/key.json + +# Set these required configuration in Feast Batch Serving +STAGING_LOCATION=gs://bucket/path +PROJECT_ID=google-cloud-project-id +DATASET_ID=bigquery-dataset-id + +# Install the Helm release using default values.yaml +helm install feast-charts/feast --name feast --version 0.3.0-alpha.1 \ + --set feast-serving-batch."application\.yaml".feast.jobs.staging-location=$STAGING_LOCATION \ + --set feast-serving-batch."store\.yaml".bigquery_config.project_id=$PROJECT_ID \ + --set feast-serving-batch."store\.yaml".bigquery_config.dataset_id=$DATASET_ID +``` + +## Parameters + +The following table lists the configurable parameters of the Feast chart and their default values. + +| Parameter | Description | Default +| --------- | ----------- | ------- +| `feast-core.enabled` | Flag to install Feast Core | `true` +| `feast-core.postgresql.enabled` | Flag to install Postgresql as Feast database | `true` +| `feast-core.postgresql.postgresqlDatabase` | Name of the database used by Feast Core | `feast` +| `feast-core.postgresql.postgresqlUsername` | Username to authenticate to Feast database | `postgres` +| `feast-core.postgresql.postgresqlPassword` | Passsword to authenticate to Feast database | `password` +| `feast-core.kafka.enabled` | Flag to install Kafka as the default source for Feast | `true` +| `feast-core.kafka.topics[0].name` | Default topic name in Kafka| `feast` +| `feast-core.kafka.topics[0].replicationFactor` | No of replication factor for the topic| `1` +| `feast-core.kafka.topics[0].partitions` | No of partitions for the topic | `1` +| `feast-core.replicaCount` | No of pods to create | `1` +| `feast-core.image.repository` | Repository for Feast Core Docker image | `gcr.io/kf-feast/feast-core` +| `feast-core.image.tag` | Tag for Feast Core Docker image | `0.3.0` +| `feast-core.image.pullPolicy` | Image pull policy for Feast Core Docker image | `IfNotPresent` +| `feast-core.application.yaml` | Configuration for Feast Core application | Refer to this [link](charts/feast-core/values.yaml) +| `feast-core.springConfigMountPath` | Directory to mount application.yaml | `/etc/feast/feast-core` +| `feast-core.gcpServiceAccount.useExistingSecret` | Flag to use existing secret for GCP service account | `false` +| `feast-core.gcpServiceAccount.existingSecret.name` | Secret name for the service account | `feast-gcp-service-account` +| `feast-core.gcpServiceAccount.existingSecret.key` | Secret key for the service account | `key.json` +| `feast-core.gcpServiceAccount.mountPath` | Directory to mount the JSON key file | `/etc/gcloud/service-accounts` +| `feast-core.jvmOptions` | Options for the JVM | `[]` +| `feast-core.livenessProbe.enabled` | Flag to enable liveness probe | `true` +| `feast-core.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `60` +| `feast-core.livenessProbe.periodSeconds` | How often to perform the probe | `10` +| `feast-core.livenessProbe.timeoutSeconds` | Timeout duration for the probe | `5` +| `feast-core.livenessProbe.successThreshold` | Minimum no of consecutive successes for the probe to be considered successful | `1` +| `feast-core.livenessProbe.failureThreshold` | Minimum no of consecutive failures for the probe to be considered failed | `5` +| `feast-core.readinessProbe.enabled` | Flag to enable readiness probe | `true` +| `feast-core.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` +| `feast-core.readinessProbe.periodSeconds` | How often to perform the probe | `10` +| `feast-core.readinessProbe.timeoutSeconds` | Timeout duration for the probe | `10` +| `feast-core.readinessProbe.successThreshold` | Minimum no of consecutive successes for the probe to be considered successful | `1` +| `feast-core.service.type` | Kubernetes Service Type | `ClusterIP` +| `feast-core.http.port` | Kubernetes Service port for HTTP request| `80` +| `feast-core.http.targetPort` | Container port for HTTP request | `8080` +| `feast-core.grpc.port` | Kubernetes Service port for GRPC request| `6565` +| `feast-core.grpc.targetPort` | Container port for GRPC request| `6565` +| `feast-core.resources` | CPU and memory allocation for the pod | `{}` +| `feast-serving-online.enabled` | Flag to install Feast Online Serving | `true` +| `feast-serving-online.redis.enabled` | Flag to install Redis in Feast Serving | `false` +| `feast-serving-online.redis.usePassword` | Flag to use password to access Redis | `false` +| `feast-serving-online.redis.cluster.enabled` | Flag to enable Redis cluster | `false` +| `feast-serving-online.core.enabled` | Flag for Feast Serving to use Feast Core in the same Helm release | `true` +| `feast-serving-online.replicaCount` | No of pods to create | `1` +| `feast-serving-online.image.repository` | Repository for Feast Serving Docker image | `gcr.io/kf-feast/feast-serving` +| `feast-serving-online.image.tag` | Tag for Feast Serving Docker image | `0.3.0` +| `feast-serving-online.image.pullPolicy` | Image pull policy for Feast Serving Docker image | `IfNotPresent` +| `feast-serving-online.application.yaml` | Application configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml) +| `feast-serving-online.store.yaml` | Store configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml) +| `feast-serving-online.springConfigMountPath` | Directory to mount application.yaml and store.yaml | `/etc/feast/feast-serving` +| `feast-serving-online.gcpServiceAccount.useExistingSecret` | Flag to use existing secret for GCP service account | `false` +| `feast-serving-online.gcpServiceAccount.existingSecret.name` | Secret name for the service account | `feast-gcp-service-account` +| `feast-serving-online.gcpServiceAccount.existingSecret.key` | Secret key for the service account | `key.json` +| `feast-serving-online.gcpServiceAccount.mountPath` | Directory to mount the JSON key file | `/etc/gcloud/service-accounts` +| `feast-serving-online.jvmOptions` | Options for the JVM | `[]` +| `feast-serving-online.livenessProbe.enabled` | Flag to enable liveness probe | `true` +| `feast-serving-online.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `60` +| `feast-serving-online.livenessProbe.periodSeconds` | How often to perform the probe | `10` +| `feast-serving-online.livenessProbe.timeoutSeconds` | Timeout duration for the probe | `5` +| `feast-serving-online.livenessProbe.successThreshold` | Minimum no of consecutive successes for the probe to be considered successful | `1` +| `feast-serving-online.livenessProbe.failureThreshold` | Minimum no of consecutive failures for the probe to be considered failed | `5` +| `feast-serving-online.readinessProbe.enabled` | Flag to enable readiness probe | `true` +| `feast-serving-online.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` +| `feast-serving-online.readinessProbe.periodSeconds` | How often to perform the probe | `10` +| `feast-serving-online.readinessProbe.timeoutSeconds` | Timeout duration for the probe | `10` +| `feast-serving-online.readinessProbe.successThreshold` | Minimum no of consecutive successes for the probe to be considered successful | `1` +| `feast-serving-online.service.type` | Kubernetes Service Type | `ClusterIP` +| `feast-serving-online.http.port` | Kubernetes Service port for HTTP request| `80` +| `feast-serving-online.http.targetPort` | Container port for HTTP request | `8080` +| `feast-serving-online.grpc.port` | Kubernetes Service port for GRPC request| `6566` +| `feast-serving-online.grpc.targetPort` | Container port for GRPC request| `6566` +| `feast-serving-online.resources` | CPU and memory allocation for the pod | `{}` +| `feast-serving-batch.enabled` | Flag to install Feast Batch Serving | `true` +| `feast-serving-batch.redis.enabled` | Flag to install Redis in Feast Serving | `false` +| `feast-serving-batch.redis.usePassword` | Flag to use password to access Redis | `false` +| `feast-serving-batch.redis.cluster.enabled` | Flag to enable Redis cluster | `false` +| `feast-serving-batch.core.enabled` | Flag for Feast Serving to use Feast Core in the same Helm release | `true` +| `feast-serving-batch.replicaCount` | No of pods to create | `1` +| `feast-serving-batch.image.repository` | Repository for Feast Serving Docker image | `gcr.io/kf-feast/feast-serving` +| `feast-serving-batch.image.tag` | Tag for Feast Serving Docker image | `0.3.0` +| `feast-serving-batch.image.pullPolicy` | Image pull policy for Feast Serving Docker image | `IfNotPresent` +| `feast-serving-batch.application.yaml` | Application configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml) +| `feast-serving-batch.store.yaml` | Store configuration for Feast Serving | Refer to this [link](charts/feast-serving/values.yaml) +| `feast-serving-batch.springConfigMountPath` | Directory to mount application.yaml and store.yaml | `/etc/feast/feast-serving` +| `feast-serving-batch.gcpServiceAccount.useExistingSecret` | Flag to use existing secret for GCP service account | `false` +| `feast-serving-batch.gcpServiceAccount.existingSecret.name` | Secret name for the service account | `feast-gcp-service-account` +| `feast-serving-batch.gcpServiceAccount.existingSecret.key` | Secret key for the service account | `key.json` +| `feast-serving-batch.gcpServiceAccount.mountPath` | Directory to mount the JSON key file | `/etc/gcloud/service-accounts` +| `feast-serving-batch.jvmOptions` | Options for the JVM | `[]` +| `feast-serving-batch.livenessProbe.enabled` | Flag to enable liveness probe | `true` +| `feast-serving-batch.livenessProbe.initialDelaySeconds` | Delay before liveness probe is initiated | `60` +| `feast-serving-batch.livenessProbe.periodSeconds` | How often to perform the probe | `10` +| `feast-serving-batch.livenessProbe.timeoutSeconds` | Timeout duration for the probe | `5` +| `feast-serving-batch.livenessProbe.successThreshold` | Minimum no of consecutive successes for the probe to be considered successful | `1` +| `feast-serving-batch.livenessProbe.failureThreshold` | Minimum no of consecutive failures for the probe to be considered failed | `5` +| `feast-serving-batch.readinessProbe.enabled` | Flag to enable readiness probe | `true` +| `feast-serving-batch.readinessProbe.initialDelaySeconds` | Delay before readiness probe is initiated | `30` +| `feast-serving-batch.readinessProbe.periodSeconds` | How often to perform the probe | `10` +| `feast-serving-batch.readinessProbe.timeoutSeconds` | Timeout duration for the probe | `10` +| `feast-serving-batch.readinessProbe.successThreshold` | Minimum no of consecutive successes for the probe to be considered successful | `1` +| `feast-serving-batch.service.type` | Kubernetes Service Type | `ClusterIP` +| `feast-serving-batch.http.port` | Kubernetes Service port for HTTP request| `80` +| `feast-serving-batch.http.targetPort` | Container port for HTTP request | `8080` +| `feast-serving-batch.grpc.port` | Kubernetes Service port for GRPC request| `6566` +| `feast-serving-batch.grpc.targetPort` | Container port for GRPC request| `6566` +| `feast-serving-batch.resources` | CPU and memory allocation for the pod | `{}` \ No newline at end of file diff --git a/infra/charts/feast-core/.helmignore b/infra/charts/feast/charts/feast-core/.helmignore similarity index 100% rename from infra/charts/feast-core/.helmignore rename to infra/charts/feast/charts/feast-core/.helmignore diff --git a/infra/charts/feast/charts/feast-core/Chart.yaml b/infra/charts/feast/charts/feast-core/Chart.yaml new file mode 100644 index 0000000000..c741a53044 --- /dev/null +++ b/infra/charts/feast/charts/feast-core/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A Helm chart for core component of Feast +name: feast-core +version: 0.3.0-alpha.1 diff --git a/infra/charts/feast/charts/feast-core/charts/kafka-0.20.1.tgz b/infra/charts/feast/charts/feast-core/charts/kafka-0.20.1.tgz new file mode 100644 index 0000000000..76a2247577 Binary files /dev/null and b/infra/charts/feast/charts/feast-core/charts/kafka-0.20.1.tgz differ diff --git a/infra/charts/feast/charts/feast-core/charts/postgresql-6.5.5.tgz b/infra/charts/feast/charts/feast-core/charts/postgresql-6.5.5.tgz new file mode 100644 index 0000000000..f61421514d Binary files /dev/null and b/infra/charts/feast/charts/feast-core/charts/postgresql-6.5.5.tgz differ diff --git a/infra/charts/feast/charts/feast-core/requirements.yaml b/infra/charts/feast/charts/feast-core/requirements.yaml new file mode 100644 index 0000000000..efe9fec508 --- /dev/null +++ b/infra/charts/feast/charts/feast-core/requirements.yaml @@ -0,0 +1,9 @@ +dependencies: +- name: postgresql + version: 6.5.5 + repository: "@stable" + condition: postgresql.enabled +- name: kafka + version: 0.20.1 + repository: "@incubator" + condition: kafka.enabled \ No newline at end of file diff --git a/infra/charts/feast-core/templates/_helpers.tpl b/infra/charts/feast/charts/feast-core/templates/_helpers.tpl similarity index 57% rename from infra/charts/feast-core/templates/_helpers.tpl rename to infra/charts/feast/charts/feast-core/templates/_helpers.tpl index aed02bef0f..7c095e11c4 100644 --- a/infra/charts/feast-core/templates/_helpers.tpl +++ b/infra/charts/feast/charts/feast-core/templates/_helpers.tpl @@ -25,33 +25,21 @@ If release name contains chart name it will be used as a full name. {{- end -}} {{/* -Create a default fully qualified app name. -We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). -If release name contains chart name it will be used as a full name. -*/}} -{{- define "postgresql.fullname" -}} -{{- $name := default "postgresql" -}} -{{- if contains $name .Release.Name -}} -{{- .Release.Name | trunc 63 | trimSuffix "-" -}} -{{- else -}} -{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} -{{- end -}} -{{- end -}} - -{{/* -Create a postgres hostname, if not specifically set by the user. +Create chart name and version as used by the chart label. */}} -{{- define "postgresql.host" -}} -{{- if .Values.postgresql.host -}} -{{- .Values.postgresql.host -}} -{{- else -}} -{{ printf "%s.%s.svc.cluster.local" (include "postgresql.fullname" .) .Release.Namespace }} -{{- end -}} +{{- define "feast-core.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} {{/* -Create chart name and version as used by the chart label. +Common labels */}} -{{- define "feast-core.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- define "feast-core.labels" -}} +app.kubernetes.io/name: {{ include "feast-core.name" . }} +helm.sh/chart: {{ include "feast-core.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} diff --git a/infra/charts/feast/charts/feast-core/templates/configmap.yaml b/infra/charts/feast/charts/feast-core/templates/configmap.yaml new file mode 100644 index 0000000000..68dc45c057 --- /dev/null +++ b/infra/charts/feast/charts/feast-core/templates/configmap.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "feast-core.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-core.name" . }} + component: core + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + application.yaml: | +{{- $config := index .Values "application.yaml"}} + +{{- if .Values.postgresql.enabled }} +{{- $datasource := dict "url" (printf "jdbc:postgresql://%s:%s/%s" (printf "%s-postgresql" .Release.Name) (.Values.postgresql.service.port | toString) (.Values.postgresql.postgresqlDatabase)) "driverClassName" "org.postgresql.Driver" }} +{{- $newConfig := dict "spring" (dict "datasource" $datasource) }} +{{- $config := mergeOverwrite $config $newConfig }} +{{- end }} + +{{- if .Values.kafka.enabled }} +{{- $topic := index .Values.kafka.topics 0 }} +{{- $options := dict "topic" $topic.name "replicationFactor" $topic.replicationFactor "partitions" $topic.partitions }} +{{- if not .Values.kafka.external.enabled }} +{{- $_ := set $options "bootstrapServers" (printf "%s:9092" (printf "%s-kafka" .Release.Name)) }} +{{- end }} +{{- $newConfig := dict "feast" (dict "stream" (dict "type" "kafka" "options" $options))}} +{{- $config := mergeOverwrite $config $newConfig }} +{{- end }} + +{{- toYaml $config | nindent 4 }} diff --git a/infra/charts/feast/charts/feast-core/templates/deployment.yaml b/infra/charts/feast/charts/feast-core/templates/deployment.yaml new file mode 100644 index 0000000000..02a533c263 --- /dev/null +++ b/infra/charts/feast/charts/feast-core/templates/deployment.yaml @@ -0,0 +1,108 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "feast-core.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-core.name" . }} + component: core + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: {{ template "feast-core.name" . }} + component: core + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ template "feast-core.name" . }} + component: core + release: {{ .Release.Name }} + spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + + volumes: + - name: {{ template "feast-core.fullname" . }}-config + configMap: + name: {{ template "feast-core.fullname" . }} + {{- if .Values.gcpServiceAccount.useExistingSecret }} + - name: {{ template "feast-core.fullname" . }}-gcpserviceaccount + secret: + secretName: {{ .Values.gcpServiceAccount.existingSecret.name }} + {{- end }} + + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + + volumeMounts: + - name: {{ template "feast-core.fullname" . }}-config + mountPath: "{{ .Values.springConfigMountPath }}" + {{- if .Values.gcpServiceAccount.useExistingSecret }} + - name: {{ template "feast-core.fullname" . }}-gcpserviceaccount + mountPath: {{ .Values.gcpServiceAccount.mountPath }} + readOnly: true + {{- end }} + + env: + {{- if .Values.postgresql.enabled }} + - name: SPRING_DATASOURCE_USERNAME + value: {{ .Values.postgresql.postgresqlUsername }} + - name: SPRING_DATASOURCE_PASSWORD + value: {{ .Values.postgresql.postgresqlPassword }} + {{- end }} + + {{- if .Values.gcpServiceAccount.useExistingSecret }} + - name: GOOGLE_APPLICATION_CREDENTIALS + value: {{ .Values.gcpServiceAccount.mountPath }}/{{ .Values.gcpServiceAccount.existingSecret.key }} + {{- end }} + + command: + - java + {{- range .Values.jvmOptions }} + - {{ . }} + {{- end }} + - -jar + - /opt/feast/feast-core.jar + - "--spring.config.location=file:{{ .Values.springConfigMountPath }}/application.yaml" + + ports: + - name: http + containerPort: {{ .Values.service.http.targetPort }} + - name: grpc + containerPort: {{ .Values.service.grpc.targetPort }} + + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: {{ .Values.service.http.targetPort }} + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: {{ .Values.service.http.targetPort }} + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + + resources: + {{- toYaml .Values.resources | nindent 10 }} diff --git a/infra/charts/feast-core/templates/core-ingress.yaml b/infra/charts/feast/charts/feast-core/templates/ingress.yaml similarity index 81% rename from infra/charts/feast-core/templates/core-ingress.yaml rename to infra/charts/feast/charts/feast-core/templates/ingress.yaml index e2cbd0e476..86fc2d3f17 100644 --- a/infra/charts/feast-core/templates/core-ingress.yaml +++ b/infra/charts/feast/charts/feast-core/templates/ingress.yaml @@ -20,9 +20,9 @@ spec: - host: {{ .host | quote }} http: paths: - - path: / - backend: - serviceName: {{ $fullName }} - servicePort: {{ .port | quote }} + - path: / + backend: + serviceName: {{ $fullName }} + servicePort: {{ .port | quote }} {{- end }} {{- end }} \ No newline at end of file diff --git a/infra/charts/feast-core/templates/core-service.yaml b/infra/charts/feast/charts/feast-core/templates/service.yaml similarity index 65% rename from infra/charts/feast-core/templates/core-service.yaml rename to infra/charts/feast/charts/feast-core/templates/service.yaml index c8b53c1228..1561398337 100644 --- a/infra/charts/feast-core/templates/core-service.yaml +++ b/infra/charts/feast/charts/feast-core/templates/service.yaml @@ -10,25 +10,32 @@ metadata: heritage: {{ .Release.Service }} {{- with .Values.service.annotations }} annotations: -{{ toYaml . | indent 4 }} + {{ toYaml . | nindent 4 }} {{- end }} spec: type: {{ .Values.service.type }} -{{- if .Values.service.loadBalancerIP }} + {{- if .Values.service.loadBalancerIP }} loadBalancerIP: {{ .Values.service.loadBalancerIP }} -{{- end }} -{{- if .Values.service.loadBalancerSourceRanges }} + {{- end }} + {{- if .Values.service.loadBalancerSourceRanges }} loadBalancerSourceRanges: -{{ toYaml .Values.service.loadBalancerSourceRanges | indent 2 }} -{{- end }} + {{ toYaml .Values.service.loadBalancerSourceRanges | nindent 2 }} + {{- end }} ports: - name: http port: {{ .Values.service.http.port }} targetPort: {{ .Values.service.http.targetPort }} + {{- if .Values.service.http.nodePort }} + nodePort: {{ .Values.service.http.nodePort }} + {{- end }} - name: grpc port: {{ .Values.service.grpc.port }} targetPort: {{ .Values.service.grpc.targetPort }} + {{- if .Values.service.grpc.nodePort }} + nodePort: {{ .Values.service.grpc.nodePort }} + {{- end }} selector: app: {{ template "feast-core.name" . }} component: core release: {{ .Release.Name }} + diff --git a/infra/charts/feast/charts/feast-core/values.yaml b/infra/charts/feast/charts/feast-core/values.yaml new file mode 100644 index 0000000000..370bf868d7 --- /dev/null +++ b/infra/charts/feast/charts/feast-core/values.yaml @@ -0,0 +1,187 @@ +# postgresql configures Postgresql that is installed as part of Feast Core. +# Refer to https://github.com/helm/charts/tree/c42002a21abf8eff839ff1d2382152bde2bbe596/stable/postgresql +# for additional configuration. +postgresql: + # enabled specifies whether Postgresql should be installed as part of Feast Core. + # + # Feast Core requires a database to store data such as the created FeatureSets + # and job statuses. If enabled, the database and service port specified below + # will override "spring.datasource.url" value in application.yaml. The + # username and password will also be set as environment variables that will + # override "spring.datasource.username/password" in application.yaml. + enabled: true + # postgresqlDatabase is the name of the database used by Feast Core. + postgresqlDatabase: feast + # postgresqlUsername is the username to authenticate to the database. + postgresqlUsername: postgres + # postgresqlPassword is the password to authenticate to the database. + postgresqlPassword: password + service: + # port is the TCP port that Postgresql will listen to + port: 5432 + +# kafka configures Kafka that is installed as part of Feast Core. +# Refer to https://github.com/helm/charts/tree/c42002a21abf8eff839ff1d2382152bde2bbe596/incubator/kafka +# for additional configuration. +kafka: + # enabled specifies whether Kafka should be installed as part of Feast Core. + # + # Feast Core requires a Kafka instance to be set as the default source for + # FeatureRows. If enabled, "feast.stream" option in application.yaml will + # be overridden by this installed Kafka configuration. + enabled: true + topics: + # topic that will be used as default in Feast Core for the default Kafka source. + - name: feast + replicationFactor: 1 + partitions: 1 + +# replicaCount is the number of pods that will be created. +replicaCount: 1 + +# image configures the Docker image for Feast Core +image: + repository: gcr.io/kf-feast/feast-core + tag: 0.3.0-alpha.1 + pullPolicy: IfNotPresent + +# application.yaml is the main configuration for Feast Core application. +# +# Feast Core is a Spring Boot app which uses this yaml configuration file. +# Refer to https://github.com/gojek/feast/blob/79eb4ab5fa3d37102c1dca9968162a98690526ba/core/src/main/resources/application.yml +# for a complete list and description of the configuration. +# +# Note that some properties defined in application.yaml may be overriden by +# Helm under certain conditions. For example, if postgresql and kafka dependencies +# are enabled. +application.yaml: + grpc: + port: 6565 + enable-reflection: true + feast: + jobs: + runner: DirectRunner + options: {} + metrics: + enabled: false + type: prometheus + host: localhost + port: 9091 + stream: + type: kafka + options: + topic: TOPIC + bootstrapServers: HOST:PORT + replicationFactor: 1 + partitions: 1 + spring: + jpa: + properties.hibernate.format_sql: true + hibernate.naming.physical-strategy=org.hibernate.boot.model.naming: PhysicalNamingStrategyStandardImpl + hibernate.ddl-auto: update + datasource: + driverClassName: org.postgresql.Driver + url: jdbc:postgresql://HOST:PORT/DATABASE + username: USERNAME + password: PASSWORD + management: + metrics: + export: + simple: + enabled: false + statsd: + enabled: false + host: localhost + port: 8125 + +# springConfigMountPath is the directory path where application.yaml will be +# mounted in the container. +springConfigMountPath: /etc/feast/feast-core + +# gcpServiceAccount is the service account that Feast Core will use. +gcpServiceAccount: + # useExistingSecret specifies Feast to use an existing secret containing Google + # Cloud service account JSON key file. + useExistingSecret: false + existingSecret: + # name is the secret name of the existing secret for the service account. + name: feast-gcp-service-account + # key is the secret key of the existing secret for the service account. + # key is normally derived from the file name of the JSON key file. + key: key.json + # mountPath is the directory path where the JSON key file will be mounted. + # the value of "existingSecret.key" is file name of the service account file. + mountPath: /etc/gcloud/service-accounts + +# jvmOptions are options that will be passed to the Java Virtual Machine (JVM) +# running Feast Core. +# +# For example, it is good practice to set min and max heap size in JVM. +# https://stackoverflow.com/questions/6902135/side-effect-for-increasing-maxpermsize-and-max-heap-size +# +# Refer to https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html +# to see other JVM options that can be set. +# +# jvmOptions: +# - -Xms1024m +# - -Xmx1024m + +livenessProbe: + enabled: true + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + +readinessProbe: + enabled: true + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 5 + +service: + type: ClusterIP + http: + port: 80 + targetPort: 8080 + # nodePort is the port number that each cluster node will listen to + # https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + # + # nodePort: + grpc: + port: 6565 + targetPort: 6565 + # nodePort is the port number that each cluster node will listen to + # https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + # + # nodePort: + +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + hosts: + # - host: chart-example.local + # port: http + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/infra/charts/feast-serving/.helmignore b/infra/charts/feast/charts/feast-serving/.helmignore similarity index 100% rename from infra/charts/feast-serving/.helmignore rename to infra/charts/feast/charts/feast-serving/.helmignore diff --git a/infra/charts/feast/charts/feast-serving/Chart.yaml b/infra/charts/feast/charts/feast-serving/Chart.yaml new file mode 100644 index 0000000000..f69b3ab826 --- /dev/null +++ b/infra/charts/feast/charts/feast-serving/Chart.yaml @@ -0,0 +1,4 @@ +apiVersion: v1 +description: A Helm chart for serving component of Feast +name: feast-serving +version: 0.3.0-alpha.1 diff --git a/infra/charts/feast/charts/feast-serving/charts/redis-9.5.0.tgz b/infra/charts/feast/charts/feast-serving/charts/redis-9.5.0.tgz new file mode 100644 index 0000000000..962893a825 Binary files /dev/null and b/infra/charts/feast/charts/feast-serving/charts/redis-9.5.0.tgz differ diff --git a/infra/charts/feast/charts/feast-serving/requirements.yaml b/infra/charts/feast/charts/feast-serving/requirements.yaml new file mode 100644 index 0000000000..fa4c1df4c1 --- /dev/null +++ b/infra/charts/feast/charts/feast-serving/requirements.yaml @@ -0,0 +1,5 @@ +dependencies: +- name: redis + version: 9.5.0 + repository: "@stable" + condition: redis.enabled diff --git a/infra/charts/feast-serving/templates/_helpers.tpl b/infra/charts/feast/charts/feast-serving/templates/_helpers.tpl similarity index 73% rename from infra/charts/feast-serving/templates/_helpers.tpl rename to infra/charts/feast/charts/feast-serving/templates/_helpers.tpl index 72e5388337..49abb6b8e5 100644 --- a/infra/charts/feast-serving/templates/_helpers.tpl +++ b/infra/charts/feast/charts/feast-serving/templates/_helpers.tpl @@ -14,8 +14,6 @@ If release name contains chart name it will be used as a full name. {{- define "feast-serving.fullname" -}} {{- if .Values.fullnameOverride -}} {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} -{{- else if contains "feast" .Release.Name -}} -{{- $name := "serving" }} {{- else -}} {{- $name := default .Chart.Name .Values.nameOverride -}} {{- if contains $name .Release.Name -}} @@ -32,3 +30,16 @@ Create chart name and version as used by the chart label. {{- define "feast-serving.chart" -}} {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} {{- end -}} + +{{/* +Common labels +*/}} +{{- define "feast-serving.labels" -}} +app.kubernetes.io/name: {{ include "feast-serving.name" . }} +helm.sh/chart: {{ include "feast-serving.chart" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end -}} diff --git a/infra/charts/feast/charts/feast-serving/templates/configmap.yaml b/infra/charts/feast/charts/feast-serving/templates/configmap.yaml new file mode 100644 index 0000000000..6c22a1016e --- /dev/null +++ b/infra/charts/feast/charts/feast-serving/templates/configmap.yaml @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "feast-serving.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-serving.name" . }} + component: serving + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + application.yaml: | +{{- $config := index .Values "application.yaml" }} + +{{- if .Values.core.enabled }} +{{- $newConfig := dict "feast" (dict "core-host" (printf "%s-feast-core" .Release.Name)) }} +{{- $config := mergeOverwrite $config $newConfig }} +{{- end }} + +{{- $store := index .Values "store.yaml" }} +{{- if eq $store.type "BIGQUERY" }} +{{- $jobStore := dict "host" (printf "%s-redis-headless" .Release.Name) "port" 6379 }} +{{- $newConfig := dict "feast" (dict "jobs" (dict "store-options" $jobStore)) }} +{{- $config := mergeOverwrite $config $newConfig }} +{{- end }} + +{{- toYaml $config | nindent 4 }} + + store.yaml: | +{{- $config := index .Values "store.yaml"}} + +{{- if and .Values.redis.enabled (eq $config.type "REDIS") }} +{{- $newConfig := dict "redis_config" (dict "host" (printf "%s-redis-headless" .Release.Name) "port" .Values.redis.redisPort) }} +{{- $config := mergeOverwrite $config $newConfig }} + +{{- if and (eq .Values.redis.master.service.type "LoadBalancer") (not (empty .Values.redis.master.service.loadBalancerIP)) }} +{{- $newConfig := dict "redis_config" (dict "host" .Values.redis.master.service.loadBalancerIP "port" .Values.redis.redisPort) }} +{{- $config := mergeOverwrite $config $newConfig }} +{{- end }} + +{{- end }} + +{{- toYaml $config | nindent 4 }} diff --git a/infra/charts/feast/charts/feast-serving/templates/deployment.yaml b/infra/charts/feast/charts/feast-serving/templates/deployment.yaml new file mode 100644 index 0000000000..c209a75ee5 --- /dev/null +++ b/infra/charts/feast/charts/feast-serving/templates/deployment.yaml @@ -0,0 +1,99 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "feast-serving.fullname" . }} + namespace: {{ .Release.Namespace }} + labels: + app: {{ template "feast-serving.name" . }} + component: serving + chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + app: {{ template "feast-serving.name" . }} + component: serving + release: {{ .Release.Name }} + template: + metadata: + labels: + app: {{ template "feast-serving.name" . }} + component: serving + release: {{ .Release.Name }} + spec: + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + + volumes: + - name: {{ template "feast-serving.fullname" . }}-config + configMap: + name: {{ template "feast-serving.fullname" . }} + {{- if .Values.gcpServiceAccount.useExistingSecret }} + - name: {{ template "feast-serving.fullname" . }}-gcpserviceaccount + secret: + secretName: {{ .Values.gcpServiceAccount.existingSecret.name }} + {{- end }} + + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + + volumeMounts: + - name: {{ template "feast-serving.fullname" . }}-config + mountPath: "{{ .Values.springConfigMountPath }}" + {{- if .Values.gcpServiceAccount.useExistingSecret }} + - name: {{ template "feast-serving.fullname" . }}-gcpserviceaccount + mountPath: {{ .Values.gcpServiceAccount.mountPath }} + readOnly: true + {{- end }} + + env: + {{- if .Values.gcpServiceAccount.useExistingSecret }} + - name: GOOGLE_APPLICATION_CREDENTIALS + value: {{ .Values.gcpServiceAccount.mountPath }}/{{ .Values.gcpServiceAccount.existingSecret.key }} + {{- end }} + + command: + - java + {{- range .Values.jvmOptions }} + - {{ . }} + {{- end }} + - -jar + - /opt/feast/feast-serving.jar + - "--spring.config.location=file:{{ .Values.springConfigMountPath }}/application.yaml" + + ports: + - name: http + containerPort: {{ .Values.service.http.targetPort }} + - name: grpc + containerPort: {{ .Values.service.grpc.targetPort }} + + {{- if .Values.livenessProbe.enabled }} + livenessProbe: + exec: + command: ["grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"] + initialDelaySeconds: {{ .Values.livenessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.livenessProbe.periodSeconds }} + successThreshold: {{ .Values.livenessProbe.successThreshold }} + timeoutSeconds: {{ .Values.livenessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.livenessProbe.failureThreshold }} + {{- end }} + + {{- if .Values.readinessProbe.enabled }} + readinessProbe: + exec: + command: ["grpc-health-probe", "-addr=:{{ .Values.service.grpc.targetPort }}"] + initialDelaySeconds: {{ .Values.readinessProbe.initialDelaySeconds }} + periodSeconds: {{ .Values.readinessProbe.periodSeconds }} + successThreshold: {{ .Values.readinessProbe.successThreshold }} + timeoutSeconds: {{ .Values.readinessProbe.timeoutSeconds }} + failureThreshold: {{ .Values.readinessProbe.failureThreshold }} + {{- end }} + + resources: + {{- toYaml .Values.resources | nindent 10 }} diff --git a/infra/charts/feast-serving/templates/serving-ingress.yaml b/infra/charts/feast/charts/feast-serving/templates/ingress.yaml similarity index 98% rename from infra/charts/feast-serving/templates/serving-ingress.yaml rename to infra/charts/feast/charts/feast-serving/templates/ingress.yaml index 2e64c0c22d..c6b4cb07a8 100644 --- a/infra/charts/feast-serving/templates/serving-ingress.yaml +++ b/infra/charts/feast/charts/feast-serving/templates/ingress.yaml @@ -25,4 +25,4 @@ spec: serviceName: {{ $fullName }} servicePort: {{ .port | quote }} {{- end }} -{{- end }} \ No newline at end of file +{{- end }} diff --git a/infra/charts/feast-serving/templates/serving-service.yaml b/infra/charts/feast/charts/feast-serving/templates/service.yaml similarity index 83% rename from infra/charts/feast-serving/templates/serving-service.yaml rename to infra/charts/feast/charts/feast-serving/templates/service.yaml index 71b47bfd00..7afdd31b45 100644 --- a/infra/charts/feast-serving/templates/serving-service.yaml +++ b/infra/charts/feast/charts/feast-serving/templates/service.yaml @@ -22,12 +22,18 @@ spec: {{ toYaml .Values.service.loadBalancerSourceRanges | indent 2 }} {{- end }} ports: - - name: grpc - port: {{ .Values.service.grpc.port }} - targetPort: {{ .Values.service.grpc.targetPort }} - name: http port: {{ .Values.service.http.port }} targetPort: {{ .Values.service.http.targetPort }} + {{- if .Values.service.http.nodePort }} + nodePort: {{ .Values.service.http.nodePort }} + {{- end }} + - name: grpc + port: {{ .Values.service.grpc.port }} + targetPort: {{ .Values.service.grpc.targetPort }} + {{- if .Values.service.grpc.nodePort }} + nodePort: {{ .Values.service.grpc.nodePort }} + {{- end }} selector: app: {{ template "feast-serving.name" . }} component: serving diff --git a/infra/charts/feast/charts/feast-serving/values.yaml b/infra/charts/feast/charts/feast-serving/values.yaml new file mode 100644 index 0000000000..ffe8684c6c --- /dev/null +++ b/infra/charts/feast/charts/feast-serving/values.yaml @@ -0,0 +1,196 @@ +# redis configures Redis that is installed as part of Feast Serving. +# Refer to https://github.com/helm/charts/tree/99430c4afdc88213c1ca08f40eeb03868ffcc9d7/stable/redis +# for additional configuration +redis: + # enabled specifies whether Redis should be installed as part of Feast Serving. + # + # If enabled, "redis_config" in store.yaml will be overwritten by Helm + # to the configuration in this Redis installation. + enabled: false + # usePassword specifies if password is required to access Redis. Note that + # Feast 0.3 does not support Redis with password. + usePassword: false + # cluster configuration for Redis. + cluster: + # enabled specifies if Redis should be installed in cluster mode. + enabled: false + +# core configures Feast Core in the same parent feast chart that this Feast +# Serving connects to. +core: + # enabled specifies that Feast Serving will use Feast Core installed + # in the same parent feast chart. If enabled, Helm will overwrite + # "feast.core-host" in application.yaml with the correct value. + enabled: true + +# replicaCount is the number of pods that will be created. +replicaCount: 1 + +# image configures the Docker image for Feast Serving +image: + repository: gcr.io/kf-feast/feast-serving + tag: 0.3.0-alpha.1 + pullPolicy: IfNotPresent + +# application.yaml is the main configuration for Feast Serving application. +# +# Feast Core is a Spring Boot app which uses this yaml configuration file. +# Refer to https://github.com/gojek/feast/blob/79eb4ab5fa3d37102c1dca9968162a98690526ba/serving/src/main/resources/application.yml +# for a complete list and description of the configuration. +# +# Note that some properties defined in application.yaml may be overridden by +# Helm under certain conditions. For example, if core is enabled, then +# "feast.core-host" will be overridden. Also, if "type: BIGQUERY" is specified +# in store.yaml, "feast.jobs.store-options" will be overridden as well with +# the default option supported in Feast 0.3. +application.yaml: + feast: + version: 0.3 + core-host: localhost + core-grpc-port: 6565 + tracing: + enabled: false + tracer-name: jaeger + service-name: feast-serving + store: + config-path: /etc/feast/feast-serving/store.yaml + redis-pool-max-size: 128 + redis-pool-max-idle: 64 + jobs: + staging-location: "" + store-type: "" + store-options: {} + grpc: + port: 6566 + enable-reflection: true + spring: + main: + web-application-type: none + +# store.yaml is the configuration for Feast Store. +# +# Refer to this link for description: +# https://github.com/gojek/feast/blob/79eb4ab5fa3d37102c1dca9968162a98690526ba/protos/feast/core/Store.proto +# +# Use the correct store configuration depending on whether the installed +# Feast Serving is "online" or "batch", by uncommenting the correct store.yaml. +# +# Note that if "redis.enabled: true" and "type: REDIS" in store.yaml, +# Helm will override "redis_config" with configuration of Redis installed +# in this chart. +# +# Note that if "type: BIGQUERY" in store.yaml, Helm assumes Feast Online serving +# is also installed with Redis store. Helm will then override "feast.jobs.store-options" +# in application.yaml with the installed Redis store configuration. This is +# because in Feast 0.3, Redis job store is required. +# +# store.yaml: +# name: online +# type: REDIS +# redis_config: +# host: localhost +# port: 6379 +# subscriptions: +# - name: ".*" +# version: ">0" +# +# store.yaml: +# name: bigquery +# type: BIGQUERY +# bigquery_config: +# project_id: PROJECT_ID +# dataset_id: DATASET_ID +# subscriptions: +# - name: ".*" +# version: ">0" + +# springConfigMountPath is the directory path where application.yaml and +# store.yaml will be mounted in the container. +springConfigMountPath: /etc/feast/feast-serving + +# gcpServiceAccount is the service account that Feast Core will use. +gcpServiceAccount: + # useExistingSecret specifies Feast to use an existing secret containing Google + # Cloud service account JSON key file. + useExistingSecret: false + existingSecret: + # name is the secret name of the existing secret for the service account. + name: feast-gcp-service-account + # key is the secret key of the existing secret for the service account. + # key is normally derived from the file name of the JSON key file. + key: key.json + # mountPath is the directory path where the JSON key file will be mounted. + # the value of "existingSecret.key" is file name of the service account file. + mountPath: /etc/gcloud/service-accounts + +# jvmOptions are options that will be passed to the Java Virtual Machine (JVM) +# running Feast Core. +# +# For example, it is good practice to set min and max heap size in JVM. +# https://stackoverflow.com/questions/6902135/side-effect-for-increasing-maxpermsize-and-max-heap-size +# +# Refer to https://docs.oracle.com/cd/E22289_01/html/821-1274/configuring-the-default-jvm-and-java-arguments.html +# to see other JVM options that can be set. +# +# jvmOptions: +# - -Xms768m +# - -Xmx768m + +livenessProbe: + enabled: false + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 5 + successThreshold: 1 + failureThreshold: 5 + +readinessProbe: + enabled: false + initialDelaySeconds: 15 + periodSeconds: 10 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 5 + +service: + type: ClusterIP + http: + port: 80 + targetPort: 8080 + # nodePort is the port number that each cluster node will listen to + # https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + # + # nodePort: + grpc: + port: 6566 + targetPort: 6566 + # nodePort is the port number that each cluster node will listen to + # https://kubernetes.io/docs/concepts/services-networking/service/#type-nodeport + # + # nodePort: + +ingress: + enabled: false + annotations: {} + # kubernetes.io/ingress.class: nginx + hosts: + # - host: chart-example.local + # port: http + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/infra/charts/feast/charts/kafka-0.17.0.tgz b/infra/charts/feast/charts/kafka-0.17.0.tgz deleted file mode 100644 index 5c1b50d5b4..0000000000 Binary files a/infra/charts/feast/charts/kafka-0.17.0.tgz and /dev/null differ diff --git a/infra/charts/feast/charts/nginx-ingress-1.24.2.tgz b/infra/charts/feast/charts/nginx-ingress-1.24.2.tgz deleted file mode 100644 index 24513f7d18..0000000000 Binary files a/infra/charts/feast/charts/nginx-ingress-1.24.2.tgz and /dev/null differ diff --git a/infra/charts/feast/charts/prometheus-pushgateway-1.0.1.tgz b/infra/charts/feast/charts/prometheus-pushgateway-1.0.1.tgz deleted file mode 100644 index 3983a95d6d..0000000000 Binary files a/infra/charts/feast/charts/prometheus-pushgateway-1.0.1.tgz and /dev/null differ diff --git a/infra/charts/feast/charts/redis-6.4.4.tgz b/infra/charts/feast/charts/redis-6.4.4.tgz deleted file mode 100644 index 462d855969..0000000000 Binary files a/infra/charts/feast/charts/redis-6.4.4.tgz and /dev/null differ diff --git a/infra/charts/feast/requirements.lock b/infra/charts/feast/requirements.lock index f2c77d4477..963bb47c6b 100644 --- a/infra/charts/feast/requirements.lock +++ b/infra/charts/feast/requirements.lock @@ -1,21 +1,12 @@ dependencies: - name: feast-core - repository: http://127.0.0.1:8879/charts - version: 0.3.0 + repository: "" + version: 0.3.0-alpha.1 - name: feast-serving - repository: http://127.0.0.1:8879/charts - version: 0.3.0 + repository: "" + version: 0.3.0-alpha.1 - name: feast-serving - repository: http://127.0.0.1:8879/charts - version: 0.3.0 -- name: redis - repository: https://kubernetes-charts.storage.googleapis.com - version: 6.4.4 -- name: kafka - repository: https://kubernetes-charts-incubator.storage.googleapis.com/ - version: 0.17.0 -- name: nginx-ingress - repository: https://kubernetes-charts.storage.googleapis.com - version: 1.24.2 -digest: sha256:47e72348460a8ffb02d384e0a2b9548f1baeeeb74ab1af5b2262e72b559fbbaa -generated: "2019-10-10T13:28:47.406676+08:00" + repository: "" + version: 0.3.0-alpha.1 +digest: sha256:98e5dbbc48287626aeef6aa550b81c1494fe2206af92e2d532134e9e30fb97d8 +generated: "2019-11-05T12:09:26.710407597+08:00" diff --git a/infra/charts/feast/requirements.yaml b/infra/charts/feast/requirements.yaml index 6c6b728e21..eedf027132 100644 --- a/infra/charts/feast/requirements.yaml +++ b/infra/charts/feast/requirements.yaml @@ -1,32 +1,12 @@ dependencies: - name: feast-core - version: 0.3.0 - repository: "@local" -# - name: postgresql -# version: 3.17.0 -# repository: "@stable" -# condition: postgres.provision + version: 0.3.0-alpha.1 + condition: feast-core.enabled - name: feast-serving - version: 0.3.0 - repository: "@local" - alias: online-serving + alias: feast-serving-batch + version: 0.3.0-alpha.1 + condition: feast-serving-batch.enabled - name: feast-serving - version: 0.3.0 - repository: "@local" - alias: warehouse-serving -- name: redis - version: 6.4.4 - repository: "@stable" - condition: redis.provision -- name: kafka - version: 0.17.0 - repository: "@incubator" - condition: kafka.provision -- name: prometheus-statsd-exporter - version: 0.1.2 - repository: "@local" - condition: prometheus-statsd-exporter.provision -- name: nginx-ingress - version: 1.24.2 - repository: "@stable" - condition: nginx-ingress.provision + alias: feast-serving-online + version: 0.3.0-alpha.1 + condition: feast-serving-online.enabled diff --git a/infra/charts/feast/values-demo.yaml b/infra/charts/feast/values-demo.yaml new file mode 100644 index 0000000000..43c760e633 --- /dev/null +++ b/infra/charts/feast/values-demo.yaml @@ -0,0 +1,28 @@ +# The following are values for installing Feast for demonstration purpose: +# - Persistence is disabled since for demo purpose data is not expected +# to be durable +# - Only online serving (no batch serving) is installed to remove dependency +# on Google Cloud services. Batch serving requires BigQuery dependency. + +feast-core: + enabled: true + postgresql: + persistence: + enabled: false + kafka: + persistence: + enabled: false + +feast-serving-online: + enabled: true + redis: + enabled: true + store.yaml: + name: redis + type: REDIS + subscriptions: + - name: ".*" + version: ">0" + +feast-serving-batch: + enabled: false diff --git a/infra/charts/feast/values-external-store.yaml b/infra/charts/feast/values-external-store.yaml new file mode 100644 index 0000000000..d012bcec56 --- /dev/null +++ b/infra/charts/feast/values-external-store.yaml @@ -0,0 +1,5 @@ +# TODO @dheryanto +# +# The following are sample values for installing Feast without setting up +# Kafka and Redis stores. In other words, using Feast with external stream +# source and stores. diff --git a/infra/charts/feast/values-production.yaml b/infra/charts/feast/values-production.yaml new file mode 100644 index 0000000000..6b53dc19ea --- /dev/null +++ b/infra/charts/feast/values-production.yaml @@ -0,0 +1,4 @@ +# TODO @dheryanto +# +# The following are sample values for installing Feast for typical production +# environment. diff --git a/infra/charts/feast/values.yaml b/infra/charts/feast/values.yaml index 9ea235fcfa..d6a5004606 100644 --- a/infra/charts/feast/values.yaml +++ b/infra/charts/feast/values.yaml @@ -1,301 +1,100 @@ -global: - postgresql: {} -# Feast core -feast-core: - fullnameOverride: "feast-core" - replicaCount: 1 - image: - registry: gcr.io/kf-feast - repository: feast-core - tag: 0.3.0-SNAPSHOT - pullPolicy: Never - - jobs: - runner: DirectRunner - options: {} - ## If using DataflowRunner, the following options are necessary: - # project: my-gcp-project - # tempLocation: gs://my-feast-bucket/temp - # region: us-central1 - # subnetwork: default - metrics: - enabled: true - ## Type of metrics sink. Only statsd is currently supported. - type: statsd - ## Host of the metrics sink. In the case of statsd, this is the host of the statsd exporter - host: feast-prometheus-statsd-exporter - ## Port of the metrics sink. - port: 9125 - - stream: - type: kafka - options: - bootstrapServers: feast-kafka:9092 - replicationFactor: 2 - partitions: 4 - - service: - # annotations: [] - http: - port: 80 - targetPort: 8080 - grpc: - port: 6565 - targetPort: 6565 - type: ClusterIP - # loadBalancerIP: - # loadBalancerSourceRanges: - # - 10.0.0.0/8 - port: 80 - - rollingUpdate: - maxSurge: 2 - maxUnavailable: 1 - - livenessProbe: - enabled: false - # initialDelaySeconds: 30 - # failureThreshold: 3 - readinessProbe: - initialDelaySeconds: 30 - failureThreshold: 3 +# Feast deployment installs the following components: +# - Feast Core +# - Feast Serving Online +# - Feast Serving Batch +# +# The configuration for different components can be referenced from: +# - charts/feast-core/values.yaml +# - charts/feast-serving/values.yaml +# +# Note that "feast-serving-online" and "feast-serving-batch" are +# aliases to "feast-serving" chart since in typical scenario two instances +# of Feast Serving: online and batch will be deployed. Both described +# using the same chart "feast-serving". +# +# The following are default values for typical Feast deployment, but not +# for production setting. Refer to "values-production.yaml" for recommended +# values in production environment. +# +# Note that the import job by default uses DirectRunner +# https://beam.apache.org/documentation/runners/direct/ +# in this configuration since it allows Feast to run in more environments +# (unlike DataflowRunner which requires Google Cloud services). +# +# A secret containing Google Cloud service account JSON key is required +# in this configuration. +# https://cloud.google.com/iam/docs/creating-managing-service-accounts +# +# The Google Cloud service account must have the following roles: +# - bigquery.dataEditor +# - bigquery.jobUser +# +# Assuming a service account JSON key file has been downloaded to +# (please name the file key.json): +# /home/user/key.json +# +# Run the following command to create the secret in your Kubernetes cluster: +# +# kubectl create secret generic feast-gcp-service-account \ +# --from-file=/home/user/key.json +# - ## A service account if using the dataflow runner or BigQuery. - serviceAccount: - name: my-service-account - key: tbd_staging.json - - ingress: - enabled: false - # annotations: - # kubernetes.io/ingress.class: nginx - # hosts: - # - host: feast-core.gods.wwhatever.local - # port: http - - resources: - # limits: - # cpu: 100m - # memory: 128Mi +feast-core: + enabled: true + jvmOptions: + - -Xms1024m + - -Xmx1024m + resources: requests: - cpu: 100m - memory: 128Mi - - nodeSelector: {} - -# serving -online-serving: - replicaCount: 1 - - image: - registry: gcr.io/kf-feast - repository: feast-serving - tag: 0.3.0-SNAPSHOT - pullPolicy: Never - - fullnameOverride: feast-serving-online - core: - host: feast-core - port: 6565 - - store: - config: - name: serving - type: REDIS - redis_config: - host: feast-redis - port: 6379 - subscriptions: - - name: .* - version: ">0" - - ## For redis serving. Configuration for the redis connection pool. - redisPool: - maxSize: 128 - maxIdle: 8 - - ## Job configuration is necessary if this serving store is used to - ## facilitate batch retrieval of features, which involve async jobs. - jobs: {} - # stagingLocation: gs://feast-bucket/bq/staging - # storeType: REDIS - # storeOptions: - # host: redis.svc.local - # port: 6379 - - tracing: - enabled: false - # tracer-name: jaeger - # service-name: feast-serving - - rollingUpdate: - maxSurge: 2 - maxUnavailable: 1 - - service: - # annotations: [] - http: - port: 80 - targetPort: 8080 - grpc: - port: 6565 - targetPort: 6565 - type: ClusterIP - # loadBalancerIP: - # loadBalancerSourceRanges: - # - 10.0.0.0/8 - port: 80 - - rollingUpdate: - maxSurge: 2 - maxUnavailable: 1 - - livenessProbe: - enabled: false - # initialDelaySeconds: 30 - # failureThreshold: 3 - readinessProbe: - initialDelaySeconds: 30 - failureThreshold: 3 - - ## A service account if using the dataflow runner or BigQuery. - # serviceAccount: - # name: my-service-account - - ingress: - enabled: false - # annotations: - # kubernetes.io/ingress.class: nginx - # hosts: - # - host: chart-example.local - # port: http - - resources: - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi + cpu: 1000m + memory: 1024Mi + gcpServiceAccount: + useExistingSecret: true + +feast-serving-online: + enabled: true + redis: + enabled: true + jvmOptions: + - -Xms1024m + - -Xmx1024m + resources: requests: - cpu: 100m - memory: 128Mi - - nodeSelector: {} - -warehouse-serving: - replicaCount: 1 - - image: - registry: gcr.io/kf-feast - repository: feast-serving - tag: 0.3.0-SNAPSHOT - pullPolicy: Never - - fullnameOverride: feast-serving-warehouse - core: - host: feast-core - port: 6565 - - store: - config: - name: warehouse - type: BIGQUERY - bigquery_config: - # Replace with your Google Cloud project configuration - projectId: google-project-id - datasetId: bigquery-dataset - subscriptions: - - name: .* - version: ">0" - - ## For redis serving. Configuration for the redis connection pool. - redisPool: - maxSize: 128 - maxIdle: 8 - - ## Job configuration is necessary if this serving store is used to - ## facilitate batch retrieval of features, which involve async jobs. - jobs: - stagingLocation: gs://zl-test-bucket/feast/staging - storeType: REDIS - storeOptions: - host: feast-redis - port: 6379 - - tracing: + cpu: 500m + memory: 1024Mi + store.yaml: + name: redis + type: REDIS + redis_config: + port: 6379 + subscriptions: + - name: ".*" + version: ">0" + +feast-serving-batch: + enabled: true + redis: enabled: false - # tracer-name: jaeger - # service-name: feast-serving - - rollingUpdate: - maxSurge: 2 - maxUnavailable: 1 - - service: - # annotations: [] - http: - port: 80 - targetPort: 8080 - grpc: - port: 6566 - targetPort: 6566 - type: ClusterIP - # loadBalancerIP: - # loadBalancerSourceRanges: - # - 10.0.0.0/8 - port: 80 - - rollingUpdate: - maxSurge: 2 - maxUnavailable: 1 - - livenessProbe: - enabled: false - # initialDelaySeconds: 30 - # failureThreshold: 3 - readinessProbe: - initialDelaySeconds: 30 - failureThreshold: 3 - - ## A service account if using the dataflow runner or BigQuery. - serviceAccount: - name: my-service-account - key: tbd_staging.json - - ingress: - enabled: false - # annotations: - # kubernetes.io/ingress.class: nginx - # hosts: - # - host: chart-example.local - # port: http - - resources: - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - requests: - cpu: 100m - memory: 128Mi - - nodeSelector: {} - -kafka: - provision: true + jvmOptions: + - -Xms1024m + - -Xmx1024m resources: requests: - cpu: 200m - memory: 128Mi - -prometheus-statsd-exporter: - provision: true - -redis: - provision: true - usePassword: false \ No newline at end of file + cpu: 500m + memory: 1024Mi + gcpServiceAccount: + useExistingSecret: true + application.yaml: + feast: + jobs: + staging-location: gs://bucket/path + store-type: REDIS + store.yaml: + name: bigquery + type: BIGQUERY + bigquery_config: + project_id: PROJECT_ID + dataset_id: DATASET_ID + subscriptions: + - name: ".*" + version: ">0" diff --git a/infra/docker/core/Dockerfile b/infra/docker/core/Dockerfile index c9c609c7bf..c4cfe34b71 100644 --- a/infra/docker/core/Dockerfile +++ b/infra/docker/core/Dockerfile @@ -1,19 +1,40 @@ -#FROM maven:3.6-jdk-8-slim as builder -#ARG REVISION=dev +# ============================================================ +# Build stage 1: Builder +# ============================================================ + +FROM maven:3.6-jdk-8-slim as builder +ARG REVISION=dev +COPY . /build +WORKDIR /build # -#WORKDIR /build -#COPY pom.xml . -#COPY core/pom.xml core/pom.xml -#COPY ingestion/pom.xml ingestion/pom.xml -#COPY serving/pom.xml serving/pom.xml +# Setting Maven repository .m2 directory relative to /build folder gives the +# user to optionally use cached repository when building the image by copying +# the existing .m2 directory to $FEAST_REPO_ROOT/.m2 # -#RUN mvn dependency:go-offline -B +ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false" +RUN mvn --also-make --projects core,ingestion -Drevision=$REVISION \ + -DskipTests=true --batch-mode package # -#COPY . /build -#WORKDIR /build -#RUN mvn --projects core,ingestion -Drevision=$REVISION -DskipTests=true --batch-mode package +# Unpack the jar and copy the files into production Docker image +# for faster startup time when starting Dataflow jobs from Feast Core. +# This is because we need to stage the classes and dependencies when using Dataflow. +# The final size of the production image will be bigger but it seems +# a good tradeoff between speed and size. +# +# https://github.com/gojek/feast/pull/291 +RUN apt-get -qq update && apt-get -y install unar && \ + unar /build/core/target/feast-core-$REVISION.jar -o /build/core/target/ + +# ============================================================ +# Build stage 2: Production +# ============================================================ FROM openjdk:8-jre as production ARG REVISION=dev -COPY ./core/target/feast-core-$REVISION.jar /usr/share/feast/feast-core.jar -CMD ["java", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-jar", "/usr/share/feast/feast-core.jar"] \ No newline at end of file +COPY --from=builder /build/core/target/feast-core-$REVISION.jar /opt/feast/feast-core.jar +COPY --from=builder /build/core/target/feast-core-$REVISION /opt/feast/feast-core +CMD ["java",\ + "-Xms2048m",\ + "-Xmx2048m",\ + "-jar",\ + "/opt/feast/feast-core.jar"] diff --git a/infra/docker/serving/Dockerfile b/infra/docker/serving/Dockerfile index e3d3e3fb15..3517183d78 100644 --- a/infra/docker/serving/Dockerfile +++ b/infra/docker/serving/Dockerfile @@ -1,16 +1,36 @@ -#FROM maven:3.6-jdk-8-slim as builder -#ARG REVISION=dev -#COPY . /build -#WORKDIR /build -#ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false" -#RUN mvn --projects serving -Drevision=$REVISION -DskipTests=true --batch-mode package +# ============================================================ +# Build stage 1: Builder +# ============================================================ -FROM openjdk:8-jre-alpine as production +FROM maven:3.6-jdk-8-slim as builder ARG REVISION=dev +COPY . /build +WORKDIR /build +# +# Setting Maven repository .m2 directory relative to /build folder gives the +# user to optionally use cached repository when building the image by copying +# the existing .m2 directory to $FEAST_REPO_ROOT/.m2 +# +ENV MAVEN_OPTS="-Dmaven.repo.local=/build/.m2/repository -DdependencyLocationsEnabled=false" +RUN mvn --also-make --projects serving -Drevision=$REVISION \ + -DskipTests=true --batch-mode package -RUN GRPC_HEALTH_PROBE_VERSION=v0.1.0-alpha.1 && \ - wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \ - chmod +x /bin/grpc_health_probe +# ============================================================ +# Build stage 2: Production +# ============================================================ -COPY ./serving/target/feast-serving-$REVISION.jar /usr/share/feast/feast-serving.jar -CMD ["java", "-XX:+UnlockExperimentalVMOptions", "-XX:+UseCGroupMemoryLimitForHeap", "-jar", "/usr/share/feast/feast-serving.jar"] +FROM openjdk:8-jre-alpine as production +ARG REVISION=dev +# +# Download grpc_health_probe to run health check for Feast Serving +# https://kubernetes.io/blog/2018/10/01/health-checking-grpc-servers-on-kubernetes/ +# +RUN wget -q https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/v0.3.1/grpc_health_probe-linux-amd64 \ + -O /usr/bin/grpc-health-probe && \ + chmod +x /usr/bin/grpc-health-probe +COPY --from=builder /build/serving/target/feast-serving-$REVISION.jar /opt/feast/feast-serving.jar +CMD ["java",\ + "-Xms1024m",\ + "-Xmx1024m",\ + "-jar",\ + "/opt/feast/feast-serving.jar"] diff --git a/serving/src/main/resources/application.yml b/serving/src/main/resources/application.yml index 46b40f688b..6ef68d710c 100644 --- a/serving/src/main/resources/application.yml +++ b/serving/src/main/resources/application.yml @@ -36,7 +36,12 @@ feast: # Type of store to store job metadata. This only needs to be set if the # serving store type is Bigquery. store-type: ${FEAST_JOB_STORE_TYPE:} - # Job store connection options. If the job store is redis, the redis instance host and port are required. + # + # Job store connection options. If the job store is redis, the following items are required: + # + # store-options: + # host: localhost + # port: 6379 store-options: {} grpc: