From 5fcc30fb882905a5bf1f7c5a80dc6f74d5477d2f Mon Sep 17 00:00:00 2001 From: Lionel Vital Date: Sat, 18 Jan 2020 00:18:43 -0800 Subject: [PATCH] Update GKE installation and chart values to work with 0.4.3 (#434) --- docs/getting-started/installing-feast.md | 15 ++- infra/charts/feast/values.yaml | 131 ++++++++++++++++------- 2 files changed, 105 insertions(+), 41 deletions(-) diff --git a/docs/getting-started/installing-feast.md b/docs/getting-started/installing-feast.md index 527f07741f..0b212037c1 100644 --- a/docs/getting-started/installing-feast.md +++ b/docs/getting-started/installing-feast.md @@ -268,7 +268,7 @@ bq mk ${FEAST_BIGQUERY_DATASET_ID} Create the service account that Feast will run as: ```bash -gcloud iam service-accounts create ${FEAST_SERVICE_ACCOUNT_NAME} +gcloud iam service-accounts create ${FEAST_S_ACCOUNT_NAME} gcloud projects add-iam-policy-binding ${FEAST_GCP_PROJECT_ID} \ --member serviceAccount:${FEAST_S_ACCOUNT_NAME}@${FEAST_GCP_PROJECT_ID}.iam.gserviceaccount.com \ @@ -324,6 +324,15 @@ PING 10.123.114.11 (10.203.164.22) 56(84) bytes of data. 64 bytes from 10.123.114.11: icmp_seq=2 ttl=63 time=51.2 ms ``` +Add firewall rules in gcloud to open up ports: +```bash +gcloud compute firewall-rules create feast-core-port --allow tcp:32090 +gcloud compute firewall-rules create feast-online-port --allow tcp:32091 +gcloud compute firewall-rules create feast-batch-port --allow tcp:32092 +gcloud compute firewall-rules create feast-redis-port --allow tcp:32101 +gcloud compute firewall-rules create feast-kafka-ports --allow tcp:31090-31095 +``` + ### 3. Set up Helm Run the following command to provide Tiller with authorization to install Feast: @@ -377,7 +386,8 @@ cp values.yaml my-feast-values.yaml Update `my-feast-values.yaml` based on your GCP and GKE environment. * Required fields are paired with comments which indicate whether they need to be replaced. -* All occurrences of `feast.example.com` should be replaced with either your domain name or the IP stored in `$FEAST_IP`. +* All occurrences of `EXTERNAL_IP` should be replaced with either your domain name or the IP stored in `$FEAST_IP`. +* Replace all occurrences of `YOUR_BUCKET_NAME` with your bucket name stored in `$FEAST_GCS_BUCKET` Install the Feast Helm chart: @@ -421,4 +431,3 @@ feast config set serving_url ${FEAST_ONLINE_SERVING_URL} ``` That's it! You can now start to use Feast! - diff --git a/infra/charts/feast/values.yaml b/infra/charts/feast/values.yaml index ebc8c802a1..a7d8ce0046 100644 --- a/infra/charts/feast/values.yaml +++ b/infra/charts/feast/values.yaml @@ -2,29 +2,29 @@ # - Feast Core # - Feast Serving Online # - Feast Serving Batch -# +# # The configuration for different components can be referenced from: # - charts/feast-core/values.yaml # - charts/feast-serving/values.yaml # # Note that "feast-serving-online" and "feast-serving-batch" are # aliases to "feast-serving" chart since in typical scenario two instances -# of Feast Serving: online and batch will be deployed. Both described +# of Feast Serving: online and batch will be deployed. Both described # using the same chart "feast-serving". # # The following are default values for typical Feast deployment, but not # for production setting. Refer to "values-production.yaml" for recommended # values in production environment. -# -# Note that the import job by default uses DirectRunner +# +# Note that the import job by default uses DirectRunner # https://beam.apache.org/documentation/runners/direct/ # in this configuration since it allows Feast to run in more environments # (unlike DataflowRunner which requires Google Cloud services). -# -# A secret containing Google Cloud service account JSON key is required -# in this configuration. +# +# A secret containing Google Cloud service account JSON key is required +# in this configuration. # https://cloud.google.com/iam/docs/creating-managing-service-accounts -# +# # The Google Cloud service account must have the following roles: # - bigquery.dataEditor # - bigquery.jobUser @@ -32,12 +32,13 @@ # Assuming a service account JSON key file has been downloaded to # (please name the file key.json): # /home/user/key.json -# +# # Run the following command to create the secret in your Kubernetes cluster: # # kubectl create secret generic feast-gcp-service-account \ # --from-file=/home/user/key.json # +# Replace every instance of EXTERNAL_IP with the external IP of your GKE cluster # ============================================================ # Feast Core @@ -51,12 +52,15 @@ feast-core: # to the client. These instances of Feast Serving however can still use # the same shared Feast Core. enabled: true - # jvmOptions are options that will be passed to the Java Virtual Machine (JVM) + # Specify what image tag to use. Keep this consistent for all components + image: + tag: "0.4.3" + # jvmOptions are options that will be passed to the Java Virtual Machine (JVM) # running Feast Core. # # For example, it is good practice to set min and max heap size in JVM. # https://stackoverflow.com/questions/6902135/side-effect-for-increasing-maxpermsize-and-max-heap-size - jvmOptions: + jvmOptions: - -Xms1024m - -Xmx1024m # resources that should be allocated to Feast Core. @@ -68,18 +72,43 @@ feast-core: memory: 2048Mi # gcpServiceAccount is the Google service account that Feast Core will use. gcpServiceAccount: - # useExistingSecret specifies Feast to use an existing secret containing + # useExistingSecret specifies Feast to use an existing secret containing # Google Cloud service account JSON key file. - # + # # This is the only supported option for now to use a service account JSON. # Feast admin is expected to create this secret before deploying Feast. useExistingSecret: true existingSecret: # name is the secret name of the existing secret for the service account. - name: feast-gcp-service-account + name: feast-gcp-service-account # key is the secret key of the existing secret for the service account. # key is normally derived from the file name of the JSON key file. key: key.json + # Setting service.type to NodePort exposes feast-core service at a static port + service: + type: NodePort + grpc: + # this is the port that is exposed outside of the cluster + nodePort: 32090 + # Make kafka externally accessible using NodePort + # Please set EXTERNAL_IP to your cluster's external IP + kafka: + external: + enabled: true + type: NodePort + domain: EXTERNAL_IP + configurationOverrides: + "advertised.listeners": |- + EXTERNAL://EXTERNAL_IP:$((31090 + ${KAFKA_BROKER_ID})) + "listener.security.protocol.map": |- + PLAINTEXT:PLAINTEXT,EXTERNAL:PLAINTEXT + application.yaml: + feast: + stream: + options: + # Point to one of your Kafka brokers + # Please set EXTERNAL_IP to your cluster's external IP + bootstrapServers: EXTERNAL_IP:31090 # ============================================================ # Feast Serving Online @@ -88,14 +117,22 @@ feast-core: feast-serving-online: # enabled specifies whether to install Feast Serving Online component. enabled: true + # Specify what image tag to use. Keep this consistent for all components + image: + tag: "0.4.3" # redis.enabled specifies whether Redis should be installed as part of Feast Serving. - # + # # If enabled is set to "false", Feast admin has to ensure there is an # existing Redis running outside Feast, that Feast Serving can connect to. + # master.service.type set to NodePort exposes Redis to outside of the cluster redis: enabled: true + master: + service: + nodePort: 32101 + type: NodePort # jvmOptions are options that will be passed to the Feast Serving JVM. - jvmOptions: + jvmOptions: - -Xms1024m - -Xmx1024m # resources that should be allocated to Feast Serving. @@ -105,23 +142,28 @@ feast-serving-online: memory: 1024Mi limits: memory: 2048Mi + # Make service accessible to outside of cluster using NodePort + service: + type: NodePort + grpc: + nodePort: 32091 # store.yaml is the configuration for Feast Store. - # + # # Refer to this link for more description: # https://github.com/gojek/feast/blob/79eb4ab5fa3d37102c1dca9968162a98690526ba/protos/feast/core/Store.proto store.yaml: name: redis type: REDIS redis_config: - # If redis.enabled is set to false, Feast admin should uncomment and - # set the host value to an "existing" Redis instance Feast will use as - # online Store. - # - # Else, if redis.enabled is set to true, no additional configuration is - # required. + # If redis.enabled is set to false, Feast admin should uncomment and + # set the host value to an "existing" Redis instance Feast will use as + # online Store. Also use the correct port for that existing instance. # + # Else, if redis.enabled is set to true, replace EXTERNAL_IP with your + # cluster's external IP. # host: redis-host - port: 6379 + host: EXTERNAL_IP + port: 32101 subscriptions: - name: "*" project: "*" @@ -134,14 +176,17 @@ feast-serving-online: feast-serving-batch: # enabled specifies whether to install Feast Serving Batch component. enabled: true + # Specify what image tag to use. Keep this consistent for all components + image: + tag: "0.4.3" # redis.enabled specifies whether Redis should be installed as part of Feast Serving. - # + # # This is usually set to "false" for Feast Serving Batch because the default # store is BigQuery. redis: enabled: false # jvmOptions are options that will be passed to the Feast Serving JVM. - jvmOptions: + jvmOptions: - -Xms1024m - -Xmx1024m # resources that should be allocated to Feast Serving. @@ -151,17 +196,22 @@ feast-serving-batch: memory: 1024Mi limits: memory: 2048Mi + # Make service accessible to outside of cluster using NodePort + service: + type: NodePort + grpc: + nodePort: 32092 # gcpServiceAccount is the service account that Feast Serving will use. gcpServiceAccount: - # useExistingSecret specifies Feast to use an existing secret containing + # useExistingSecret specifies Feast to use an existing secret containing # Google Cloud service account JSON key file. - # + # # This is the only supported option for now to use a service account JSON. # Feast admin is expected to create this secret before deploying Feast. useExistingSecret: true existingSecret: # name is the secret name of the existing secret for the service account. - name: feast-gcp-service-account + name: feast-gcp-service-account # key is the secret key of the existing secret for the service account. # key is normally derived from the file name of the JSON key file. key: key.json @@ -172,28 +222,33 @@ feast-serving-batch: # for a complete list and description of the configuration. application.yaml: feast: - jobs: - # staging-location specifies the URI to store intermediate files for + jobs: + # staging-location specifies the URI to store intermediate files for # batch serving (required if using BigQuery as Store). - # - # Please set the value to an "existing" Google Cloud Storage URI that + # + # Please set the value to an "existing" Google Cloud Storage URI that # Feast serving has write access to. - staging-location: gs://bucket/path - # Type of store to store job metadata. + staging-location: gs://YOUR_BUCKET_NAME/serving/batch + # Type of store to store job metadata. # - # This default configuration assumes that Feast Serving Online is + # This default configuration assumes that Feast Serving Online is # enabled as well. So Feast Serving Batch will share the same # Redis instance to store job statuses. store-type: REDIS + store-options: + # Use the externally exposed redis instance deployed by Online service + # Please set EXTERNAL_IP to your cluster's external IP + host: EXTERNAL_IP + port: 32101 # store.yaml is the configuration for Feast Store. - # + # # Refer to this link for more description: # https://github.com/gojek/feast/blob/79eb4ab5fa3d37102c1dca9968162a98690526ba/protos/feast/core/Store.proto store.yaml: name: bigquery type: BIGQUERY bigquery_config: - # project_id specifies the Google Cloud Project. Please set this to the + # project_id specifies the Google Cloud Project. Please set this to the # project id you are using BigQuery in. project_id: PROJECT_ID # dataset_id specifies an "existing" BigQuery dataset Feast Serving Batch