Allow use of secure gRPC in Feast Python client. (feast-dev#459)

* Allow use of secure gRPC in Feast Python client.

* Add tests for secure gRPC in Python client.

(cherry picked from commit a576a53)

sdk/python/feast/client.py

@@ -21,14 +21,14 @@
 from collections import OrderedDict
 from math import ceil
 from typing import Dict, List, Tuple, Union, Optional
-from typing import List
 from urllib.parse import urlparse
 
 import fastavro
 import grpc
 import pandas as pd
 import pyarrow as pa
 import pyarrow.parquet as pq
+
 from feast.core.CoreService_pb2 import (
     GetFeastCoreVersionRequest,
     ListFeatureSetsResponse,
@@ -48,11 +48,11 @@
 from feast.core.FeatureSet_pb2 import FeatureSetStatus
 from feast.feature_set import FeatureSet, Entity
 from feast.job import Job
-from feast.serving.ServingService_pb2 import FeatureReference
 from feast.loaders.abstract_producer import get_producer
 from feast.loaders.file import export_source_to_staging_location
 from feast.loaders.ingest import KAFKA_CHUNK_PRODUCTION_TIMEOUT
 from feast.loaders.ingest import get_feature_row_chunks
+from feast.serving.ServingService_pb2 import FeatureReference
 from feast.serving.ServingService_pb2 import GetFeastServingInfoResponse
 from feast.serving.ServingService_pb2 import (
     GetOnlineFeaturesRequest,
@@ -69,9 +69,11 @@
 
 GRPC_CONNECTION_TIMEOUT_DEFAULT = 3  # type: int
 GRPC_CONNECTION_TIMEOUT_APPLY = 600  # type: int
-FEAST_SERVING_URL_ENV_KEY = "FEAST_SERVING_URL"  # type: str
-FEAST_CORE_URL_ENV_KEY = "FEAST_CORE_URL"  # type: str
-FEAST_PROJECT_ENV_KEY = "FEAST_PROJECT"  # type: str
+FEAST_CORE_URL_ENV_KEY = "FEAST_CORE_URL"
+FEAST_SERVING_URL_ENV_KEY = "FEAST_SERVING_URL"
+FEAST_PROJECT_ENV_KEY = "FEAST_PROJECT"
+FEAST_CORE_SECURE_ENV_KEY = "FEAST_CORE_SECURE"
+FEAST_SERVING_SECURE_ENV_KEY = "FEAST_SERVING_SECURE"
 BATCH_FEATURE_REQUEST_WAIT_TIME_SECONDS = 300
 CPU_COUNT = os.cpu_count()  # type: int
 
@@ -82,7 +84,8 @@ class Client:
     """
 
     def __init__(
-        self, core_url: str = None, serving_url: str = None, project: str = None
+        self, core_url: str = None, serving_url: str = None, project: str = None,
+        core_secure: bool = None, serving_secure: bool = None
     ):
         """
         The Feast Client should be initialized with at least one service url
@@ -91,10 +94,14 @@ def __init__(
             core_url: Feast Core URL. Used to manage features
             serving_url: Feast Serving URL. Used to retrieve features
             project: Sets the active project. This field is optional.
-        """
-        self._core_url = core_url
-        self._serving_url = serving_url
-        self._project = project
+            core_secure: Use client-side SSL/TLS for Core gRPC API
+            serving_secure: Use client-side SSL/TLS for Serving gRPC API
+        """
+        self._core_url: str = core_url
+        self._serving_url: str = serving_url
+        self._project: str = project
+        self._core_secure: bool = core_secure
+        self._serving_secure: bool = serving_secure
         self.__core_channel: grpc.Channel = None
         self.__serving_channel: grpc.Channel = None
         self._core_service_stub: CoreServiceStub = None
@@ -149,6 +156,52 @@ def serving_url(self, value: str):
         """
         self._serving_url = value
 
+    @property
+    def core_secure(self) -> bool:
+        """
+        Retrieve Feast Core client-side SSL/TLS setting
+
+        Returns:
+            Whether client-side SSL/TLS is enabled
+        """
+
+        if self._core_secure is not None:
+            return self._core_secure
+        return os.getenv(FEAST_CORE_SECURE_ENV_KEY, "").lower() is "true"
+
+    @core_secure.setter
+    def core_secure(self, value: bool):
+        """
+        Set the Feast Core client-side SSL/TLS setting
+
+        Args:
+            value: True to enable client-side SSL/TLS
+        """
+        self._core_secure = value
+
+    @property
+    def serving_secure(self) -> bool:
+        """
+        Retrieve Feast Serving client-side SSL/TLS setting
+
+        Returns:
+            Whether client-side SSL/TLS is enabled
+        """
+
+        if self._serving_secure is not None:
+            return self._serving_secure
+        return os.getenv(FEAST_SERVING_SECURE_ENV_KEY, "").lower() is "true"
+
+    @serving_secure.setter
+    def serving_secure(self, value: bool):
+        """
+        Set the Feast Serving client-side SSL/TLS setting
+
+        Args:
+            value: True to enable client-side SSL/TLS
+        """
+        self._serving_secure = value
+
     def version(self):
         """
         Returns version information from Feast Core and Feast Serving
@@ -185,7 +238,10 @@ def _connect_core(self, skip_if_connected: bool = True):
             raise ValueError("Please set Feast Core URL.")
 
         if self.__core_channel is None:
-            self.__core_channel = grpc.insecure_channel(self.core_url)
+            if self.core_secure or self.core_url.endswith(":443"):
+                self.__core_channel = grpc.secure_channel(self.core_url, grpc.ssl_channel_credentials())
+            else:
+                self.__core_channel = grpc.insecure_channel(self.core_url)
 
         try:
             grpc.channel_ready_future(self.__core_channel).result(
@@ -214,7 +270,10 @@ def _connect_serving(self, skip_if_connected=True):
             raise ValueError("Please set Feast Serving URL.")
 
         if self.__serving_channel is None:
-            self.__serving_channel = grpc.insecure_channel(self.serving_url)
+            if self.serving_secure or self.serving_url.endswith(":443"):
+                self.__serving_channel = grpc.secure_channel(self.serving_url, grpc.ssl_channel_credentials())
+            else:
+                self.__serving_channel = grpc.insecure_channel(self.serving_url)
 
         try:
             grpc.channel_ready_future(self.__serving_channel).result(

sdk/python/requirements-ci.txt

@@ -11,6 +11,7 @@ mock==2.0.0
 pandas==0.*
 protobuf==3.*
 pytest
+pytest-lazy-fixture==0.6.3
 pytest-mock
 pytest-timeout
 PyYAML==5.1.2

sdk/python/tests/data/localhost.crt

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIIC5zCCAc+gAwIBAgIJAKzukpnyuwsVMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV
+BAMMCWxvY2FsaG9zdDAgFw0yMDAyMTcxMTE4NDNaGA8zMDE5MDYyMDExMTg0M1ow
+FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAqoanhiy4EUZjPA/m8IWk50OyTjKAnqZvEW5glqmTHP6lQbfyWQnzj3Ny
+c++4Xn901FO2v07h+7lE3BScjgCX6klsLOHRnWcLX8lQygR6zzO+Oey1yXuCebBA
+yhrsqgTDC/8zoCxe0W3t0vqvE4AJs3tJHq5Y1ba/X9OiKKsDZuMSSsbdd4qVEL6y
+BD8PRNLT/iiD84Kq58GZtOI3fJls8E/bYbvksugcPI3kmlU4Plg3VrVplMl3DcMz
+7BbvQP6jmVqdPtUT7+lL0C5CsNqbdDOIwg09+Gwus+A/g8PerBBd+ZCmdvSa9LYJ
+OmlJszgZPIL9AagXLfuGQvNN2Y6WowIDAQABozowODAUBgNVHREEDTALgglsb2Nh
+bGhvc3QwCwYDVR0PBAQDAgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3
+DQEBCwUAA4IBAQAuF1/VeQL73Y1FKrBX4bAb/Rdh2+Dadpi+w1pgEOi3P4udmQ+y
+Xn9GwwLRQmHRLjyCT5KT8lNHdldPdlBamqPGGku449aCAjA/YHVHhcHaXl0MtPGq
+BfKhHYSsvI2sIymlzZIvvIaf04yuJ1g+L0j8Px4Ecor9YwcKDZmpnIXLgdUtUrIQ
+5Omrb4jImX6q8jp6Bjplb4H3o4TqKoa74NLOWUiH5/Rix3Lo8MRoEVbX2GhKk+8n
+0eD3AuyrI1i+ce7zY8qGJKKFHGLDWPA/+006ZIS4j/Hr2FWo07CPFQ4/3gdJ8Erw
+SzgO9vvIhQrBJn2CIH4+P5Cb1ktdobNWW9XK
+-----END CERTIFICATE-----

sdk/python/tests/data/localhost.key

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCqhqeGLLgRRmM8
+D+bwhaTnQ7JOMoCepm8RbmCWqZMc/qVBt/JZCfOPc3Jz77hef3TUU7a/TuH7uUTc
+FJyOAJfqSWws4dGdZwtfyVDKBHrPM7457LXJe4J5sEDKGuyqBMML/zOgLF7Rbe3S
++q8TgAmze0kerljVtr9f06IoqwNm4xJKxt13ipUQvrIEPw9E0tP+KIPzgqrnwZm0
+4jd8mWzwT9thu+Sy6Bw8jeSaVTg+WDdWtWmUyXcNwzPsFu9A/qOZWp0+1RPv6UvQ
+LkKw2pt0M4jCDT34bC6z4D+Dw96sEF35kKZ29Jr0tgk6aUmzOBk8gv0BqBct+4ZC
+803ZjpajAgMBAAECggEADE4FHphxe8WheX8IQgjSumFXJ29bepc14oMdcyGvXOM/
+F3vnf+dI7Ov+sUD2A9OcoYmc4TcW9WwL/Pl7xn9iduRvatmsn3gFCRdkvf8OwY7R
+Riq/f1drNc6zDiJdO3N2g5IZrpAlE2WkSJoQMg8GJC5cO1uHS3yRWJ/Tzq1wZGcW
+Dot9hAFgN0qNdP0xFkOsPM5ptC3DjLqsZWboJhIM19hgsIYaWQWHvcYlCcWTVhkj
+FYzvLj5GrzAgyE89RpdXus670q5E2R2Rlnja21TfcxK0UOdIrKghZ0jxZMsXEwdB
+8V7kIzL5kh//RhT/dIt0mHNMSdLFFx3yMTb2wTzpWQKBgQDRiCRslDSjiNSFySkn
+6IivAwJtV2gLSxV05D9u9lrrlskHogrZUJkpVF1VzSnwv/ASaCZX4AGTtNPaz+vy
+yDviwfjADsuum8jkzoxKCHnR1HVMyX+vm/g+pE20PMskTUuDE4zROtrqo9Ky0afv
+94mJrf93Q815rsbEM5osugaeBQKBgQDQWAPTKy1wcG7edwfu3EaLYHPZ8pW9MldP
+FvCLTMwSDkSzU+wA4BGE/5Tuu0WHSAfUc5C1LnMQXKBQXun+YCaBR6GZjUAmntz3
+poBIOYaxe651zqzCmo4ip1h5wIfPvynsyGmhsbpDSNhvXFgH2mF3XSY1nduKSRHu
+389cHk3ahwKBgA4gAWSYcRv9I2aJcw7PrDcwGr/IPqlUPHQO1v/h96seFRtAnz6b
+IlgY6dnY5NTn+4UiJEOUREbyz71Weu949CCLNvurg6uXsOlLy0VKYPv2OJoek08B
+UrDWXq6h0of19fs2HC4Wq59Zv+ByJcIVi94OLsSZe4aSc6/SUrhlKgEJAoGBAIvR
+5Y88NNx2uBEYdPx6W+WBr34e7Rrxw+JSFNCHk5SyeqyWr5XOyjMliv/EMl8dmhOc
+Ewtkxte+MeB+Mi8CvBSay/rO7rR8fPK+jOzrnldSF7z8HLjlHGppQFlFOl/TfQFp
+ZmqbadNp+caShImQp0SCAPiOnh1p+F0FWpYJyFnVAoGAKhSRP0iUmd+tId94px2m
+G248BhcM9/0r+Y3yRX1eBx5eBzlzPUPcW1MSbhiZ1DIyLZ/MyObl98A1oNBGun11
+H/7Mq0E8BcJoXmt/6Z+2NhREBV9tDNuINyS/coYBV7H50pnSqyPpREPxNmu3Ukbm
+u7ggLRfH+DexDysbpbCZ9l4=
+-----END PRIVATE KEY-----

sdk/python/tests/data/localhost.pem

@@ -0,0 +1,18 @@
+-----BEGIN CERTIFICATE-----
+MIIC5zCCAc+gAwIBAgIJAKzukpnyuwsVMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV
+BAMMCWxvY2FsaG9zdDAgFw0yMDAyMTcxMTE4NDNaGA8zMDE5MDYyMDExMTg0M1ow
+FDESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB
+CgKCAQEAqoanhiy4EUZjPA/m8IWk50OyTjKAnqZvEW5glqmTHP6lQbfyWQnzj3Ny
+c++4Xn901FO2v07h+7lE3BScjgCX6klsLOHRnWcLX8lQygR6zzO+Oey1yXuCebBA
+yhrsqgTDC/8zoCxe0W3t0vqvE4AJs3tJHq5Y1ba/X9OiKKsDZuMSSsbdd4qVEL6y
+BD8PRNLT/iiD84Kq58GZtOI3fJls8E/bYbvksugcPI3kmlU4Plg3VrVplMl3DcMz
+7BbvQP6jmVqdPtUT7+lL0C5CsNqbdDOIwg09+Gwus+A/g8PerBBd+ZCmdvSa9LYJ
+OmlJszgZPIL9AagXLfuGQvNN2Y6WowIDAQABozowODAUBgNVHREEDTALgglsb2Nh
+bGhvc3QwCwYDVR0PBAQDAgeAMBMGA1UdJQQMMAoGCCsGAQUFBwMBMA0GCSqGSIb3
+DQEBCwUAA4IBAQAuF1/VeQL73Y1FKrBX4bAb/Rdh2+Dadpi+w1pgEOi3P4udmQ+y
+Xn9GwwLRQmHRLjyCT5KT8lNHdldPdlBamqPGGku449aCAjA/YHVHhcHaXl0MtPGq
+BfKhHYSsvI2sIymlzZIvvIaf04yuJ1g+L0j8Px4Ecor9YwcKDZmpnIXLgdUtUrIQ
+5Omrb4jImX6q8jp6Bjplb4H3o4TqKoa74NLOWUiH5/Rix3Lo8MRoEVbX2GhKk+8n
+0eD3AuyrI1i+ce7zY8qGJKKFHGLDWPA/+006ZIS4j/Hr2FWo07CPFQ4/3gdJ8Erw
+SzgO9vvIhQrBJn2CIH4+P5Cb1ktdobNWW9XK
+-----END CERTIFICATE-----