Fix materialize for None (#1481)

* fix materialize for None Signed-off-by: qooba <dev@qooba.net> * fix materialize for None Signed-off-by: qooba <dev@qooba.net> * fix materialize for None Signed-off-by: qooba <dev@qooba.net> * fix materialize for None - add test Signed-off-by: qooba <dev@qooba.net> * Fix materialize - correct lint errors Signed-off-by: qooba <dev@qooba.net>
feast-dev · May 10, 2021 · b482e21 · b482e21
1 parent 8712826
commit b482e21
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 8 deletions.
diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py
@@ -296,7 +296,7 @@ def _coerce_datetime(ts):
         feature_dict = {}
         for feature in feature_view.features:
             idx = table.column_names.index(feature.name)
-            value = python_value_to_proto_value(row[idx])
+            value = python_value_to_proto_value(row[idx], feature.dtype)
             feature_dict[feature.name] = value
         event_timestamp_idx = table.column_names.index(
             feature_view.input.event_timestamp_column

diff --git a/sdk/python/feast/type_map.py b/sdk/python/feast/type_map.py
@@ -327,8 +327,10 @@ def _python_value_to_proto_value(feast_value_type, value) -> ProtoValue:
     raise Exception(f"Unsupported data type: ${str(type(value))}")
 
 
-def python_value_to_proto_value(value: Any) -> ProtoValue:
-    value_type = python_type_to_feast_value_type("", value)
+def python_value_to_proto_value(
+    value: Any, feature_type: ValueType = None
+) -> ProtoValue:
+    value_type = python_type_to_feast_value_type("", value) if value else feature_type
     return _python_value_to_proto_value(value_type, value)
 
 

diff --git a/sdk/python/tests/test_offline_online_store_consistency.py b/sdk/python/tests/test_offline_online_store_consistency.py
@@ -4,7 +4,7 @@
 import uuid
 from datetime import datetime, timedelta
 from pathlib import Path
-from typing import Iterator, Tuple, Union
+from typing import Iterator, Optional, Tuple, Union
 
 import pandas as pd
 import pytest
@@ -26,7 +26,7 @@ def create_dataset() -> pd.DataFrame:
     ts = pd.Timestamp(now).round("ms")
     data = {
         "id": [1, 2, 1, 3, 3],
-        "value": [0.1, 0.2, 0.3, 4, 5],
+        "value": [0.1, None, 0.3, 4, 5],
         "ts_1": [
             ts - timedelta(hours=4),
             ts,
@@ -147,13 +147,17 @@ def check_offline_and_online_features(
     fv: FeatureView,
     driver_id: int,
     event_timestamp: datetime,
-    expected_value: float,
+    expected_value: Optional[float],
 ) -> None:
     # Check online store
     response_dict = fs.get_online_features(
         [f"{fv.name}:value"], [{"driver": driver_id}]
     ).to_dict()
-    assert abs(response_dict[f"{fv.name}__value"][0] - expected_value) < 1e-6
+
+    if expected_value:
+        assert abs(response_dict[f"{fv.name}__value"][0] - expected_value) < 1e-6
+    else:
+        assert response_dict[f"{fv.name}__value"][0] is None
 
     # Check offline store
     df = fs.get_historical_features(
@@ -163,7 +167,11 @@ def check_offline_and_online_features(
         feature_refs=[f"{fv.name}:value"],
     ).to_df()
 
-    assert abs(df.to_dict()[f"{fv.name}__value"][0] - expected_value) < 1e-6
+    if expected_value:
+        assert abs(df.to_dict()[f"{fv.name}__value"][0] - expected_value) < 1e-6
+    else:
+        df = df.where(pd.notnull(df), None)
+        assert df.to_dict()[f"{fv.name}__value"][0] is None
 
 
 def run_offline_online_store_consistency_test(
@@ -181,6 +189,10 @@ def run_offline_online_store_consistency_test(
         fs=fs, fv=fv, driver_id=1, event_timestamp=end_date, expected_value=0.3
     )
 
+    check_offline_and_online_features(
+        fs=fs, fv=fv, driver_id=2, event_timestamp=end_date, expected_value=None
+    )
+
     # check prior value for materialize_incremental()
     check_offline_and_online_features(
         fs=fs, fv=fv, driver_id=3, event_timestamp=end_date, expected_value=4