From 65f3b892c5818d40cf05024a9535283ba14f590a Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Wed, 2 Feb 2022 10:18:17 +0000 Subject: [PATCH 1/5] Add `last_updated_timestamp` field to ODFV Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> --- protos/feast/core/OnDemandFeatureView.proto | 3 +++ sdk/python/feast/on_demand_feature_view.py | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/protos/feast/core/OnDemandFeatureView.proto b/protos/feast/core/OnDemandFeatureView.proto index 31fe90a9ba..58feff5bfd 100644 --- a/protos/feast/core/OnDemandFeatureView.proto +++ b/protos/feast/core/OnDemandFeatureView.proto @@ -55,6 +55,9 @@ message OnDemandFeatureViewSpec { message OnDemandFeatureViewMeta { // Time where this Feature View is created google.protobuf.Timestamp created_timestamp = 1; + + // Time where this Feature View is last updated + google.protobuf.Timestamp last_updated_timestamp = 2; } message OnDemandInput { diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index 789422add4..c008e4dcdd 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -1,7 +1,8 @@ import copy import functools +from datetime import datetime from types import MethodType -from typing import Dict, List, Type, Union +from typing import Dict, List, Optional, Type, Union import dill import pandas as pd @@ -47,6 +48,7 @@ class OnDemandFeatureView(BaseFeatureView): input_feature_view_projections: Dict[str, FeatureViewProjection] input_request_data_sources: Dict[str, RequestDataSource] udf: MethodType + last_updated_timestamp: Optional[datetime] = None @log_exceptions def __init__( @@ -72,6 +74,8 @@ def __init__( self.udf = udf + self.last_updated_timestamp: Optional[datetime] = None + @property def proto_class(self) -> Type[OnDemandFeatureViewProto]: return OnDemandFeatureViewProto @@ -119,6 +123,8 @@ def to_proto(self) -> OnDemandFeatureViewProto: meta = OnDemandFeatureViewMeta() if self.created_timestamp: meta.created_timestamp.FromDatetime(self.created_timestamp) + if self.last_updated_timestamp: + meta.last_updated_timestamp.FromDatetime(self.last_updated_timestamp) inputs = {} for input_ref, fv_projection in self.input_feature_view_projections.items(): inputs[input_ref] = OnDemandInput( @@ -194,6 +200,10 @@ def from_proto(cls, on_demand_feature_view_proto: OnDemandFeatureViewProto): on_demand_feature_view_obj.created_timestamp = ( on_demand_feature_view_proto.meta.created_timestamp.ToDatetime() ) + if on_demand_feature_view_proto.meta.HasField("last_updated_timestamp"): + on_demand_feature_view_obj.last_updated_timestamp = ( + on_demand_feature_view_proto.meta.last_updated_timestamp.ToDatetime() + ) return on_demand_feature_view_obj From c1080e140b7fbd7fcd42a76b249caf7903ef7f25 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Wed, 2 Feb 2022 10:21:08 +0000 Subject: [PATCH 2/5] Add missing fields to FeatureView classes Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> --- sdk/python/feast/base_feature_view.py | 4 ++++ sdk/python/feast/feature_view.py | 1 + sdk/python/feast/on_demand_feature_view.py | 3 ++- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/base_feature_view.py b/sdk/python/feast/base_feature_view.py index 97180266d7..1892df264b 100644 --- a/sdk/python/feast/base_feature_view.py +++ b/sdk/python/feast/base_feature_view.py @@ -28,12 +28,16 @@ class BaseFeatureView(ABC): """A FeatureView defines a logical grouping of features to be served.""" + created_timestamp: Optional[datetime] = None + last_updated_timestamp: Optional[datetime] = None + @abstractmethod def __init__(self, name: str, features: List[Feature]): self._name = name self._features = features self._projection = FeatureViewProjection.from_definition(self) self.created_timestamp: Optional[datetime] = None + self.last_updated_timestamp: Optional[datetime] = None @property def name(self) -> str: diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index 57b60c0503..60bf980c7b 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -75,6 +75,7 @@ class FeatureView(BaseFeatureView): input: DataSource batch_source: DataSource stream_source: Optional[DataSource] = None + created_timestamp: Optional[datetime] = None last_updated_timestamp: Optional[datetime] = None materialization_intervals: List[Tuple[datetime, datetime]] diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index c008e4dcdd..b83db22a2c 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -48,6 +48,7 @@ class OnDemandFeatureView(BaseFeatureView): input_feature_view_projections: Dict[str, FeatureViewProjection] input_request_data_sources: Dict[str, RequestDataSource] udf: MethodType + created_timestamp: Optional[datetime] = None last_updated_timestamp: Optional[datetime] = None @log_exceptions @@ -73,7 +74,7 @@ def __init__( self.input_feature_view_projections[input_ref] = odfv_input.projection self.udf = udf - + self.created_timestamp: Optional[datetime] = None self.last_updated_timestamp: Optional[datetime] = None @property From a736e2f149d6483f32abc9383b75bd0b2260fd25 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Wed, 2 Feb 2022 10:22:08 +0000 Subject: [PATCH 3/5] Set `last_updated_timestamp` when applying FeatureView Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> --- sdk/python/feast/registry.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index 3352c54ffd..028bffa6c4 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -374,7 +374,8 @@ def apply_feature_view( """ feature_view.ensure_valid() if not feature_view.created_timestamp: - feature_view.created_timestamp = datetime.now() + feature_view.created_timestamp = datetime.utcnow() + feature_view.last_updated_timestamp = datetime.utcnow() feature_view_proto = feature_view.to_proto() feature_view_proto.spec.project = project self._prepare_registry_for_changes() From 8aa39d3713065014262b44c9ece44f409b1ed89d Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Wed, 2 Feb 2022 10:54:09 +0000 Subject: [PATCH 4/5] Correctly set created and updated fields Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> --- sdk/python/feast/registry.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/sdk/python/feast/registry.py b/sdk/python/feast/registry.py index 028bffa6c4..23cd0d1ab9 100644 --- a/sdk/python/feast/registry.py +++ b/sdk/python/feast/registry.py @@ -231,6 +231,12 @@ def apply_entity(self, entity: Entity, project: str, commit: bool = True): commit: Whether the change should be persisted immediately """ entity.is_valid() + + now = datetime.utcnow() + if not entity.created_timestamp: + entity._created_timestamp = now + entity._last_updated_timestamp = now + entity_proto = entity.to_proto() entity_proto.spec.project = project self._prepare_registry_for_changes() @@ -278,6 +284,11 @@ def apply_feature_service( feature_service: A feature service that will be registered project: Feast project that this entity belongs to """ + now = datetime.utcnow() + if not feature_service.created_timestamp: + feature_service.created_timestamp = now + feature_service.last_updated_timestamp = now + feature_service_proto = feature_service.to_proto() feature_service_proto.spec.project = project @@ -373,9 +384,12 @@ def apply_feature_view( commit: Whether the change should be persisted immediately """ feature_view.ensure_valid() + + now = datetime.utcnow() if not feature_view.created_timestamp: - feature_view.created_timestamp = datetime.utcnow() - feature_view.last_updated_timestamp = datetime.utcnow() + feature_view.created_timestamp = now + feature_view.last_updated_timestamp = now + feature_view_proto = feature_view.to_proto() feature_view_proto.spec.project = project self._prepare_registry_for_changes() @@ -499,6 +513,7 @@ def apply_materialization( existing_feature_view.materialization_intervals.append( (start_date, end_date) ) + existing_feature_view.last_updated_timestamp = datetime.utcnow() feature_view_proto = existing_feature_view.to_proto() feature_view_proto.spec.project = project del self.cached_registry_proto.feature_views[idx] @@ -687,6 +702,11 @@ def apply_saved_dataset( project: Feast project that this dataset belongs to commit: Whether the change should be persisted immediately """ + now = datetime.utcnow() + if not saved_dataset.created_timestamp: + saved_dataset.created_timestamp = now + saved_dataset.last_updated_timestamp = now + saved_dataset_proto = saved_dataset.to_proto() saved_dataset_proto.spec.project = project self._prepare_registry_for_changes() From 5de401d0fc823787ef020b76060bb3021e8b6c21 Mon Sep 17 00:00:00 2001 From: Judah Rand <17158624+judahrand@users.noreply.github.com> Date: Wed, 2 Feb 2022 18:45:13 +0000 Subject: [PATCH 5/5] Remove logic duplicated by parent class Signed-off-by: Judah Rand <17158624+judahrand@users.noreply.github.com> --- sdk/python/feast/base_feature_view.py | 4 ++-- sdk/python/feast/feature_view.py | 7 +------ sdk/python/feast/on_demand_feature_view.py | 7 +------ 3 files changed, 4 insertions(+), 14 deletions(-) diff --git a/sdk/python/feast/base_feature_view.py b/sdk/python/feast/base_feature_view.py index 1892df264b..b2178ec631 100644 --- a/sdk/python/feast/base_feature_view.py +++ b/sdk/python/feast/base_feature_view.py @@ -28,8 +28,8 @@ class BaseFeatureView(ABC): """A FeatureView defines a logical grouping of features to be served.""" - created_timestamp: Optional[datetime] = None - last_updated_timestamp: Optional[datetime] = None + created_timestamp: Optional[datetime] + last_updated_timestamp: Optional[datetime] @abstractmethod def __init__(self, name: str, features: List[Feature]): diff --git a/sdk/python/feast/feature_view.py b/sdk/python/feast/feature_view.py index 60bf980c7b..2c1d0675d4 100644 --- a/sdk/python/feast/feature_view.py +++ b/sdk/python/feast/feature_view.py @@ -74,9 +74,7 @@ class FeatureView(BaseFeatureView): online: bool input: DataSource batch_source: DataSource - stream_source: Optional[DataSource] = None - created_timestamp: Optional[datetime] = None - last_updated_timestamp: Optional[datetime] = None + stream_source: Optional[DataSource] materialization_intervals: List[Tuple[datetime, datetime]] @log_exceptions @@ -137,9 +135,6 @@ def __init__( self.materialization_intervals = [] - self.created_timestamp: Optional[datetime] = None - self.last_updated_timestamp: Optional[datetime] = None - # Note: Python requires redefining hash in child classes that override __eq__ def __hash__(self): return super().__hash__() diff --git a/sdk/python/feast/on_demand_feature_view.py b/sdk/python/feast/on_demand_feature_view.py index b83db22a2c..04b7f33cc6 100644 --- a/sdk/python/feast/on_demand_feature_view.py +++ b/sdk/python/feast/on_demand_feature_view.py @@ -1,8 +1,7 @@ import copy import functools -from datetime import datetime from types import MethodType -from typing import Dict, List, Optional, Type, Union +from typing import Dict, List, Type, Union import dill import pandas as pd @@ -48,8 +47,6 @@ class OnDemandFeatureView(BaseFeatureView): input_feature_view_projections: Dict[str, FeatureViewProjection] input_request_data_sources: Dict[str, RequestDataSource] udf: MethodType - created_timestamp: Optional[datetime] = None - last_updated_timestamp: Optional[datetime] = None @log_exceptions def __init__( @@ -74,8 +71,6 @@ def __init__( self.input_feature_view_projections[input_ref] = odfv_input.projection self.udf = udf - self.created_timestamp: Optional[datetime] = None - self.last_updated_timestamp: Optional[datetime] = None @property def proto_class(self) -> Type[OnDemandFeatureViewProto]: