From 9eb0d4702177d1cb2581afe305cd6ac4499c1e6c Mon Sep 17 00:00:00 2001 From: Pradithya Aria Date: Fri, 18 Jan 2019 18:56:51 +0800 Subject: [PATCH 1/8] Change serving to only get latest value of feature --- protos/feast/serving/Serving.proto | 68 ++-- .../serving/grpc/ServingGrpcService.java | 29 +- .../serving/http/ServingHttpService.java | 14 +- .../main/java/feast/serving/model/Pair.java | 45 --- .../serving/model/RequestDetailWithSpec.java | 31 -- .../service/BigTableFeatureStorage.java | 206 ++-------- .../feast/serving/service/FeastServing.java | 42 +-- .../service/FeatureRetrievalDispatcher.java | 186 ++------- .../feast/serving/service/FeatureStorage.java | 62 +-- .../serving/service/RedisFeatureStorage.java | 348 +++-------------- .../feast/serving/util/EntityMapBuilder.java | 45 +-- .../serving/util/FeatureValueListBuilder.java | 166 -------- .../feast/serving/util/RequestHelper.java | 93 ++--- .../java/feast/serving/util/StatsUtil.java | 12 +- .../java/feast/serving/util/TimeUtil.java | 55 +-- .../feast/serving/grpc/FeastServingTest.java | 38 +- .../serving/grpc/ServingGrpcServiceTest.java | 165 ++++---- .../BigTableFeatureStorageTestITCase.java | 234 +++--------- .../FeatureRetrievalDispatcherTest.java | 135 +------ .../service/RedisFeatureStorageTest.java | 355 ++---------------- .../serving/testutil/BigTablePopulator.java | 141 ++----- .../testutil/FeatureStoragePopulator.java | 117 +----- .../serving/testutil/RedisPopulator.java | 76 +--- .../java/feast/serving/testutil/TimeUtil.java | 60 +++ .../serving/util/EntityMapBuilderTest.java | 103 ++--- .../serving/util/FeatureBuilderTest.java | 187 --------- 26 files changed, 579 insertions(+), 2434 deletions(-) delete mode 100644 serving/src/main/java/feast/serving/model/Pair.java delete mode 100644 serving/src/main/java/feast/serving/model/RequestDetailWithSpec.java delete mode 100644 serving/src/main/java/feast/serving/util/FeatureValueListBuilder.java create mode 100644 serving/src/test/java/feast/serving/testutil/TimeUtil.java delete mode 100644 serving/src/test/java/feast/serving/util/FeatureBuilderTest.java diff --git a/protos/feast/serving/Serving.proto b/protos/feast/serving/Serving.proto index cdaa9d76b9..a05a142ae7 100644 --- a/protos/feast/serving/Serving.proto +++ b/protos/feast/serving/Serving.proto @@ -26,51 +26,28 @@ option java_outer_classname = "ServingAPIProto"; option go_package = "github.com/gojek/feast/protos/generated/go/feast/serving"; service ServingAPI { - // Query features from Feast - rpc QueryFeatures (QueryFeatures.Request) returns (QueryFeatures.Response) {}; + rpc QueryFeatures (QueryFeaturesRequest) returns (QueryFeaturesResponse) {}; } -message QueryFeatures { - message Request { - // e.g. "driver", "customer", "city". - string entityName = 1; - // List of entity ID. - repeated string entityId = 2; - // List of request details, contains: featureId, type of query, and limit. - repeated RequestDetail requestDetails = 3; - // Filter specifying only to retrieve features having timestamp within this range. - TimestampRange timestampRange = 4; - } - - message Response { - // e.g. "driver", "customer", "city". - string entityName = 1; - // map of entity ID and its entity's properties. - map entities = 2; - } -} - -message RequestDetail { - // feature ID to be included in the query. +message QueryFeaturesRequest { + // e.g. "driver", "customer", "city". + string entityName = 1; + // List of entity ID. + repeated string entityId = 2; + // List of feature ID. // feature ID is in the form of [entity_name].[granularity].[feature_name] // e.g: "driver.day.total_accepted_booking" // all requested feature ID shall have same entity name. - string featureId = 1; - // request type either LAST or LIST. - // LAST : return only the latest value based on timestamp. - // LIST : return list of historical data sorted by timestamp. - RequestType type = 2; - // only applicable to LIST. - // length of the returned list <= limit. - // default = 0 - int32 limit = 3; + repeated string featureId = 3; + // [optional] time range filter. If not specified all feature with any timestamp will be returned. + TimestampRange timeRange = 4; } -enum RequestType { - // LAST : return only the latest value based on timestamp. (default) - LAST = 0; - // LIST : return list of historical data sorted by timestamp. - LIST = 1; +message QueryFeaturesResponse { + // Entity name of the response + string entityName = 1; + // map of entity ID and its entity's properties. + map entities = 2; } // range of timestamp for querying @@ -84,15 +61,12 @@ message TimestampRange { message Entity { // map of feature ID and its feature value. - map features = 1; + map features = 1; } -message FeatureValueList { - // list of feature value - // if "type" in "requestDetail" is "LAST", then the length will always be 1. - // if "type" in "requestDetail" is "LIST", then the length is <= "limit". - feast.types.ValueList valueList = 1; - // list of timestamp of the value. - // the i-th timestamps correspond to the i-th value. - feast.types.TimestampList timestampList = 2; +message FeatureValue { + // value of feature + feast.types.Value value = 1; + // timestamp of the feature + google.protobuf.Timestamp timestamp = 2; } diff --git a/serving/src/main/java/feast/serving/grpc/ServingGrpcService.java b/serving/src/main/java/feast/serving/grpc/ServingGrpcService.java index 6877ae33a3..51622d8ea6 100644 --- a/serving/src/main/java/feast/serving/grpc/ServingGrpcService.java +++ b/serving/src/main/java/feast/serving/grpc/ServingGrpcService.java @@ -17,16 +17,14 @@ package feast.serving.grpc; -import static feast.serving.util.RequestHelper.checkTimestampRange; import static feast.serving.util.RequestHelper.validateRequest; import static feast.serving.util.StatsUtil.makeStatsdTags; import static io.grpc.Status.Code.INTERNAL; -import static io.grpc.Status.Code.NOT_FOUND; import com.timgroup.statsd.StatsDClient; import feast.serving.ServingAPIGrpc.ServingAPIImplBase; -import feast.serving.ServingAPIProto.QueryFeatures.Request; -import feast.serving.ServingAPIProto.QueryFeatures.Response; +import feast.serving.ServingAPIProto.QueryFeaturesRequest; +import feast.serving.ServingAPIProto.QueryFeaturesResponse; import feast.serving.service.FeastServing; import feast.serving.util.TimeUtil; import io.grpc.Status; @@ -39,9 +37,7 @@ import org.lognet.springboot.grpc.GRpcService; import org.springframework.beans.factory.annotation.Autowired; -/** - * Grpc service implementation for Serving API. - */ +/** Grpc service implementation for Serving API. */ @Slf4j @GRpcService public class ServingGrpcService extends ServingAPIImplBase { @@ -57,25 +53,19 @@ public ServingGrpcService(FeastServing feast, Tracer tracer, StatsDClient statsD this.statsDClient = statsDClient; } - /** - * Query feature values from Feast storage. - */ + /** Query feature values from Feast storage. */ @Override - public void queryFeatures(Request request, StreamObserver responseObserver) { + public void queryFeatures(QueryFeaturesRequest request, StreamObserver responseObserver) { long currentMicro = TimeUtil.microTime(); - Span span = - tracer - .buildSpan("ServingGrpcService-queryFeatures") - .start(); + Span span = tracer.buildSpan("ServingGrpcService-queryFeatures").start(); String[] tags = makeStatsdTags(request); statsDClient.increment("query_features_count", tags); statsDClient.gauge("query_features_entity_count", request.getEntityIdCount(), tags); - statsDClient.gauge("query_features_feature_count", request.getRequestDetailsCount(), tags); + statsDClient.gauge("query_features_feature_count", request.getFeatureIdCount(), tags); try (Scope scope = tracer.scopeManager().activate(span, false)) { Span innerSpan = scope.span(); validateRequest(request); - Request validRequest = checkTimestampRange(request); - Response response = feast.queryFeatures(validRequest); + QueryFeaturesResponse response = feast.queryFeatures(request); innerSpan.log("calling onNext"); responseObserver.onNext(response); @@ -90,8 +80,7 @@ public void queryFeatures(Request request, StreamObserver responseObse new StatusRuntimeException( Status.fromCode(INTERNAL).withDescription(e.getMessage()).withCause(e))); } finally { - statsDClient - .gauge("query_features_latency_us", TimeUtil.microTime() - currentMicro, tags); + statsDClient.gauge("query_features_latency_us", TimeUtil.microTime() - currentMicro, tags); span.finish(); } } diff --git a/serving/src/main/java/feast/serving/http/ServingHttpService.java b/serving/src/main/java/feast/serving/http/ServingHttpService.java index ef50d45bcd..48164a9d49 100644 --- a/serving/src/main/java/feast/serving/http/ServingHttpService.java +++ b/serving/src/main/java/feast/serving/http/ServingHttpService.java @@ -17,17 +17,16 @@ package feast.serving.http; -import feast.serving.ServingAPIProto.QueryFeatures; -import feast.serving.ServingAPIProto.QueryFeatures.Request; +import static feast.serving.util.RequestHelper.validateRequest; + +import feast.serving.ServingAPIProto.QueryFeaturesRequest; +import feast.serving.ServingAPIProto.QueryFeaturesResponse; import feast.serving.service.FeastServing; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; -import static feast.serving.util.RequestHelper.checkTimestampRange; -import static feast.serving.util.RequestHelper.validateRequest; - /** HTTP endpoint for Serving API. */ @RestController public class ServingHttpService { @@ -42,9 +41,8 @@ public ServingHttpService(FeastServing feastServing) { value = "/api/v1/features/request", produces = "application/json", consumes = "application/json") - public QueryFeatures.Response queryFeature(@RequestBody QueryFeatures.Request request) { + public QueryFeaturesResponse queryFeature(@RequestBody QueryFeaturesRequest request) { validateRequest(request); - Request validRequest = checkTimestampRange(request); - return feastServing.queryFeatures(validRequest); + return feastServing.queryFeatures(request); } } diff --git a/serving/src/main/java/feast/serving/model/Pair.java b/serving/src/main/java/feast/serving/model/Pair.java deleted file mode 100644 index ac4035ee38..0000000000 --- a/serving/src/main/java/feast/serving/model/Pair.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.serving.model; - -import lombok.EqualsAndHashCode; - -/** - * Pair value containing L-typed {@code left} and R-typed {@code right}. - * - * @param type of the left value. - * @param type of the right value. - */ -@EqualsAndHashCode -public class Pair { - private final L left; - private final R right; - - public Pair(L left, R right) { - this.left = left; - this.right = right; - } - - public L getLeft() { - return left; - } - - public R getRight() { - return right; - } -} diff --git a/serving/src/main/java/feast/serving/model/RequestDetailWithSpec.java b/serving/src/main/java/feast/serving/model/RequestDetailWithSpec.java deleted file mode 100644 index d98b182849..0000000000 --- a/serving/src/main/java/feast/serving/model/RequestDetailWithSpec.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.serving.model; - -import feast.serving.ServingAPIProto.RequestDetail; -import feast.specs.FeatureSpecProto.FeatureSpec; -import lombok.AllArgsConstructor; -import lombok.Data; - -/** Composite type to hold information about request detail and the associated feature spec. */ -@Data -@AllArgsConstructor -public class RequestDetailWithSpec { - RequestDetail requestDetail; - FeatureSpec featureSpec; -} diff --git a/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java b/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java index 50d2b775d6..d279249cde 100644 --- a/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java +++ b/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java @@ -23,18 +23,15 @@ import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.exception.FeatureRetrievalException; import feast.serving.model.FeatureValue; -import feast.serving.model.Pair; import feast.serving.util.TimeUtil; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.storage.BigTableProto.BigTableRowKey; -import feast.types.GranularityProto.Granularity; +import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.Value; import java.io.IOException; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.List; -import java.util.Map; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.hadoop.hbase.Cell; @@ -42,80 +39,34 @@ import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.Get; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.ResultScanner; -import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; -import org.apache.hadoop.hbase.filter.MultiRowRangeFilter; -import org.apache.hadoop.hbase.filter.MultiRowRangeFilter.RowRange; /** Connector to BigTable instance. */ @Slf4j public class BigTableFeatureStorage implements FeatureStorage { public static final String TYPE = "bigtable"; - + private static final byte[] DEFAULT_COLUMN_FAMILY = "default".getBytes(); public static String OPT_BIGTABLE_PROJECT = "project"; public static String OPT_BIGTABLE_INSTANCE = "instance"; public static String OPT_BIGTABLE_TABLE_PREFIX = "tablePrefix"; - public static String SERVING_OPT_BIGTABLE_COLUMN_FAMILY = "family"; - private final Connection connection; - private static final byte[] DEFAULT_COLUMN_FAMILY = "default".getBytes(); - public BigTableFeatureStorage(Connection connection) { this.connection = connection; } /** {@inheritDoc} */ @Override - public List getCurrentFeature( - String entityName, List entityIds, FeatureSpec featureSpec) - throws FeatureRetrievalException { - return getCurrentFeatureInternal(entityName, entityIds, featureSpec); - } - - /** {@inheritDoc} */ - @Override - public List getCurrentFeatures( - String entityName, List entityIds, List featureSpecs) { - List featureValues = new ArrayList<>(entityIds.size() * featureSpecs.size()); - for (FeatureSpec featureSpec : featureSpecs) { - featureValues.addAll(getCurrentFeatureInternal(entityName, entityIds, featureSpec)); - } - return featureValues; - } - - /** {@inheritDoc} */ - @Override - public List getNLatestFeatureWithinTimestampRange( + public List getFeature( String entityName, - List entityIds, - Pair featureSpecLimitPair, - TimestampRange tsRange) - throws FeatureRetrievalException { - FeatureSpec featureSpec = featureSpecLimitPair.getLeft(); - int limit = featureSpecLimitPair.getRight(); - - return getNLatestFeatureWithinTimestampRangeInternal( - entityName, entityIds, featureSpec, limit, tsRange); - } - - /** {@inheritDoc} */ - @Override - public List getNLatestFeaturesWithinTimestampRange( - String entityName, - List entityIds, - List> featureSpecAndLimitPairs, + Collection entityIds, + Collection featureSpecs, TimestampRange tsRange) { - List featureValues = new ArrayList<>(entityIds.size() * featureSpecAndLimitPairs.size()); - for(Pair featureSpecLimitPair: featureSpecAndLimitPairs) { - FeatureSpec featureSpec = featureSpecLimitPair.getLeft(); - int limit = featureSpecLimitPair.getRight(); - - featureValues.addAll(getNLatestFeatureWithinTimestampRangeInternal( - entityName, entityIds, featureSpec, limit, tsRange)); + List featureValues = new ArrayList<>(entityIds.size() * featureSpecs.size()); + for (FeatureSpec featureSpec : featureSpecs) { + featureValues.addAll(getCurrentFeatureInternal(entityName, entityIds, featureSpec, tsRange)); } return featureValues; } @@ -126,14 +77,18 @@ public List getNLatestFeaturesWithinTimestampRange( * @param entityName entity name. * @param entityIds list of entity id. * @param featureSpec spec of the feature. + * @param tsRange time range filter * @return list of feature value. */ private List getCurrentFeatureInternal( - String entityName, List entityIds, FeatureSpec featureSpec) { + String entityName, + Collection entityIds, + FeatureSpec featureSpec, + TimestampRange tsRange) { List features = new ArrayList<>(entityIds.size()); String featureId = featureSpec.getId(); byte[] featureIdBytes = featureSpec.getId().getBytes(); - List gets = createGets(entityIds, featureIdBytes, getColumnFamily(featureSpec)); + List gets = createGets(entityIds, featureSpec, tsRange); try (Table table = connection.getTable(TableName.valueOf(entityName))) { Result[] results = table.get(gets); for (Result result : results) { @@ -168,76 +123,25 @@ private List getCurrentFeatureInternal( } /** - * Internal implementation of getting N latest feature within certain time range given a list of entity ID. - * - * @param entityName entity name - * @param entityIds list of entity id for which the feature should be retrieved. - * @param featureSpec spec of the feature. - * @param limit maximum number of value to be retrieved per feature. - * @param tsRange timestamp range. - * @return list of feature value. - */ - private List getNLatestFeatureWithinTimestampRangeInternal( - String entityName, - List entityIds, - FeatureSpec featureSpec, - int limit, - TimestampRange tsRange) { - Scan scan = createScanFromFeatureSpec(featureSpec); - Granularity.Enum granularity = featureSpec.getGranularity(); - String featureId = featureSpec.getId(); - byte[] featureIdBytes = featureSpec.getId().getBytes(); - String revMilliStart = roundedReversedMillis(tsRange.getStart(), granularity); - String revMilliEnd = roundedReversedMillis(tsRange.getEnd(), granularity); - scan.setFilter(createMultiRowRangeFilter(entityIds, revMilliStart, revMilliEnd)); - - List features = new ArrayList<>(); - Map featureCount = new HashMap<>(); // map of entity ID to number of feature. - try (Table table = connection.getTable(TableName.valueOf(entityName)); - ResultScanner scanResult = table.getScanner(scan)) { - for (Result row : scanResult) { - Cell currentCell = row.getColumnLatestCell(getColumnFamily(featureSpec), featureIdBytes); - if (currentCell == null) { - continue; - } - - BigTableRowKey rowKey = BigTableRowKey.parseFrom(currentCell.getRowArray()); - String entityId = rowKey.getEntityKey(); - byte[] cellValue = currentCell.getValueArray(); - - if (cellValue == null) { - continue; - } - - Integer count = featureCount.computeIfAbsent(entityId, k -> 0); - if (count < limit) { - Timestamp timestamp = Timestamps.fromMillis(currentCell.getTimestamp()); - Value value = Value.parseFrom(cellValue); - FeatureValue featureValue = new FeatureValue(featureId, entityId, value, timestamp); - features.add(featureValue); - - count++; - featureCount.put(entityId, count); - } - } - - return features; - } catch (IOException e) { - log.error("Error while retrieving feature: {}", e); - throw new FeatureRetrievalException("Unable to retrieve feature from BigTable", e); - } - } - - /** - * Create list of BigTable's Get operation. + * Create list of get operation for retrieving a feature of several entities optionally filtered + * by its timestamp. * * @param entityIds list of entity ID. - * @param featureIdBytes byte array value of a feature ID. - * @param columnFamily byte array value of column family + * @param featureSpec feature spec + * @param tsRange time filter * @return list of Get operation. */ private List createGets( - Collection entityIds, byte[] featureIdBytes, byte[] columnFamily) { + Collection entityIds, FeatureSpec featureSpec, TimestampRange tsRange) { + byte[] featureIdBytes = featureSpec.getId().getBytes(); + byte[] columnFamily = getColumnFamily(featureSpec); + long start = 0; + long end = 0; + if (TimeUtil.isTimeFilterRequired(tsRange)) { + start = Timestamps.toMillis(tsRange.getStart()); + end = Timestamps.toMillis(tsRange.getEnd()); + } + List gets = new ArrayList<>(); for (String entityId : entityIds) { String entityIdPrefix = DigestUtils.sha1Hex(entityId.getBytes()).substring(0, 7); @@ -245,7 +149,9 @@ private List createGets( Get get = new Get(btKey.toByteArray()); get.addColumn(columnFamily, featureIdBytes); try { - // for some reason Get.readVersions has checked exception. + if (!featureSpec.getGranularity().equals(Enum.NONE) && tsRange != null) { + get.setTimeRange(start, end); + } get.readVersions(1); } catch (IOException e) { log.error("should not happen"); @@ -255,40 +161,6 @@ private List createGets( return gets; } - /** - * Create BigTable's scan operation for a certain feature. - * - * @param featureSpec spec of feature. - * @return BigTable's scan operation. - */ - private Scan createScanFromFeatureSpec(FeatureSpec featureSpec) { - Scan scan = new Scan(); - byte[] columnFamily = getColumnFamily(featureSpec); - scan.addColumn(columnFamily, featureSpec.getId().getBytes()); - scan.readVersions(1); - return scan; - } - - /** - * Create BigTables's multi row range filter to scan several entity ID within a timerange - * - * @param entityIds list of entity id to be scanned. - * @param reversedMilliStart starting timerange. - * @param reversedMillisEnd end timerange. - * @return instance of {@link MultiRowRangeFilter}. - */ - private MultiRowRangeFilter createMultiRowRangeFilter( - Collection entityIds, String reversedMilliStart, String reversedMillisEnd) { - List rowRangeList = new ArrayList<>(); - for (String entityId : entityIds) { - String entityIdPrefix = DigestUtils.sha1Hex(entityId.getBytes()).substring(0, 7); - BigTableRowKey startRow = createRowKey(entityIdPrefix, entityId, reversedMillisEnd); - BigTableRowKey stopRow = createRowKey(entityIdPrefix, entityId, reversedMilliStart); - rowRangeList.add(new RowRange(startRow.toByteArray(), true, stopRow.toByteArray(), true)); - } - return new MultiRowRangeFilter(rowRangeList); - } - /** * Create BigTableRowKey based on entityId, timestamp, and granularity. * @@ -306,23 +178,9 @@ private BigTableRowKey createRowKey( .build(); } - /** - * Get rounded and reversed millis value of a timestamp. - * - * @param ts timestamp - * @param granularity granularity of feature. - * @return string value of timestamp after being rounded and reversed. - */ - private String roundedReversedMillis(Timestamp ts, Granularity.Enum granularity) { - if (granularity.equals(Granularity.Enum.NONE)) { - return "0"; - } - Timestamp roundedEnd = TimeUtil.roundFloorTimestamp(ts, granularity); - return String.valueOf(Long.MAX_VALUE - roundedEnd.getSeconds() * 1000); - } - /** * Get column family of a feature from its spec. + * * @param fs feature's spec * @return byte array value of the column family. */ diff --git a/serving/src/main/java/feast/serving/service/FeastServing.java b/serving/src/main/java/feast/serving/service/FeastServing.java index 1194c3cf1e..ad89adf813 100644 --- a/serving/src/main/java/feast/serving/service/FeastServing.java +++ b/serving/src/main/java/feast/serving/service/FeastServing.java @@ -17,11 +17,10 @@ package feast.serving.service; +import com.google.common.collect.Sets; import feast.serving.ServingAPIProto.Entity; -import feast.serving.ServingAPIProto.QueryFeatures.Request; -import feast.serving.ServingAPIProto.QueryFeatures.Response; -import feast.serving.ServingAPIProto.RequestDetail; -import feast.serving.model.RequestDetailWithSpec; +import feast.serving.ServingAPIProto.QueryFeaturesRequest; +import feast.serving.ServingAPIProto.QueryFeaturesResponse; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.specs.StorageSpecProto.StorageSpec; import io.opentracing.Scope; @@ -66,15 +65,15 @@ public FeastServing( * @param request feature query request. * @return response of the query containing the feature values. */ - public Response queryFeatures(Request request) { + public QueryFeaturesResponse queryFeatures(QueryFeaturesRequest request) { try (Scope scope = tracer.buildSpan("FeastServing-queryFeatures").startActive(true)) { - List requestDetails = joinRequestDetailsWithFeatureSpec(request); + Collection featureSpecs = getFeatureSpecs(request.getFeatureIdList()); // create connection to feature storage if necessary checkAndConnectFeatureStorage( - requestDetails + featureSpecs .stream() - .map(r -> r.getFeatureSpec().getDataStores().getServing().getId()) + .map(featureSpec -> featureSpec.getDataStores().getServing().getId()) .collect(Collectors.toList())); scope.span().log("start retrieving all feature"); @@ -82,13 +81,13 @@ public Response queryFeatures(Request request) { featureRetrievalDispatcher.dispatchFeatureRetrieval( request.getEntityName(), request.getEntityIdList(), - requestDetails, - request.getTimestampRange()); + featureSpecs, + request.getTimeRange()); scope.span().log("finished retrieving all feature"); // build response - return Response.newBuilder() + return QueryFeaturesResponse.newBuilder() .setEntityName(request.getEntityName()) .putAllEntities(result) .build(); @@ -121,23 +120,14 @@ private void checkAndConnectFeatureStorage(Collection storageIds) { /** * Attach request details with associated feature spec. * - * @param request - * @return + * @param featureIds collection of feature ID + * @return collection of feature spec */ - private List joinRequestDetailsWithFeatureSpec(Request request) { + private Collection getFeatureSpecs(Collection featureIds) { // dedup feature ID. - Collection featuresId = - request - .getRequestDetailsList() - .stream() - .map(RequestDetail::getFeatureId) - .collect(Collectors.toSet()); + Collection featureIdSet = Sets.newHashSet(featureIds); - Map featureSpecMap = specStorage.getFeatureSpecs(featuresId); - return request - .getRequestDetailsList() - .stream() - .map(rd -> new RequestDetailWithSpec(rd, featureSpecMap.get(rd.getFeatureId()))) - .collect(Collectors.toList()); + Map featureSpecMap = specStorage.getFeatureSpecs(featureIdSet); + return featureSpecMap.values(); } } diff --git a/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java b/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java index 74b0e98a5d..7673476128 100644 --- a/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java +++ b/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java @@ -23,28 +23,22 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import feast.serving.ServingAPIProto.Entity; -import feast.serving.ServingAPIProto.RequestType; import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.config.AppConfig; import feast.serving.exception.FeatureRetrievalException; import feast.serving.model.FeatureValue; -import feast.serving.model.Pair; -import feast.serving.model.RequestDetailWithSpec; import feast.serving.util.EntityMapBuilder; import feast.specs.FeatureSpecProto.FeatureSpec; import io.opentracing.Scope; import io.opentracing.Span; import io.opentracing.Tracer; import java.util.ArrayList; -import java.util.HashMap; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; -import lombok.AllArgsConstructor; -import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -78,110 +72,48 @@ public FeatureRetrievalDispatcher( * * @param entityName entity name of the feature. * @param entityIds list of entity ids. - * @param requests list of request. + * @param featureSpecs list of request. * @param timestampRange timestamp range of feature to be retrieved. * @return map of entityID and Entity instance. */ public Map dispatchFeatureRetrieval( String entityName, - List entityIds, - List requests, + Collection entityIds, + Collection featureSpecs, TimestampRange timestampRange) { - GroupedRequest groupedRequest = groupRequestByTypeAndStorage(requests); + Map> groupedFeatureSpecs = groupByStorage(featureSpecs); - if (groupedRequest.getNbThreadRequired() <= 1) { - return runInCurrentThread( - entityName, entityIds, timestampRange, groupedRequest.getRequests()); + if (groupedFeatureSpecs.size() <= 1) { + return runInCurrentThread(entityName, entityIds, timestampRange, groupedFeatureSpecs); } else { - return runWithExecutorService( - entityName, entityIds, timestampRange, groupedRequest.getRequests()); + return runWithExecutorService(entityName, entityIds, timestampRange, groupedFeatureSpecs); } } - /** - * Group request by its type and storage ID of the feature. - * - * @param requestDetailWithSpecs list of requests. - * @return requests grouped by request type and storage ID. - */ - private GroupedRequest groupRequestByTypeAndStorage( - List requestDetailWithSpecs) { - Map> groupedByRequestType = - requestDetailWithSpecs.stream() - .collect(groupingBy(r -> r.getRequestDetail().getType())); - - int nbThreadRequired = 0; - Map>> groupedByTypeAndStorageId = - new HashMap<>(); - for (Map.Entry> requestEntry : - groupedByRequestType.entrySet()) { - RequestType requestType = requestEntry.getKey(); - List requestDetails = requestEntry.getValue(); - - Map> groupedByStorageId = - groupRequestByStorage(requestDetails); - groupedByTypeAndStorageId.put(requestType, groupedByStorageId); - - nbThreadRequired += groupedByStorageId.size(); - } - return new GroupedRequest(groupedByTypeAndStorageId, nbThreadRequired); - } - /** * Execute request in current thread. * * @param entityName entity name of of the feature. * @param entityIds list of entity ID of the feature to be retrieved. * @param tsRange timestamp range of the feature to be retrieved. - * @param groupedRequest request grouped by type and storage ID. + * @param groupedFeatureSpecs feature spec grouped by storage ID. * @return entity map containing the result of feature retrieval. */ private Map runInCurrentThread( String entityName, - List entityIds, + Collection entityIds, TimestampRange tsRange, - Map>> groupedRequest) { - try (Scope scope = tracer.buildSpan("FeatureRetrievalDispatcher-runInCurrentThread") - .startActive(true)) { - Span span = scope.span(); - if (groupedRequest.size() > 1) { - throw new IllegalArgumentException( - "runInCurrentThread required more than one thread to run"); - } - - RequestType requestType = groupedRequest.keySet().iterator().next(); - Map> request = groupedRequest.get(requestType); + Map> groupedFeatureSpecs) { + try (Scope scope = + tracer.buildSpan("FeatureRetrievalDispatcher-runInCurrentThread").startActive(true)) { - if (request.size() > 1) { - throw new IllegalArgumentException( - "runInCurrentThread required more than one thread to run"); - } - - String storageId = request.keySet().iterator().next(); - List requestDetailWithSpecs = request.get(storageId); + String storageId = groupedFeatureSpecs.keySet().iterator().next(); + List featureSpecs = groupedFeatureSpecs.get(storageId); FeatureStorage featureStorage = featureStorageRegistry.get(storageId); List featureValues; - if (RequestType.LAST == requestType) { - span.setTag("request-type", "last"); - List featureSpecs = - requestDetailWithSpecs - .stream() - .map(RequestDetailWithSpec::getFeatureSpec) - .collect(Collectors.toList()); - featureValues = featureStorage.getCurrentFeatures(entityName, entityIds, featureSpecs); - } else { - span.setTag("request-type", "list"); - List> featureSpecLimitPairs = - requestDetailWithSpecs - .stream() - .map(r -> new Pair<>(r.getFeatureSpec(), r.getRequestDetail().getLimit())) - .collect(Collectors.toList()); - featureValues = - featureStorage.getNLatestFeaturesWithinTimestampRange( - entityName, entityIds, featureSpecLimitPairs, tsRange); - } + featureValues = featureStorage.getFeature(entityName, entityIds, featureSpecs, tsRange); EntityMapBuilder builder = new EntityMapBuilder(); builder.addFeatureValueList(featureValues); @@ -195,60 +127,31 @@ private Map runInCurrentThread( * @param entityName entity name of the feature. * @param entityIds list of entity ID. * @param tsRange timestamp range of the feature. - * @param groupedRequest request grouped by type and serving storage ID. + * @param groupedFeatureSpec feature specs grouped by serving storage ID. * @return entity map containing result of feature retrieval. */ private Map runWithExecutorService( String entityName, - List entityIds, + Collection entityIds, TimestampRange tsRange, - Map>> groupedRequest) { - try (Scope scope = tracer.buildSpan("FeatureRetrievalDispatcher-runWithExecutorService") - .startActive(true)) { + Map> groupedFeatureSpec) { + try (Scope scope = + tracer.buildSpan("FeatureRetrievalDispatcher-runWithExecutorService") + .startActive(true)) { Span span = scope.span(); List> futures = new ArrayList<>(); EntityMapBuilder entityMapBuilder = new EntityMapBuilder(); - for (Map.Entry>> requestEntryPerType : - groupedRequest.entrySet()) { - RequestType requestType = requestEntryPerType.getKey(); - for (Map.Entry> requestEntryPerStorage : - requestEntryPerType.getValue().entrySet()) { - List requests = requestEntryPerStorage.getValue(); - FeatureStorage featureStorage = featureStorageRegistry - .get(requestEntryPerStorage.getKey()); - if (requestType == RequestType.LAST) { - List featureSpecs = - requests - .stream() - .map(RequestDetailWithSpec::getFeatureSpec) - .collect(Collectors.toList()); - - futures.add( - executorService.submit( - () -> { - List featureValues = - featureStorage.getCurrentFeatures(entityName, entityIds, featureSpecs); - entityMapBuilder.addFeatureValueList(featureValues); - return null; - })); - } else { - List> featureSpecLimitPairs = - requests - .stream() - .map(r -> new Pair<>(r.getFeatureSpec(), r.getRequestDetail().getLimit())) - .collect(Collectors.toList()); - - futures.add( - executorService.submit( - () -> { - List featureValues = - featureStorage.getNLatestFeaturesWithinTimestampRange( - entityName, entityIds, featureSpecLimitPairs, tsRange); - entityMapBuilder.addFeatureValueList(featureValues); - return null; - })); - } - } + for (Map.Entry> entry : groupedFeatureSpec.entrySet()) { + FeatureStorage featureStorage = featureStorageRegistry.get(entry.getKey()); + List featureSpecs = entry.getValue(); + futures.add( + executorService.submit( + () -> { + List featureValues = + featureStorage.getFeature(entityName, entityIds, featureSpecs, tsRange); + entityMapBuilder.addFeatureValueList(featureValues); + return null; + })); } span.log("submitted all task"); ListenableFuture> combined = Futures.allAsList(futures); @@ -273,27 +176,12 @@ private Map runWithExecutorService( /** * Group request by its serving storage ID. * - * @param requestDetailWithSpecs list of request. + * @param featureSpecs list of request. * @return request grouped by serving storage ID. */ - private Map> groupRequestByStorage( - List requestDetailWithSpecs) { - return requestDetailWithSpecs + private Map> groupByStorage(Collection featureSpecs) { + return featureSpecs .stream() - .collect(groupingBy(r -> r.getFeatureSpec().getDataStores().getServing().getId())); - } - - @AllArgsConstructor - @Getter - private static class GroupedRequest { - - /** - * request grouped by type and its serving storage ID - */ - private final Map>> requests; - /** - * number of thread required to execute all request in parallel - */ - private final int nbThreadRequired; + .collect(groupingBy(featureSpec -> featureSpec.getDataStores().getServing().getId())); } } diff --git a/serving/src/main/java/feast/serving/service/FeatureStorage.java b/serving/src/main/java/feast/serving/service/FeatureStorage.java index 992ac5ae6c..ebcd2675e3 100644 --- a/serving/src/main/java/feast/serving/service/FeatureStorage.java +++ b/serving/src/main/java/feast/serving/service/FeatureStorage.java @@ -20,74 +20,26 @@ import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.exception.FeatureRetrievalException; import feast.serving.model.FeatureValue; -import feast.serving.model.Pair; import feast.specs.FeatureSpecProto.FeatureSpec; +import java.util.Collection; import java.util.List; /** Abstraction of Feast Storage. */ public interface FeatureStorage { - /** - * Get current value of the given feature for the specified entities. - * - *

It returns the most recent value if the feature is time-series (granularity other than NONE) - * - * @param entityName entity name, e.g. 'driver', 'customer', 'area' - * @param entityIds list of entity id. - * @param featureSpec feature spec for which the feature should be retrieved. - * @return list of feature value. - * @throws FeatureRetrievalException if anything goes wrong during feature retrieval. - */ - List getCurrentFeature( - String entityName, List entityIds, FeatureSpec featureSpec); /** * Get current value of several feature for the specified entities. * - *

It returns the most recent value if the feature is time-series (granularity other than NONE) - * * @param entityName entity name, e.g. 'driver', 'customer', 'area' * @param entityIds list of entity id. - * @param featureSpecs list of feature spec for which the feature should be retrieved. + * @param featureSpecs list of feature spec for which the feature should be retrieved. * @param + * @param tsRange allowed event timestamp of the feature to be returned. Pass null to not filter * + * by time. * @return list of feature value. * @throws FeatureRetrievalException if anything goes wrong during feature retrieval. */ - List getCurrentFeatures( - String entityName, List entityIds, List featureSpecs); - - /** - * Get N latest value of a feature within a timestamp range. - * - *

The number of data returned is less than or equal to {@code n}. - * - * @param entityName entity name - * @param entityIds list of entity id that the feature should be retrieved. - * @param featureSpecAndLimitPair pair between feature spec and the maximum number of value to be - * returned. - * @param tsRange timerange filter. - * @return map of entity id and the feature values. - * @throws FeatureRetrievalException if anything goes wrong during feature retrieval. - */ - List getNLatestFeatureWithinTimestampRange( - String entityName, - List entityIds, - Pair featureSpecAndLimitPair, - TimestampRange tsRange); - - /** - * Get N latest value of several features within a timestamp range. - * - *

The number of data returned is less than or equal to {@code n}. - * - * @param entityName entity name - * @param entityIds list of entity id that the feature should be retrieved. - * @param featureSpecAndLimitPairs list of pair between feature spec and the maximum number of - * value to be returned. - * @param tsRange timerange filter. - * @return map of entity id and the feature values. - * @throws FeatureRetrievalException if anything goes wrong during feature retrieval. - */ - List getNLatestFeaturesWithinTimestampRange( + List getFeature( String entityName, - List entityIds, - List> featureSpecAndLimitPairs, + Collection entityIds, + Collection featureSpecs, TimestampRange tsRange); } diff --git a/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java b/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java index 71c9a4b449..3bff50549d 100644 --- a/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java +++ b/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java @@ -18,54 +18,43 @@ package feast.serving.service; import com.google.protobuf.InvalidProtocolBufferException; +import com.google.protobuf.Timestamp; +import com.google.protobuf.util.Timestamps; import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.exception.FeatureRetrievalException; import feast.serving.model.FeatureValue; -import feast.serving.model.Pair; -import feast.serving.util.SpecUtil; import feast.serving.util.TimeUtil; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.storage.RedisProto.RedisBucketKey; import feast.storage.RedisProto.RedisBucketValue; +import feast.types.GranularityProto.Granularity.Enum; import io.opentracing.Scope; import io.opentracing.Span; import io.opentracing.Tracer; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; -import java.util.HashMap; import java.util.List; -import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import lombok.AllArgsConstructor; import lombok.Getter; -import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisPool; -import redis.clients.jedis.Pipeline; -import redis.clients.jedis.Response; -/** - * Class for retrieving features from a Redis instance. - */ +/** Class for retrieving features from a Redis instance. */ @Slf4j public class RedisFeatureStorage implements FeatureStorage { public static final String TYPE = "redis"; - - public static String OPT_REDIS_HOST = "host"; - public static String OPT_REDIS_PORT = "port"; - - public static String OPT_REDIS_BUCKET_SIZE = "bucketSize"; // ISO8601 Period - // BUCKET_ID_ZERO is used to identify feature with granularity NONE or latest value of a // time-series feature. private static final long BUCKET_ID_ZERO = 0; - private static final byte[][] ZERO_LENGTH_ARRAY = new byte[0][0]; - + public static String OPT_REDIS_HOST = "host"; + public static String OPT_REDIS_PORT = "port"; + public static String OPT_REDIS_BUCKET_SIZE = "bucketSize"; // ISO8601 Period private final JedisPool jedisPool; private final Tracer tracer; @@ -79,145 +68,44 @@ public RedisFeatureStorage(JedisPool jedisPool, Tracer tracer) { this.tracer = tracer; } - /** - * {@inheritDoc} - */ - @Override - public List getCurrentFeature( - String entityName, List entityIds, FeatureSpec featureSpec) - throws FeatureRetrievalException { - try (Scope scope = tracer.buildSpan("Redis-getCurrentFeature").startActive(true)) { - String featureId = featureSpec.getId(); - String featureIdSha1Prefix = makeFeatureIdSha1Prefix(featureId); - List getRequests = new ArrayList<>(entityIds.size()); - for (String entityId : entityIds) { - RedisBucketKey key = makeBucketKey(entityId, featureIdSha1Prefix, BUCKET_ID_ZERO); - getRequests.add(new GetRequest(entityId, featureId, key)); - } - scope.span().log("completed request creation"); - return sendAndProcessMultiGet(getRequests); - } - } - - /** - * {@inheritDoc} - */ + /** {@inheritDoc} */ @Override - public List getCurrentFeatures( - String entityName, List entityIds, List featureSpecs) { - try (Scope scope = tracer.buildSpan("Redis-getCurrentFeatures").startActive(true)) { + public List getFeature( + String entityName, + Collection entityIds, + Collection featureSpecs, + TimestampRange tsRange) { + try (Scope scope = tracer.buildSpan("Redis-getFeature").startActive(true)) { List getRequests = new ArrayList<>(entityIds.size() * featureSpecs.size()); + // Granularity NONE is currently treated differently since the event timestamp is always EPOCH + List getRequestsGranularityNone = new ArrayList<>(); for (FeatureSpec featureSpec : featureSpecs) { String featureId = featureSpec.getId(); String featureIdSha1Prefix = makeFeatureIdSha1Prefix(featureId); for (String entityId : entityIds) { RedisBucketKey key = makeBucketKey(entityId, featureIdSha1Prefix, BUCKET_ID_ZERO); - getRequests.add(new GetRequest(entityId, featureId, key)); - } - } - scope.span().log("completed request creation"); - return sendAndProcessMultiGet(getRequests); - } - } - - /** - * {@inheritDoc} - */ - @Override - public List getNLatestFeatureWithinTimestampRange( - String entityName, - List entityIds, - Pair featureSpecLimitPair, - TimestampRange tsRange) - throws FeatureRetrievalException { - try (Scope scope = tracer.buildSpan("Redis-getNLatestFeatureWithinTimestampRange") - .startActive(true)) { - List buckets = - makePipelinedBucketKeys( - entityIds, featureSpecLimitPair.getLeft(), featureSpecLimitPair.getRight(), tsRange); - sendAndSyncPipelinedBucket(buckets); - - List getRequests = new ArrayList<>(); - Map featureCount = - new HashMap<>(entityIds.size()); // map of entity id to the number of value. - for (PipelinedBucketKeys bucket : buckets) { - for (byte[] rawKey : bucket.getBucket().get()) { - Integer count = featureCount.computeIfAbsent(bucket.getEntityId(), k -> 0); - if (count < bucket.getLimit()) { - RedisBucketKey key; - try { - key = RedisBucketKey.parseFrom(rawKey); - } catch (InvalidProtocolBufferException e) { - log.warn("Unable to parse one of key: {}", rawKey); - continue; - } - getRequests.add(new GetRequest(bucket.entityId, bucket.featureId, key)); - count++; - featureCount.put(bucket.getEntityId(), count); + if (featureSpec.getGranularity().equals(Enum.NONE)) { + getRequestsGranularityNone.add(new GetRequest(entityId, featureId, key)); } else { - break; - } - } - } - - return sendAndProcessMultiGet(getRequests); - } - } - - /** - * {@inheritDoc} - */ - @Override - public List getNLatestFeaturesWithinTimestampRange( - String entityName, - List entityIds, - List> featureSpecAndLimitPairs, - TimestampRange tsRange) { - try (Scope scope = tracer.buildSpan("Redis-getNLatestFeaturesWithinTimestampRange") - .startActive(true)) { - - List buckets = new ArrayList<>(); - for (Pair featureSpecAndLimitPair : featureSpecAndLimitPairs) { - buckets.addAll( - makePipelinedBucketKeys( - entityIds, - featureSpecAndLimitPair.getLeft(), - featureSpecAndLimitPair.getRight(), - tsRange)); - } - sendAndSyncPipelinedBucket(buckets); - - List getRequests = new ArrayList<>(); - Map> entityCountMap = new HashMap<>(entityIds.size()); - for (PipelinedBucketKeys bucket : buckets) { - for (byte[] rawKey : bucket.getBucket().get()) { - Map featureCountMap = - entityCountMap.computeIfAbsent( - bucket.getEntityId(), k -> new HashMap<>(featureSpecAndLimitPairs.size())); - Integer count = featureCountMap.computeIfAbsent(bucket.getFeatureId(), k -> 0); - - if (count < bucket.getLimit()) { - String entityId = bucket.getEntityId(); - String featureId = bucket.getFeatureId(); - RedisBucketKey key; - try { - key = RedisBucketKey.parseFrom(rawKey); - } catch (InvalidProtocolBufferException e) { - log.warn("Unable to parse one of key: {}", rawKey); - continue; - } - getRequests.add(new GetRequest(entityId, featureId, key)); - count++; - featureCountMap.put(bucket.getFeatureId(), count); - entityCountMap.put(bucket.getEntityId(), featureCountMap); - } else { - break; } } } + scope.span().log("completed request creation"); + List unfilteredResult = sendAndProcessMultiGet(getRequests); + List granularityNoneResult = sendAndProcessMultiGet(getRequestsGranularityNone); + List combinedResult = + new ArrayList<>(unfilteredResult.size() + granularityNoneResult.size()); + if (!TimeUtil.isTimeFilterRequired(tsRange)) { + combinedResult.addAll(unfilteredResult); + combinedResult.addAll(granularityNoneResult); + return combinedResult; + } - return sendAndProcessMultiGet(getRequests); + List filteredResult = filterByTimeRange(unfilteredResult, tsRange); + combinedResult.addAll(granularityNoneResult); + combinedResult.addAll(filteredResult); + return combinedResult; } } @@ -228,8 +116,7 @@ public List getNLatestFeaturesWithinTimestampRange( * @return list of feature value. */ private List sendAndProcessMultiGet(List getRequests) { - try (Scope scope = tracer.buildSpan("Redis-sendAndProcessMultiGet") - .startActive(true)) { + try (Scope scope = tracer.buildSpan("Redis-sendAndProcessMultiGet").startActive(true)) { Span span = scope.span(); if (getRequests.isEmpty()) { @@ -274,8 +161,7 @@ private List sendAndProcessMultiGet(List getRequests) */ private List processMGet(List requests, List results) throws InvalidProtocolBufferException { - try (Scope scope = tracer.buildSpan("Redis-processMGet") - .startActive(true)) { + try (Scope scope = tracer.buildSpan("Redis-processMGet").startActive(true)) { int keySize = requests.size(); List featureValues = new ArrayList<>(keySize); @@ -298,99 +184,6 @@ private List processMGet(List requests, List r } } - /** - * Send and wait for result of a pipelined zrevrangeByScore. - * - * @param buckets list of keys to be scanned. - */ - private void sendAndSyncPipelinedBucket(List buckets) { - try (Scope scope = tracer.buildSpan("Redis-sendAndSyncPipelinedBucket") - .startActive(true); - Jedis jedis = jedisPool.getResource()) { - Pipeline pipeline = jedis.pipelined(); - - // retrieve all redis keys from index bucket. - for (PipelinedBucketKeys bucket : buckets) { - bucket.setBucket( - pipeline.zrevrangeByScore( - bucket.getBucketKey().toByteArray(), - bucket.getMaxTimestamp(), - bucket.getMinTimestamp())); - } - - pipeline.sync(); - } catch (Exception e) { - log.error("Exception while retrieving bucket keys in redis", e); - throw new FeatureRetrievalException("Unable to retrieve feature from Redis", e); - } - } - - /** - * Make a list of bucket keys to scan. - * - * @param entityIds list of entity id. - * @param featureSpec feature spec of the feature to be scanned. - * @param limit maximum number of value returned from request. - * @param tsRange timestamp range of value to be returned. - * @return list of bucket keys to be scanned in pipelined command. - */ - private List makePipelinedBucketKeys( - List entityIds, FeatureSpec featureSpec, int limit, TimestampRange tsRange) { - try (Scope scope = tracer.buildSpan("Redis-makePipelinedBucketKeys") - .startActive(true)) { - List buckets = new ArrayList<>(); - String featureId = featureSpec.getId(); - String featureIdSha1Prefix = makeFeatureIdSha1Prefix(featureId); - - long maxTimestamp = - TimeUtil.roundFloorTimestamp(tsRange.getEnd(), featureSpec.getGranularity()).getSeconds(); - long minTimestamp = - TimeUtil.roundFloorTimestamp(tsRange.getStart(), featureSpec.getGranularity()) - .getSeconds(); - - for (String entityId : entityIds) { - for (RedisBucketKey bucketKey : - makeBucketKeysToScan( - entityId, featureIdSha1Prefix, minTimestamp, maxTimestamp, featureSpec)) { - buckets.add( - new PipelinedBucketKeys( - entityId, featureId, bucketKey, null, minTimestamp, maxTimestamp, limit)); - } - } - return buckets; - } - } - - /** - * Generate list of bucket keys to be scanned for range query. - * - * @param entityId entity ID of the feature - * @param featureIdSha1Prefix first 7 bytes of feature ID SHA1 prefix. - * @param roundedStart start time of range query rounded into granularity. - * @param roundedEnd end time of range query rounded into granualrity. - * @param featureSpec feature spec of the feature. - * @return list of redis bucket key to be scanned. - */ - private List makeBucketKeysToScan( - String entityId, - String featureIdSha1Prefix, - long roundedStart, - long roundedEnd, - FeatureSpec featureSpec) { - List bucketKeys = new ArrayList<>(); - long bucketSizeInSecond = SpecUtil.getBucketSize(featureSpec).getStandardSeconds(); - - for (long i = roundedStart; i <= roundedEnd; i += bucketSizeInSecond) { - bucketKeys.add( - RedisBucketKey.newBuilder() - .setEntityKey(entityId) - .setFeatureIdSha1Prefix(featureIdSha1Prefix) - .setBucketId(i / bucketSizeInSecond) - .build()); - } - return bucketKeys; - } - /** * Create {@link RedisBucketKey}. * @@ -417,64 +210,39 @@ private String makeFeatureIdSha1Prefix(String featureId) { return DigestUtils.sha1Hex(featureId.getBytes()).substring(0, 7); } - @AllArgsConstructor - @Getter - @Setter - private static class PipelinedBucketKeys { - - /** - * Entity Id of the feature - */ - private String entityId; - - /** - * Feature Id of the feature - */ - private String featureId; - - /** - * bucket key - */ - private RedisBucketKey bucketKey; - - /** - * Content of bucket - */ - private Response> bucket; - - /** - * Rounded lower bound of timestamp to be scanned - */ - private long minTimestamp; - - /** - * Rounded upper bound of timestamp to be scanned - */ - private long maxTimestamp; - - /** - * Maximum number of value to be returned - */ - private int limit; + /** + * Filter list of feature value by its timestamp + * + * @param unfilteredResult unflitered list of feature value + * @param tsRange time range filter + * @return filtered result + */ + private List filterByTimeRange( + List unfilteredResult, TimestampRange tsRange) { + return unfilteredResult + .stream() + .filter( + v -> { + Timestamp start = tsRange.getStart(); + Timestamp end = tsRange.getEnd(); + + return Timestamps.compare(start, v.getTimestamp()) <= 0 + && Timestamps.compare(v.getTimestamp(), end) <= 0; + }) + .collect(Collectors.toList()); } @AllArgsConstructor @Getter private static class GetRequest { - /** - * Entity Id of the get request - */ + /** Entity Id of the get request */ private String entityId; - /** - * Feature Id of the get request - */ + /** Feature Id of the get request */ private String featureId; - /** - * Bucket key of the request - */ + /** Bucket key of the request */ private RedisBucketKey key; } } diff --git a/serving/src/main/java/feast/serving/util/EntityMapBuilder.java b/serving/src/main/java/feast/serving/util/EntityMapBuilder.java index 6bec1574b7..5dc6a52e86 100644 --- a/serving/src/main/java/feast/serving/util/EntityMapBuilder.java +++ b/serving/src/main/java/feast/serving/util/EntityMapBuilder.java @@ -17,40 +17,28 @@ package feast.serving.util; -import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; +import feast.serving.ServingAPIProto; import feast.serving.ServingAPIProto.Entity; import feast.serving.model.FeatureValue; -import java.util.Comparator; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.SortedSet; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentSkipListSet; /** Utility class for converting response from Feature Storage into {@link Map}. */ public class EntityMapBuilder { - // for sorting based on timestamp, newest first. - private static final Comparator TIMESTAMP_COMPARATOR = - new FeatureValueTimestampComparator().reversed(); - - private final Map>> backingMap = - new ConcurrentHashMap<>(); + private final Map> backingMap = new ConcurrentHashMap<>(); public void addFeatureValueList(List features) { for (FeatureValue feature : features) { String entityId = feature.getEntityId(); String featureId = feature.getFeatureId(); - Map> featureIdToFeatureValueMap = + Map featureIdToFeatureValueMap = backingMap.computeIfAbsent(entityId, k -> new ConcurrentHashMap<>()); - SortedSet featureValues = - featureIdToFeatureValueMap.computeIfAbsent( - featureId, k -> new ConcurrentSkipListSet<>(TIMESTAMP_COMPARATOR)); - featureValues.add(feature); + featureIdToFeatureValueMap.put(featureId, feature); } } @@ -61,28 +49,21 @@ public void addFeatureValueList(List features) { */ public Map toEntityMap() { Map resultMap = new HashMap<>(); - for (Map.Entry>> entity : backingMap.entrySet()) { + for (Map.Entry> entity : backingMap.entrySet()) { String entityId = entity.getKey(); Entity.Builder entityBuilder = Entity.newBuilder(); - for (Map.Entry> feature : entity.getValue().entrySet()) { + for (Map.Entry feature : entity.getValue().entrySet()) { String featureId = feature.getKey(); - FeatureValueListBuilder featureBuilder = new FeatureValueListBuilder(); - for (FeatureValue featureValue : feature.getValue()) { - featureBuilder.addValue(featureValue.getValue(), featureValue.getTimestamp()); - } - entityBuilder.putFeatures(featureId, featureBuilder.build()); + FeatureValue featureValue = feature.getValue(); + ServingAPIProto.FeatureValue featureValueProto = ServingAPIProto.FeatureValue.newBuilder() + .setTimestamp(featureValue.getTimestamp()) + .setValue(featureValue.getValue()) + .build(); + + entityBuilder.putFeatures(featureId, featureValueProto); } resultMap.put(entityId, entityBuilder.build()); } return resultMap; } - - private static class FeatureValueTimestampComparator implements Comparator { - @Override - public int compare(FeatureValue feature1, FeatureValue feature2) { - Timestamp t1 = feature1.getTimestamp(); - Timestamp t2 = feature2.getTimestamp(); - return Timestamps.compare(t1, t2); - } - } } diff --git a/serving/src/main/java/feast/serving/util/FeatureValueListBuilder.java b/serving/src/main/java/feast/serving/util/FeatureValueListBuilder.java deleted file mode 100644 index 7db0ced77a..0000000000 --- a/serving/src/main/java/feast/serving/util/FeatureValueListBuilder.java +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.serving.util; - -import com.google.protobuf.GeneratedMessageV3; -import com.google.protobuf.Timestamp; -import feast.serving.ServingAPIProto.FeatureValueList; -import feast.types.ValueProto.*; - -/** Utilitu class for building {@link FeatureValueList}. */ -public class FeatureValueListBuilder { - - private boolean isInitialized = false; - private Value.ValCase type; - - private GeneratedMessageV3.Builder valueBuilder; - private TimestampList.Builder tsBuilder; - - public boolean addValue(Value newVal, Timestamp ts) { - initializeIfNecessary(newVal, ts); - switch (newVal.getValCase()) { - case BYTESVAL: - BytesList.Builder bytesBuilder = (BytesList.Builder) valueBuilder; - bytesBuilder.addVal(newVal.getBytesVal()); - break; - case STRINGVAL: - StringList.Builder stringBuilder = (StringList.Builder) valueBuilder; - stringBuilder.addVal(newVal.getStringVal()); - break; - case INT32VAL: - Int32List.Builder int32Builder = (Int32List.Builder) valueBuilder; - int32Builder.addVal(newVal.getInt32Val()); - break; - case INT64VAL: - Int64List.Builder int64Builder = (Int64List.Builder) valueBuilder; - int64Builder.addVal(newVal.getInt64Val()); - break; - case DOUBLEVAL: - DoubleList.Builder doubleBuilder = (DoubleList.Builder) valueBuilder; - doubleBuilder.addVal(newVal.getDoubleVal()); - break; - case FLOATVAL: - FloatList.Builder floatBuilder = (FloatList.Builder) valueBuilder; - floatBuilder.addVal(newVal.getFloatVal()); - break; - case BOOLVAL: - BoolList.Builder boolBuilder = (BoolList.Builder) valueBuilder; - boolBuilder.addVal(newVal.getBoolVal()); - break; - case TIMESTAMPVAL: - TimestampList.Builder tsListBuilder = (TimestampList.Builder) valueBuilder; - tsListBuilder.addVal(newVal.getTimestampVal()); - break; - default: - throw new IllegalArgumentException("unsupported type"); - } - if (ts != null) { - tsBuilder.addVal(ts); - } - return true; - } - - public FeatureValueList build() { - ValueList valueList = null; - switch (type) { - case BYTESVAL: - valueList = - ValueList.newBuilder().setBytesList(((BytesList.Builder) valueBuilder).build()).build(); - break; - case STRINGVAL: - valueList = - ValueList.newBuilder() - .setStringList(((StringList.Builder) valueBuilder).build()) - .build(); - break; - case INT32VAL: - valueList = - ValueList.newBuilder().setInt32List(((Int32List.Builder) valueBuilder).build()).build(); - break; - case INT64VAL: - valueList = - ValueList.newBuilder().setInt64List(((Int64List.Builder) valueBuilder).build()).build(); - break; - case DOUBLEVAL: - valueList = - ValueList.newBuilder() - .setDoubleList(((DoubleList.Builder) valueBuilder).build()) - .build(); - break; - case FLOATVAL: - valueList = - ValueList.newBuilder().setFloatList(((FloatList.Builder) valueBuilder).build()).build(); - break; - case BOOLVAL: - valueList = - ValueList.newBuilder().setBoolList(((BoolList.Builder) valueBuilder).build()).build(); - break; - case TIMESTAMPVAL: - valueList = - ValueList.newBuilder() - .setTimestampList(((TimestampList.Builder) valueBuilder).build()) - .build(); - break; - } - - FeatureValueList.Builder featureBuilder = FeatureValueList.newBuilder().setValueList(valueList); - if (tsBuilder != null) { - featureBuilder.setTimestampList(tsBuilder.build()); - } - return featureBuilder.build(); - } - - private void initializeIfNecessary(Value val, Timestamp ts) { - if (isInitialized) { - return; - } - - this.type = val.getValCase(); - this.tsBuilder = (ts != null) ? TimestampList.newBuilder() : null; - switch (type) { - case BYTESVAL: - valueBuilder = BytesList.newBuilder(); - break; - case STRINGVAL: - valueBuilder = StringList.newBuilder(); - break; - case INT32VAL: - valueBuilder = Int32List.newBuilder(); - break; - case INT64VAL: - valueBuilder = Int64List.newBuilder(); - break; - case DOUBLEVAL: - valueBuilder = DoubleList.newBuilder(); - break; - case FLOATVAL: - valueBuilder = FloatList.newBuilder(); - break; - case BOOLVAL: - valueBuilder = BoolList.newBuilder(); - break; - case TIMESTAMPVAL: - valueBuilder = TimestampList.newBuilder(); - break; - default: - throw new IllegalArgumentException("unsupported type"); - } - - isInitialized = true; - } -} diff --git a/serving/src/main/java/feast/serving/util/RequestHelper.java b/serving/src/main/java/feast/serving/util/RequestHelper.java index 7b978ed61f..ed7d7194a0 100644 --- a/serving/src/main/java/feast/serving/util/RequestHelper.java +++ b/serving/src/main/java/feast/serving/util/RequestHelper.java @@ -20,70 +20,57 @@ import com.google.common.base.Strings; import com.google.protobuf.Timestamp; import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.QueryFeatures.Request; -import feast.serving.ServingAPIProto.RequestDetail; -import feast.serving.ServingAPIProto.TimestampRange; +import feast.serving.ServingAPIProto.QueryFeaturesRequest; public class RequestHelper { - private RequestHelper() {} + private RequestHelper() {} - public static void validateRequest(Request request) { - // entity name shall present - if (Strings.isNullOrEmpty(request.getEntityName())) { - throw new IllegalArgumentException("entity name must be set"); - } - - // entity id list shall not be empty - if (request.getEntityIdList().size() <= 0) { - throw new IllegalArgumentException("entity ID must be provided"); - } + public static void validateRequest(QueryFeaturesRequest request) { + // entity name shall present + if (Strings.isNullOrEmpty(request.getEntityName())) { + throw new IllegalArgumentException("entity name must be set"); + } - // request detail shall not be empty - if (request.getRequestDetailsList().size() <= 0) { - throw new IllegalArgumentException("request details must be provided"); - } + // entity id list shall not be empty + if (request.getEntityIdList().size() <= 0) { + throw new IllegalArgumentException("entity ID must be provided"); + } - // feature id in each request detail shall have same entity name - String entityName = request.getEntityName(); - for (RequestDetail requestDetail : request.getRequestDetailsList()) { - String featureId = requestDetail.getFeatureId(); - if (!featureId.substring(0, featureId.indexOf(".")).equals(entityName)) { - throw new IllegalArgumentException( - "entity name of all feature ID in request details must be: " + entityName); - } - } + // feature IDs shall not be empty + if (request.getFeatureIdCount() <= 0) { + throw new IllegalArgumentException("feature id must be provided"); } - public static Request checkTimestampRange(Request request) { - Request.Builder requestBuilder = Request.newBuilder(request); + // feature id in each request detail shall have same entity name + String entityName = request.getEntityName(); + for (String featureId : request.getFeatureIdList()) { + if (!featureId.substring(0, featureId.indexOf(".")).equals(entityName)) { + throw new IllegalArgumentException( + "entity name of all feature ID in request details must be: " + entityName); + } + } - Timestamp defaultTs = - Timestamp.newBuilder().setSeconds(System.currentTimeMillis() / 1000).build(); + checkTimestampRange(request); + } - // default timestamp range - if (!request.hasTimestampRange()) { - requestBuilder.setTimestampRange( - TimestampRange.newBuilder().setStart(defaultTs).setEnd(defaultTs).build()); - } else if (request.getTimestampRange().getStart().getSeconds() == 0 - && request.getTimestampRange().getStart().getNanos() == 0) { - Timestamp end = request.getTimestampRange().getEnd(); - requestBuilder.setTimestampRange( - TimestampRange.newBuilder(request.getTimestampRange()).setStart(end)); - } else if (request.getTimestampRange().getEnd().getSeconds() == 0 - && request.getTimestampRange().getEnd().getNanos() == 0) { - requestBuilder.setTimestampRange( - TimestampRange.newBuilder(request.getTimestampRange()).setEnd(defaultTs)); - } + private static void checkTimestampRange(QueryFeaturesRequest request) { + if (!request.hasTimeRange()) { + return; + } - Request newRequest = requestBuilder.build(); - Timestamp start = newRequest.getTimestampRange().getStart(); - Timestamp end = newRequest.getTimestampRange().getEnd(); + Timestamp start = request.getTimeRange().getStart(); + Timestamp end = request.getTimeRange().getEnd(); + if (Timestamps.EPOCH.equals(start)) { + throw new IllegalArgumentException("start time must not be empty"); + } - if (Timestamps.compare(start, end) > 0) { - throw new IllegalArgumentException( - "'end' of timestampRange must be before or same time as 'start'"); - } + if (Timestamps.EPOCH.equals(end)) { + throw new IllegalArgumentException("end time must not be empty"); + } - return newRequest; + if (Timestamps.compare(start, end) > 0) { + throw new IllegalArgumentException( + "'end' of timestampRange must be before or same time as 'start'"); } + } } diff --git a/serving/src/main/java/feast/serving/util/StatsUtil.java b/serving/src/main/java/feast/serving/util/StatsUtil.java index 7782bcb6e0..86437218e8 100644 --- a/serving/src/main/java/feast/serving/util/StatsUtil.java +++ b/serving/src/main/java/feast/serving/util/StatsUtil.java @@ -17,8 +17,7 @@ package feast.serving.util; import com.google.common.base.Strings; -import feast.serving.ServingAPIProto.QueryFeatures.Request; -import feast.serving.ServingAPIProto.RequestDetail; +import feast.serving.ServingAPIProto.QueryFeaturesRequest; import io.grpc.Context; import io.grpc.Context.Key; import java.net.SocketAddress; @@ -40,7 +39,7 @@ private StatsUtil() { *

The tags contain information about feature's ids of the request and the client requesting * it. */ - public static String[] makeStatsdTags(Request request) { + public static String[] makeStatsdTags(QueryFeaturesRequest request) { List featureTags = makeFeatureTags(request); String remoteAddrTag = makeRemoteAddressTag(); String[] tags = featureTags.toArray(new String[featureTags.size() + 1]); @@ -48,10 +47,9 @@ public static String[] makeStatsdTags(Request request) { return tags; } - private static List makeFeatureTags(Request request) { - List tags = new ArrayList<>(request.getRequestDetailsCount()); - for (RequestDetail requestDetail : request.getRequestDetailsList()) { - String featureId = requestDetail.getFeatureId(); + private static List makeFeatureTags(QueryFeaturesRequest request) { + List tags = new ArrayList<>(request.getFeatureIdCount()); + for (String featureId : request.getFeatureIdList()) { if (Strings.isNullOrEmpty(featureId)) { continue; } diff --git a/serving/src/main/java/feast/serving/util/TimeUtil.java b/serving/src/main/java/feast/serving/util/TimeUtil.java index bb3ba17873..643c544e0f 100644 --- a/serving/src/main/java/feast/serving/util/TimeUtil.java +++ b/serving/src/main/java/feast/serving/util/TimeUtil.java @@ -17,9 +17,7 @@ package feast.serving.util; -import com.google.protobuf.Timestamp; -import feast.types.GranularityProto.Granularity.Enum; -import org.joda.time.*; +import feast.serving.ServingAPIProto.TimestampRange; /** Utility class for time-related function. */ public class TimeUtil { @@ -29,59 +27,18 @@ public class TimeUtil { private TimeUtil() {} - /** - * Round down timestamp to the nearest granularity. - * - * @param timestamp original timestamp. - * @param granularity granularity of the rounded timestamp. - * @return - */ - public static Timestamp roundFloorTimestamp(Timestamp timestamp, Enum granularity) { - MutableDateTime dt = new MutableDateTime(DateTimeZone.UTC); - DateTimeField roundingField; - switch (granularity) { - case DAY: - roundingField = dt.getChronology().dayOfMonth(); - break; - case HOUR: - roundingField = dt.getChronology().hourOfDay(); - break; - case MINUTE: - roundingField = dt.getChronology().minuteOfHour(); - break; - case SECOND: - roundingField = dt.getChronology().secondOfMinute(); - break; - case NONE: - return Timestamp.newBuilder().setSeconds(0).setNanos(0).build(); - default: - throw new RuntimeException("Unrecognised time series granularity"); - } - dt.setMillis(timestamp.getSeconds() * 1000 + timestamp.getNanos() / 1000000); - dt.setRounding(roundingField, MutableDateTime.ROUND_FLOOR); - return dateTimeToTimestamp(dt.toDateTime()); + public static boolean isTimeFilterRequired(TimestampRange timeRange) { + return timeRange != null && !TimestampRange.getDefaultInstance().equals(timeRange); } /** - * Returns the current value of the running Java Virtual Machine's - * high-resolution time source, in microseconds. + * Returns the current value of the running Java Virtual Machine's high-resolution time source, in + * microseconds. + * * @return current micro time. * @see System#nanoTime() */ public static long microTime() { return System.nanoTime() / NANO_IN_MICRO; } - - /** - * Convert {@link DateTime} into {@link Timestamp} - * - * @param dateTime - * @return - */ - private static Timestamp dateTimeToTimestamp(DateTime dateTime) { - return Timestamp.newBuilder() - .setSeconds(dateTime.getMillis() / MILLI_IN_SECOND) - .setNanos((int) (dateTime.getMillis() % MILLI_IN_SECOND) * NANO_IN_MILLI) - .build(); - } } diff --git a/serving/src/test/java/feast/serving/grpc/FeastServingTest.java b/serving/src/test/java/feast/serving/grpc/FeastServingTest.java index 2a7cac282d..ca0a10fa7e 100644 --- a/serving/src/test/java/feast/serving/grpc/FeastServingTest.java +++ b/serving/src/test/java/feast/serving/grpc/FeastServingTest.java @@ -17,24 +17,23 @@ package feast.serving.grpc; -import static junit.framework.TestCase.assertNotNull; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.collection.IsIterableContainingInAnyOrder.containsInAnyOrder; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; import static org.mockito.Mockito.verify; import com.google.protobuf.Timestamp; import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.QueryFeatures.Request; -import feast.serving.ServingAPIProto.QueryFeatures.Response; -import feast.serving.ServingAPIProto.RequestDetail; +import feast.serving.ServingAPIProto.QueryFeaturesRequest; +import feast.serving.ServingAPIProto.QueryFeaturesResponse; import feast.serving.ServingAPIProto.TimestampRange; -import feast.serving.model.RequestDetailWithSpec; import feast.serving.service.FeastServing; import feast.serving.service.FeatureRetrievalDispatcher; import feast.serving.service.FeatureStorageRegistry; import feast.serving.service.SpecStorage; import feast.serving.testutil.FakeSpecStorage; +import feast.specs.FeatureSpecProto.FeatureSpec; import io.opentracing.util.GlobalTracer; import java.util.Arrays; import java.util.Collection; @@ -67,21 +66,19 @@ public void setUp() throws Exception { @Test public void shouldReturnSameEntityNameAsRequest() { String entityName = "driver"; - RequestDetail requestDetail = - RequestDetail.newBuilder().setFeatureId("driver.day.total_completed_booking").build(); TimestampRange tsRange = TimestampRange.newBuilder() .setStart(Timestamps.EPOCH) .setEnd(Timestamp.newBuilder().setSeconds(1234).build()) .build(); - Request request = - Request.newBuilder() + QueryFeaturesRequest request = + QueryFeaturesRequest.newBuilder() .setEntityName(entityName) - .addRequestDetails(requestDetail) - .setTimestampRange(tsRange) + .addFeatureId("driver.day.total_completed_booking") + .setTimeRange(tsRange) .build(); - Response response = feast.queryFeatures(request); + QueryFeaturesResponse response = feast.queryFeatures(request); assertNotNull(response); assertThat(response.getEntityName(), equalTo(entityName)); @@ -91,32 +88,31 @@ public void shouldReturnSameEntityNameAsRequest() { public void shouldPassValidRequestToFeatureRetrievalDispatcher() { String entityName = "driver"; Collection entityIds = Arrays.asList("entity1", "entity2", "entity3"); - RequestDetail req1 = - RequestDetail.newBuilder().setFeatureId("driver.day.total_completed_booking").build(); + Collection featureIds = Arrays.asList("driver.day.total_completed_booking"); TimestampRange tsRange = TimestampRange.newBuilder() .setStart(Timestamps.EPOCH) .setEnd(Timestamp.newBuilder().setSeconds(1234).build()) .build(); - Request request = - Request.newBuilder() + QueryFeaturesRequest request = + QueryFeaturesRequest.newBuilder() .setEntityName(entityName) .addAllEntityId(entityIds) - .addRequestDetails(req1) - .setTimestampRange(tsRange) + .addAllFeatureId(featureIds) + .setTimeRange(tsRange) .build(); ArgumentCaptor entityNameArg = ArgumentCaptor.forClass(String.class); ArgumentCaptor> entityIdsArg = ArgumentCaptor.forClass(List.class); - ArgumentCaptor> requestsArg = ArgumentCaptor.forClass(List.class); + ArgumentCaptor> featureSpecArg = ArgumentCaptor.forClass(Collection.class); ArgumentCaptor tsRangeArg = ArgumentCaptor.forClass(TimestampRange.class); - Response response = feast.queryFeatures(request); + QueryFeaturesResponse response = feast.queryFeatures(request); verify(featureRetrievalDispatcher) .dispatchFeatureRetrieval( entityNameArg.capture(), entityIdsArg.capture(), - requestsArg.capture(), + featureSpecArg.capture(), tsRangeArg.capture()); assertNotNull(response); diff --git a/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java b/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java index e52848bc0b..9e4e6cea3a 100644 --- a/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java +++ b/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java @@ -17,45 +17,35 @@ package feast.serving.grpc; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.verify; + import com.google.protobuf.Timestamp; import com.google.protobuf.util.Timestamps; import com.timgroup.statsd.StatsDClient; +import feast.serving.ServingAPIProto.QueryFeaturesRequest; +import feast.serving.ServingAPIProto.QueryFeaturesResponse; +import feast.serving.ServingAPIProto.TimestampRange; +import feast.serving.service.FeastServing; import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; import io.jaegertracing.Configuration; import io.opentracing.Tracer; +import java.util.Arrays; import org.junit.Before; import org.junit.Test; -import feast.serving.ServingAPIProto.QueryFeatures.Request; -import feast.serving.ServingAPIProto.QueryFeatures.Response; -import feast.serving.ServingAPIProto.RequestDetail; -import feast.serving.ServingAPIProto.RequestType; -import feast.serving.ServingAPIProto.TimestampRange; -import feast.serving.service.FeastServing; -import org.mockito.ArgumentCaptor; import org.mockito.Mock; import org.mockito.MockitoAnnotations; -import java.util.Arrays; - -import static org.hamcrest.Matchers.closeTo; -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertThat; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - public class ServingGrpcServiceTest { @Mock private FeastServing mockFeast; - @Mock private StreamObserver mockStreamObserver; - - @Mock private Response mockResponse; + @Mock private StreamObserver mockStreamObserver; @Mock private StatsDClient statsDClient; - private Request validRequest; + private QueryFeaturesRequest validRequest; private ServingGrpcService service; @@ -63,16 +53,6 @@ public class ServingGrpcServiceTest { public void setUp() throws Exception { MockitoAnnotations.initMocks(this); - RequestDetail req1 = - RequestDetail.newBuilder().setFeatureId("driver.day.completed_booking").build(); - - RequestDetail req2 = - RequestDetail.newBuilder() - .setFeatureId("driver.none.last_opportunity") - .setType(RequestType.LIST) - .setLimit(5) - .build(); - TimestampRange tsRange = TimestampRange.newBuilder() .setStart(Timestamp.newBuilder().setSeconds(10).build()) @@ -80,11 +60,12 @@ public void setUp() throws Exception { .build(); validRequest = - Request.newBuilder() + QueryFeaturesRequest.newBuilder() .setEntityName("driver") .addAllEntityId(Arrays.asList("driver1", "driver2", "driver3")) - .addAllRequestDetails(Arrays.asList(req1, req2)) - .setTimestampRange(tsRange) + .addAllFeatureId( + Arrays.asList("driver.day.completed_booking", "driver.none.last_opportunity")) + .setTimeRange(tsRange) .build(); Tracer tracer = Configuration.fromEnv("dummy").getTracer(); @@ -98,113 +79,95 @@ public void shouldPassValidRequestAsIs() { } @Test - public void shouldUseNowAsStartAndEndInTimestampRangeIfMissing() { - Request missingTsRangeReq = Request.newBuilder(validRequest).clearTimestampRange().build(); - - ArgumentCaptor argCaptor = ArgumentCaptor.forClass(Request.class); - - when(mockFeast.queryFeatures(argCaptor.capture())).thenReturn(mockResponse); - service.queryFeatures(missingTsRangeReq, mockStreamObserver); - - Timestamp now = Timestamp.newBuilder().setSeconds(System.currentTimeMillis() / 1000).build(); - TimestampRange actualTsRange = argCaptor.getValue().getTimestampRange(); - assertThat( - (double) actualTsRange.getStart().getSeconds(), - closeTo((double) now.getSeconds(), (double) 1)); - assertThat( - (double) actualTsRange.getEnd().getSeconds(), - closeTo((double) now.getSeconds(), (double) 1)); - System.out.println(argCaptor.getValue()); + public void shouldCallOnErrorIfEntityNameIsNotSet() { + QueryFeaturesRequest missingEntityName = + QueryFeaturesRequest.newBuilder(validRequest).clearEntityName().build(); + + service.queryFeatures(missingEntityName, mockStreamObserver); + + verify(mockStreamObserver).onError(any(StatusRuntimeException.class)); } @Test - public void shouldUseEndIfStartTimestampRangeIsMissing() { - Timestamp end = Timestamp.newBuilder().setSeconds(1234).build(); - Request missingStartTsRangeReq = - Request.newBuilder(validRequest) - .clearTimestampRange() - .setTimestampRange(TimestampRange.newBuilder().setEnd(end)) - .build(); - - ArgumentCaptor argCaptor = ArgumentCaptor.forClass(Request.class); + public void shouldCallOnErrorIfEntityIdsIsNotSet() { + QueryFeaturesRequest missingEntityIds = + QueryFeaturesRequest.newBuilder(validRequest).clearEntityId().build(); - when(mockFeast.queryFeatures(argCaptor.capture())).thenReturn(mockResponse); - service.queryFeatures(missingStartTsRangeReq, mockStreamObserver); + service.queryFeatures(missingEntityIds, mockStreamObserver); - TimestampRange actualTsRange = argCaptor.getValue().getTimestampRange(); - assertThat(actualTsRange.getStart(), equalTo(end)); - System.out.println(argCaptor.getValue()); + verify(mockStreamObserver).onError(any(StatusRuntimeException.class)); } @Test - public void shouldUseNowIfEndTimestampRangeIsMissing() { - Timestamp start = Timestamp.newBuilder().setSeconds(1234).build(); - Request missingEndTsRangeReq = - Request.newBuilder(validRequest) - .clearTimestampRange() - .setTimestampRange(TimestampRange.newBuilder().setStart(start)) - .build(); - - ArgumentCaptor argCaptor = ArgumentCaptor.forClass(Request.class); + public void shouldCallOnErrorIfFeatureIdsIsNotSet() { + QueryFeaturesRequest missingRequestDetails = + QueryFeaturesRequest.newBuilder(validRequest).clearFeatureId().build(); - when(mockFeast.queryFeatures(argCaptor.capture())).thenReturn(mockResponse); - service.queryFeatures(missingEndTsRangeReq, mockStreamObserver); + service.queryFeatures(missingRequestDetails, mockStreamObserver); - Timestamp now = Timestamp.newBuilder().setSeconds(System.currentTimeMillis() / 1000).build(); - TimestampRange actualTsRange = argCaptor.getValue().getTimestampRange(); - assertThat( - (double) actualTsRange.getEnd().getSeconds(), - closeTo((double) now.getSeconds(), (double) 1)); - System.out.println(argCaptor.getValue()); + verify(mockStreamObserver).onError(any(StatusRuntimeException.class)); } @Test - public void shouldCallOnErrorIfEntityNameIsNotSet() { - Request missingEntityName = Request.newBuilder(validRequest).clearEntityName().build(); + public void shouldCallOnErrorIfFeatureIdsContainsDifferentEntity() { + QueryFeaturesRequest differentEntityReq = + QueryFeaturesRequest.newBuilder(validRequest) + .addFeatureId("customer.day.order_made") + .build(); - service.queryFeatures(missingEntityName, mockStreamObserver); + service.queryFeatures(differentEntityReq, mockStreamObserver); verify(mockStreamObserver).onError(any(StatusRuntimeException.class)); } @Test - public void shouldCallOnErrorIfEntityIdsIsNotSet() { - Request missingEntityIds = Request.newBuilder(validRequest).clearEntityId().build(); + public void shouldCallOnErrorIfTimestampRangeStartIsAfterEnd() { + TimestampRange invalidTsRange = + TimestampRange.newBuilder() + .setStart(Timestamps.fromSeconds(1001)) + .setEnd(Timestamps.fromSeconds(1000)) + .build(); - service.queryFeatures(missingEntityIds, mockStreamObserver); + QueryFeaturesRequest invalidTsRangeReq = + QueryFeaturesRequest.newBuilder(validRequest).setTimeRange(invalidTsRange).build(); + + service.queryFeatures(invalidTsRangeReq, mockStreamObserver); verify(mockStreamObserver).onError(any(StatusRuntimeException.class)); } @Test - public void shouldCallOnErrorIfRequestDetailsIsNotSet() { - Request missingRequestDetails = Request.newBuilder(validRequest).clearRequestDetails().build(); + public void shouldCallOnErrorIfTimestampRangeDoesntHaveStartAndEnd() { + TimestampRange invalidTsRange = TimestampRange.newBuilder().build(); - service.queryFeatures(missingRequestDetails, mockStreamObserver); + QueryFeaturesRequest invalidTsRangeReq = + QueryFeaturesRequest.newBuilder(validRequest).setTimeRange(invalidTsRange).build(); + + service.queryFeatures(invalidTsRangeReq, mockStreamObserver); verify(mockStreamObserver).onError(any(StatusRuntimeException.class)); } @Test - public void shouldCallOnErrorIfRequestDetailsContainsDifferentEntity() { - RequestDetail req3 = RequestDetail.newBuilder().setFeatureId("customer.day.order_made").build(); - Request differentEntityReq = Request.newBuilder(validRequest).addRequestDetails(req3).build(); + public void shouldCallOnErrorIfTimestampRangeDoesntHaveStart() { + TimestampRange invalidTsRange = + TimestampRange.newBuilder().setEnd(Timestamps.fromSeconds(1001)).build(); - service.queryFeatures(differentEntityReq, mockStreamObserver); + QueryFeaturesRequest invalidTsRangeReq = + QueryFeaturesRequest.newBuilder(validRequest).setTimeRange(invalidTsRange).build(); + + service.queryFeatures(invalidTsRangeReq, mockStreamObserver); verify(mockStreamObserver).onError(any(StatusRuntimeException.class)); } @Test - public void shouldCallOnErrorIfTimestampRangeStartIsAfterEnd() { + public void shouldCallOnErrorIfTimestampRangeDoesntHaveEnd() { TimestampRange invalidTsRange = - TimestampRange.newBuilder() - .setStart(Timestamps.fromSeconds(1001)) - .setEnd(Timestamps.fromSeconds(1000)) - .build(); + TimestampRange.newBuilder().setStart(Timestamps.fromSeconds(1001)).build(); - Request invalidTsRangeReq = - Request.newBuilder(validRequest).setTimestampRange(invalidTsRange).build(); + QueryFeaturesRequest invalidTsRangeReq = + QueryFeaturesRequest.newBuilder(validRequest).setTimeRange(invalidTsRange).build(); service.queryFeatures(invalidTsRangeReq, mockStreamObserver); diff --git a/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java b/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java index bac3110456..82f4edf94e 100644 --- a/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java +++ b/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java @@ -22,12 +22,6 @@ import com.google.protobuf.Timestamp; import com.google.protobuf.util.Durations; import com.google.protobuf.util.Timestamps; -import feast.serving.model.Pair; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.client.Connection; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.model.FeatureValue; import feast.serving.testutil.BigTablePopulator; @@ -35,12 +29,18 @@ import feast.types.GranularityProto.Granularity; import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.ValueType; - -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.Connection; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; public class BigTableFeatureStorageTestITCase { private static final String ENTITY_NAME = "test_entity"; - private static final int NUM_OF_DAYS_DATA = 2; // 2 days data. // The object under test BigTableFeatureStorage featureStorage; @@ -48,8 +48,7 @@ public class BigTableFeatureStorageTestITCase { private BigTablePopulator bigTablePopulator; private List entityIds; - private Timestamp start; - private Timestamp end; + private Timestamp now; private Connection connection; @Before @@ -62,8 +61,7 @@ public void setUp() throws Exception { featureStorage = new BigTableFeatureStorage(connection); entityIds = createEntityIds(10); - end = Timestamp.newBuilder().setSeconds(System.currentTimeMillis() / 1000).build(); - start = Timestamps.subtract(end, Durations.fromSeconds(NUM_OF_DAYS_DATA * 24 * 60 * 60)); + now = Timestamp.newBuilder().setSeconds(System.currentTimeMillis() / 1000).build(); } @After @@ -72,67 +70,7 @@ public void tearDown() throws Exception { } @Test - public void getCurrentFeature_shouldWorkForGranularityNone() { - FeatureSpec featureSpec = - FeatureSpec.newBuilder() - .setEntity(ENTITY_NAME) - .setId("test_entity.none.feature_granularity_none") - .setName("feature_granularity_none") - .setGranularity(Granularity.Enum.NONE) - .setValueType(ValueType.Enum.STRING) - .build(); - - List featureSpecs = Arrays.asList(featureSpec); - bigTablePopulator.populate( - ENTITY_NAME, entityIds, featureSpecs, Timestamps.EPOCH, Timestamps.EPOCH); - List results = - featureStorage.getCurrentFeature(ENTITY_NAME, entityIds, featureSpec); - - bigTablePopulator.validateCurrentValueGranularityNone(results, entityIds, featureSpecs); - } - - @Test - public void getCurrentFeature_shouldGracefullyHandleMissingEntity() { - FeatureSpec featureSpec = - FeatureSpec.newBuilder() - .setEntity(ENTITY_NAME) - .setId("test_entity.none.feature_granularity_none") - .setName("feature_granularity_none") - .setGranularity(Granularity.Enum.NONE) - .setValueType(ValueType.Enum.STRING) - .build(); - - List featureSpecs = Arrays.asList(featureSpec); - List entityIdsWithMissingEntity = new ArrayList<>(entityIds); - entityIdsWithMissingEntity.add("100"); - bigTablePopulator.populate( - ENTITY_NAME, entityIds, featureSpecs, Timestamps.EPOCH, Timestamps.EPOCH); - List results = - featureStorage.getCurrentFeature(ENTITY_NAME, entityIdsWithMissingEntity, featureSpec); - bigTablePopulator.validateCurrentValueGranularityNone(results, entityIds, featureSpecs); - } - - @Test - public void getCurrentFeature_shouldWorkForOtherGranularity() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec = createFeatureSpec("feature_1", granularity, ValueType.Enum.STRING); - - List featureSpecs = Collections.singletonList(spec); - bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, start, end); - - List result = - featureStorage.getCurrentFeature(ENTITY_NAME, entityIds, spec); - - bigTablePopulator.validateCurrentValueOtherGranularity(result, entityIds, featureSpecs, end); - } - } - - - @Test - public void getCurrentFeatures_shouldWorkForGranularityNone() { + public void getFeatures_shouldWorkForGranularityNone() { FeatureSpec featureSpec1 = FeatureSpec.newBuilder() .setEntity(ENTITY_NAME) @@ -152,16 +90,18 @@ public void getCurrentFeatures_shouldWorkForGranularityNone() { .build(); List featureSpecs = Arrays.asList(featureSpec1, featureSpec2); - bigTablePopulator.populate( - ENTITY_NAME, entityIds, featureSpecs, Timestamps.EPOCH, Timestamps.EPOCH); + bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, now); List results = - featureStorage.getCurrentFeatures(ENTITY_NAME, entityIds, featureSpecs); + featureStorage.getFeature(ENTITY_NAME, entityIds, featureSpecs, null); + List results2 = + featureStorage.getFeature(ENTITY_NAME, entityIds, featureSpecs, TimestampRange.getDefaultInstance()); - bigTablePopulator.validateCurrentValueGranularityNone(results, entityIds, featureSpecs); + bigTablePopulator.validate(results, entityIds, featureSpecs, null); + bigTablePopulator.validate(results2, entityIds, featureSpecs, null); } @Test - public void getCurrentFeatures_shouldGracefullyHandleMissingEntity() { + public void getFeatures_shouldGracefullyHandleMissingEntity() { FeatureSpec featureSpec1 = FeatureSpec.newBuilder() .setEntity(ENTITY_NAME) @@ -183,15 +123,14 @@ public void getCurrentFeatures_shouldGracefullyHandleMissingEntity() { List featureSpecs = Arrays.asList(featureSpec1, featureSpec2); List entityIdsWithMissingEntity = new ArrayList<>(entityIds); entityIdsWithMissingEntity.add("100"); - bigTablePopulator.populate( - ENTITY_NAME, entityIds, featureSpecs, Timestamps.EPOCH, Timestamps.EPOCH); + bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, now); List results = - featureStorage.getCurrentFeatures(ENTITY_NAME, entityIdsWithMissingEntity, featureSpecs); - bigTablePopulator.validateCurrentValueGranularityNone(results, entityIds, featureSpecs); + featureStorage.getFeature(ENTITY_NAME, entityIdsWithMissingEntity, featureSpecs, null); + bigTablePopulator.validate(results, entityIds, featureSpecs, null); } @Test - public void getCurrentFeatures_shouldWorkForOtherGranularity() { + public void getFeatures_shouldWorkForOtherGranularity() { for (Granularity.Enum granularity : Granularity.Enum.values()) { if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { continue; @@ -200,123 +139,44 @@ public void getCurrentFeatures_shouldWorkForOtherGranularity() { FeatureSpec spec2 = createFeatureSpec("feature_2", granularity, ValueType.Enum.STRING); List featureSpecs = Arrays.asList(spec1, spec2); - bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, start, end); + bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, now); List result = - featureStorage.getCurrentFeatures(ENTITY_NAME, entityIds, featureSpecs); + featureStorage.getFeature(ENTITY_NAME, entityIds, featureSpecs, null); - bigTablePopulator.validateCurrentValueOtherGranularity(result, entityIds, featureSpecs, end); + bigTablePopulator.validate(result, entityIds, featureSpecs, null); } } @Test - public void getNLatestFeatureWithinTimerange_shouldWorkForTimeseriesData() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec = createFeatureSpec("feature_1", granularity, ValueType.Enum.STRING); - - List featureSpecs = Collections.singletonList(spec); - bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, start, end); - - int limit = 5; - Pair featureSpecLimitPair = new Pair<>(spec, limit); - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - List result = - featureStorage.getNLatestFeatureWithinTimestampRange( - ENTITY_NAME, entityIds, featureSpecLimitPair, tsRange); - bigTablePopulator.validateValueWithinTimerange(result, - entityIds, Collections.singletonList(featureSpecLimitPair), tsRange); - } - } - - @Test - public void getNLatestFeatureWithinTimerange_shouldGracefullyHandleMissingEntity() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec = createFeatureSpec("feature_1", granularity, ValueType.Enum.STRING); - - List entityIdsWithMissingEntity = new ArrayList<>(entityIds); - entityIdsWithMissingEntity.add("100"); + public void getFeatures_shouldFilterOutOldData() { + String entityIdWithOldData = "entity_old_data"; + Granularity.Enum granularity = Enum.MINUTE; + Timestamp fiveMinutesAgo = Timestamps.subtract(now, Durations.fromSeconds(300)); - Collection featureSpecs = Collections.singletonList(spec); - bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, start, end); + Timestamp start = Timestamps.subtract(now, Durations.fromSeconds(60)); + Timestamp end = now; + TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - int limit = 5; - Pair featureSpecLimitPair = new Pair<>(spec, limit); - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - List result = - featureStorage.getNLatestFeatureWithinTimestampRange( - ENTITY_NAME, entityIdsWithMissingEntity, featureSpecLimitPair, tsRange); - bigTablePopulator.validateValueWithinTimerange(result, - entityIds, Collections.singletonList(featureSpecLimitPair), tsRange); - } - } + FeatureSpec spec1 = createFeatureSpec("feature_1", granularity, ValueType.Enum.STRING); + FeatureSpec spec2 = createFeatureSpec("feature_2", granularity, ValueType.Enum.STRING); + List featureSpecs = Arrays.asList(spec1, spec2); + bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, now); + bigTablePopulator.populate( + ENTITY_NAME, Collections.singletonList(entityIdWithOldData), featureSpecs, fiveMinutesAgo); - @Test - public void getNLatestFeaturesWithinTimerange_shouldWorkForTimeseriesData() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec1 = createFeatureSpec("feature_1", granularity, ValueType.Enum.STRING); - FeatureSpec spec2 = createFeatureSpec("feature_2", granularity, ValueType.Enum.STRING); - - List featureSpecs = Arrays.asList(spec1, spec2); - bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, start, end); - - int limit1 = 5; - int limit2 = 3; - Pair featureSpecLimitPair1 = new Pair<>(spec1, limit1); - Pair featureSpecLimitPair2 = new Pair<>(spec2, limit2); - - List> featureSpecLimitPairs = Arrays.asList(featureSpecLimitPair1, featureSpecLimitPair2); - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - List result = - featureStorage.getNLatestFeaturesWithinTimestampRange( - ENTITY_NAME, entityIds, featureSpecLimitPairs, tsRange); - bigTablePopulator.validateValueWithinTimerange(result, - entityIds, featureSpecLimitPairs, tsRange); - } - } - - @Test - public void getNLatestFeaturesWithinTimerange_shouldGracefullyHandleMissingEntity() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec1 = createFeatureSpec("feature_1", granularity, ValueType.Enum.STRING); - FeatureSpec spec2 = createFeatureSpec("feature_2", granularity, ValueType.Enum.STRING); - - - List featureSpecs = Arrays.asList(spec1, spec2); - bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, start, end); - - int limit1 = 5; - int limit2 = 3; - Pair featureSpecLimitPair1 = new Pair<>(spec1, limit1); - Pair featureSpecLimitPair2 = new Pair<>(spec2, limit2); - - List> featureSpecLimitPairs = Arrays.asList(featureSpecLimitPair1, featureSpecLimitPair2); - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); + List allEntityIds = new ArrayList<>(entityIds); + allEntityIds.add(entityIdWithOldData); + List result = + featureStorage.getFeature(ENTITY_NAME, allEntityIds, featureSpecs, tsRange); + System.out.println(result); - List entityIdsWithMissingEntity = new ArrayList<>(entityIds); - entityIdsWithMissingEntity.add("100"); - List result = - featureStorage.getNLatestFeaturesWithinTimestampRange( - ENTITY_NAME, entityIdsWithMissingEntity, featureSpecLimitPairs, tsRange); - bigTablePopulator.validateValueWithinTimerange(result, - entityIds, featureSpecLimitPairs, tsRange); - } + bigTablePopulator.validate(result, entityIds, featureSpecs, tsRange); } - - private FeatureSpec createFeatureSpec(String featureName, Enum granularity, ValueType.Enum valType) { + private FeatureSpec createFeatureSpec( + String featureName, Enum granularity, ValueType.Enum valType) { String entityName = ENTITY_NAME; String featureId = String.format("%s.%s.%s", entityName, granularity.toString().toLowerCase(), featureName); diff --git a/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java b/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java index 14509804ea..b02671ee6f 100644 --- a/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java +++ b/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java @@ -28,12 +28,8 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.RequestDetail; -import feast.serving.ServingAPIProto.RequestType; import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.config.AppConfig; -import feast.serving.model.Pair; -import feast.serving.model.RequestDetailWithSpec; import feast.specs.FeatureSpecProto.DataStore; import feast.specs.FeatureSpecProto.DataStores; import feast.specs.FeatureSpecProto.FeatureSpec; @@ -71,22 +67,19 @@ public void setUp() throws Exception { public void shouldUseCurrentThreadIfRequestIsSmallEnough() { String entityName = "entity"; FeatureStorage featureStorage = mock(FeatureStorage.class); - when(featureStorage.getCurrentFeatures(any(String.class), any(List.class), any(List.class))) + when(featureStorage.getFeature( + any(String.class), any(List.class), any(List.class), any(TimestampRange.class))) .thenReturn(Collections.emptyList()); when(featureStorageRegistry.get(any(String.class))).thenReturn(featureStorage); String featureId = "entity.none.feature_1"; FeatureSpec featureSpec = FeatureSpec.newBuilder().setId(featureId).build(); - RequestDetail requestDetail = - RequestDetail.newBuilder().setType(RequestType.LAST).setFeatureId(featureId).build(); - RequestDetailWithSpec requestDetailWithSpec = - new RequestDetailWithSpec(requestDetail, featureSpec); dispatcher.dispatchFeatureRetrieval( - entityName, entityIds, Collections.singletonList(requestDetailWithSpec), null); + entityName, entityIds, Collections.singletonList(featureSpec), null); verifyZeroInteractions(executorService); verify(featureStorage) - .getCurrentFeatures(entityName, entityIds, Collections.singletonList(featureSpec)); + .getFeature(entityName, entityIds, Collections.singletonList(featureSpec), null); } @Test @@ -96,44 +89,26 @@ public void shouldUseCurrentThreadIfRequestFromSameStorage() { FeatureStorage redis1 = mock(FeatureStorage.class); when(featureStorageRegistry.get(storageId1)).thenReturn(redis1); - when(redis1.getCurrentFeatures(any(String.class), any(List.class), any(List.class))) + when(redis1.getFeature( + any(String.class), any(List.class), any(List.class), any(TimestampRange.class))) .thenReturn(Collections.emptyList()); String entityName = "entity"; String featureId1 = "entity.none.feature_1"; - int limit1 = 5; FeatureSpec featureSpec1 = FeatureSpec.newBuilder() .setId(featureId1) .setDataStores( DataStores.newBuilder().setServing(DataStore.newBuilder().setId(storageId1))) .build(); - RequestDetail requestDetail1 = - RequestDetail.newBuilder() - .setType(RequestType.LIST) - .setLimit(limit1) - .setFeatureId(featureId1) - .build(); String featureId2 = "entity.none.feature_2"; - int limit2 = 1; FeatureSpec featureSpec2 = FeatureSpec.newBuilder() - .setId(featureId1) + .setId(featureId2) .setDataStores( DataStores.newBuilder().setServing(DataStore.newBuilder().setId(storageId1))) .build(); - RequestDetail requestDetail2 = - RequestDetail.newBuilder() - .setType(RequestType.LIST) - .setLimit(limit2) - .setFeatureId(featureId2) - .build(); - - RequestDetailWithSpec requestDetailWithSpec1 = - new RequestDetailWithSpec(requestDetail1, featureSpec1); - RequestDetailWithSpec requestDetailWithSpec2 = - new RequestDetailWithSpec(requestDetail2, featureSpec2); TimestampRange tsRange = TimestampRange.newBuilder() @@ -143,14 +118,14 @@ public void shouldUseCurrentThreadIfRequestFromSameStorage() { dispatcher.dispatchFeatureRetrieval( entityName, entityIds, - Arrays.asList(requestDetailWithSpec1, requestDetailWithSpec2), + Arrays.asList(featureSpec1, featureSpec2), tsRange); verify(redis1) - .getNLatestFeaturesWithinTimestampRange( + .getFeature( entityName, entityIds, - Arrays.asList(new Pair<>(featureSpec1, limit1), new Pair<>(featureSpec2, limit2)), + Arrays.asList(featureSpec1, featureSpec2), tsRange); verifyZeroInteractions(executorService); } @@ -165,60 +140,11 @@ public void shouldUseExecutorServiceIfRequestFromMoreThanOneStorage() { when(featureStorageRegistry.get(storageId1)).thenReturn(redis1); when(featureStorageRegistry.get(storageId2)).thenReturn(redis2); - when(redis1.getCurrentFeatures(any(String.class), any(List.class), any(List.class))) - .thenReturn(Collections.emptyList()); - when(redis2.getCurrentFeatures(any(String.class), any(List.class), any(List.class))) - .thenReturn(Collections.emptyList()); - - String entityName = "entity"; - String featureId1 = "entity.none.feature_1"; - FeatureSpec featureSpec1 = - FeatureSpec.newBuilder() - .setId(featureId1) - .setDataStores( - DataStores.newBuilder().setServing(DataStore.newBuilder().setId(storageId1))) - .build(); - RequestDetail requestDetail1 = - RequestDetail.newBuilder().setType(RequestType.LAST).setFeatureId(featureId1).build(); - - String featureId2 = "entity.none.feature_2"; - FeatureSpec featureSpec2 = - FeatureSpec.newBuilder() - .setId(featureId1) - .setDataStores( - DataStores.newBuilder().setServing(DataStore.newBuilder().setId(storageId2))) - .build(); - RequestDetail requestDetail2 = - RequestDetail.newBuilder().setType(RequestType.LAST).setFeatureId(featureId2).build(); - - RequestDetailWithSpec requestDetailWithSpec1 = - new RequestDetailWithSpec(requestDetail1, featureSpec1); - RequestDetailWithSpec requestDetailWithSpec2 = - new RequestDetailWithSpec(requestDetail2, featureSpec2); - - dispatcher.dispatchFeatureRetrieval( - entityName, entityIds, Arrays.asList(requestDetailWithSpec1, requestDetailWithSpec2), null); - - verify(redis1) - .getCurrentFeatures(entityName, entityIds, Collections.singletonList(featureSpec1)); - verify(redis2) - .getCurrentFeatures(entityName, entityIds, Collections.singletonList(featureSpec2)); - verify(executorService, atLeast(2)).submit(any(Callable.class)); - } - - @Test - public void shouldUseExecutorServiceIfMultipleRequestType() { - String storageId1 = "REDIS1"; - String storageId2 = "REDIS2"; - - FeatureStorage redis1 = mock(FeatureStorage.class); - FeatureStorage redis2 = mock(FeatureStorage.class); - when(featureStorageRegistry.get(storageId1)).thenReturn(redis1); - when(featureStorageRegistry.get(storageId2)).thenReturn(redis2); - - when(redis1.getCurrentFeatures(any(String.class), any(List.class), any(List.class))) + when(redis1.getFeature( + any(String.class), any(List.class), any(List.class), any(TimestampRange.class))) .thenReturn(Collections.emptyList()); - when(redis2.getCurrentFeatures(any(String.class), any(List.class), any(List.class))) + when(redis2.getFeature( + any(String.class), any(List.class), any(List.class), any(TimestampRange.class))) .thenReturn(Collections.emptyList()); String entityName = "entity"; @@ -229,45 +155,20 @@ public void shouldUseExecutorServiceIfMultipleRequestType() { .setDataStores( DataStores.newBuilder().setServing(DataStore.newBuilder().setId(storageId1))) .build(); - RequestDetail requestDetail1 = - RequestDetail.newBuilder().setType(RequestType.LAST).setFeatureId(featureId1).build(); String featureId2 = "entity.none.feature_2"; - int limit = 5; FeatureSpec featureSpec2 = FeatureSpec.newBuilder() - .setId(featureId1) + .setId(featureId2) .setDataStores( DataStores.newBuilder().setServing(DataStore.newBuilder().setId(storageId2))) .build(); - RequestDetail requestDetail2 = - RequestDetail.newBuilder() - .setType(RequestType.LIST) - .setLimit(limit) - .setFeatureId(featureId2) - .build(); - - RequestDetailWithSpec requestDetailWithSpec1 = - new RequestDetailWithSpec(requestDetail1, featureSpec1); - RequestDetailWithSpec requestDetailWithSpec2 = - new RequestDetailWithSpec(requestDetail2, featureSpec2); - TimestampRange tsRange = - TimestampRange.newBuilder() - .setStart(Timestamps.fromSeconds(0)) - .setEnd(Timestamps.fromSeconds(10)) - .build(); dispatcher.dispatchFeatureRetrieval( - entityName, - entityIds, - Arrays.asList(requestDetailWithSpec1, requestDetailWithSpec2), - tsRange); + entityName, entityIds, Arrays.asList(featureSpec1, featureSpec2), null); - verify(redis1) - .getCurrentFeatures(entityName, entityIds, Collections.singletonList(featureSpec1)); - verify(redis2) - .getNLatestFeaturesWithinTimestampRange( - entityName, entityIds, Collections.singletonList(new Pair<>(featureSpec2, limit)), tsRange); + verify(redis1).getFeature(entityName, entityIds, Collections.singletonList(featureSpec1), null); + verify(redis2).getFeature(entityName, entityIds, Collections.singletonList(featureSpec2), null); verify(executorService, atLeast(2)).submit(any(Callable.class)); } diff --git a/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java b/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java index c6cd539069..7c6dd43e48 100644 --- a/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java +++ b/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java @@ -18,15 +18,12 @@ package feast.serving.service; import static junit.framework.TestCase.fail; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertThat; import com.google.protobuf.Timestamp; import com.google.protobuf.util.Durations; import com.google.protobuf.util.Timestamps; import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.model.FeatureValue; -import feast.serving.model.Pair; import feast.serving.testutil.RedisPopulator; import feast.specs.FeatureSpecProto.DataStore; import feast.specs.FeatureSpecProto.DataStores; @@ -39,7 +36,6 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; -import org.joda.time.Duration; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -50,7 +46,6 @@ public class RedisFeatureStorageTest { public static final String REDIS_HOST = "localhost"; public static final int REDIS_PORT = 6377; - private static final int NUM_OF_DAYS_DATA = 2; // embedded redis RedisServer redisServer; @@ -61,8 +56,7 @@ public class RedisFeatureStorageTest { private List entityIds; private String entityName; - private Timestamp end; - private Timestamp start; + private Timestamp now; @Before public void setUp() throws Exception { @@ -81,93 +75,11 @@ public void setUp() throws Exception { entityIds = createEntityIds(10); entityName = "entity"; - end = Timestamp.newBuilder().setSeconds(System.currentTimeMillis() / 1000).build(); - start = Timestamps.subtract(end, Durations.fromSeconds(NUM_OF_DAYS_DATA * 24 * 60 * 60)); + now = Timestamp.newBuilder().setSeconds(System.currentTimeMillis() / 1000).build(); } @Test - public void getCurrentFeature_shouldWorkForGranularityNone() { - FeatureSpec featureSpec = - FeatureSpec.newBuilder() - .setId("entity.none.feature_1") - .setEntity(entityName) - .setGranularity(Enum.NONE) - .setValueType(ValueType.Enum.STRING) - .build(); - - List featureSpecs = Collections.singletonList(featureSpec); - redisPopulator.populate(entityName, entityIds, featureSpecs, null, null); - - List result = redisFs.getCurrentFeature(entityName, entityIds, featureSpec); - redisPopulator.validateCurrentValueGranularityNone(result, entityIds, featureSpecs); - } - - @Test - public void getCurrentFeature_shouldNotReturnMissingValue() { - FeatureSpec featureSpec = - FeatureSpec.newBuilder() - .setId("entity.none.feature_1") - .setEntity(entityName) - .setGranularity(Enum.NONE) - .setValueType(ValueType.Enum.STRING) - .build(); - - List featureSpecs = Collections.singletonList(featureSpec); - redisPopulator.populate(entityName, entityIds, featureSpecs, null, null); - - // add entity without feature - List requestEntityIds = new ArrayList<>(entityIds); - requestEntityIds.add("100"); - List result = - redisFs.getCurrentFeature(entityName, requestEntityIds, featureSpec); - redisPopulator.validateCurrentValueGranularityNone(result, entityIds, featureSpecs); - } - - @Test - public void getCurrentFeature_shouldReturnLastValueForOtherGranularity() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec = - createFeatureSpec("feature_1",granularity); - - List featureSpecs = Collections.singletonList(spec); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - - List result = redisFs.getCurrentFeature(entityName, entityIds, spec); - redisPopulator.validateCurrentValueOtherGranularity(result, entityIds, featureSpecs, end); - } - } - - - @Test - public void getCurrentFeatures_shouldWorkForGranularityNone() { - FeatureSpec featureSpec1 = - FeatureSpec.newBuilder() - .setId("entity.none.feature_1") - .setEntity(entityName) - .setGranularity(Enum.NONE) - .setValueType(ValueType.Enum.STRING) - .build(); - - FeatureSpec featureSpec2 = - FeatureSpec.newBuilder() - .setId("entity.none.feature_2") - .setEntity(entityName) - .setGranularity(Enum.NONE) - .setValueType(ValueType.Enum.STRING) - .build(); - - List featureSpecs = Arrays.asList(featureSpec1, featureSpec2); - redisPopulator.populate(entityName, entityIds, featureSpecs, null, null); - - List result = redisFs.getCurrentFeatures(entityName, entityIds, featureSpecs); - redisPopulator.validateCurrentValueGranularityNone(result, entityIds, featureSpecs); - } - - @Test - public void getCurrentFeatures_shouldNotReturnMissingValue() { + public void getFeatures_shouldNotReturnMissingValue() { FeatureSpec featureSpec1 = FeatureSpec.newBuilder() .setId("entity.none.feature_1") @@ -185,267 +97,68 @@ public void getCurrentFeatures_shouldNotReturnMissingValue() { .build(); List featureSpecs = Arrays.asList(featureSpec1, featureSpec2); - redisPopulator.populate(entityName, entityIds, featureSpecs, null, null); + redisPopulator.populate(entityName, entityIds, featureSpecs, now); // add entity without feature List requestEntityIds = new ArrayList<>(entityIds); requestEntityIds.add("100"); List result = - redisFs.getCurrentFeatures(entityName, requestEntityIds, featureSpecs); - redisPopulator.validateCurrentValueGranularityNone(result, entityIds, featureSpecs); - } - - @Test - public void getCurrentFeatures_shouldReturnLastValueForOtherGranularity() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec1 = - createFeatureSpec("feature_1", granularity); - FeatureSpec spec2 = - createFeatureSpec("feature_2", granularity); - List featureSpecs = Arrays.asList(spec1, spec2); - - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - - List result = redisFs.getCurrentFeatures(entityName, entityIds, featureSpecs); - redisPopulator.validateCurrentValueOtherGranularity(result, entityIds, featureSpecs, end); - } + redisFs.getFeature(entityName, requestEntityIds, featureSpecs, null); + redisPopulator.validate(result, entityIds, featureSpecs, null); } @Test - public void getNLatestFeatureWithinTimerange_shouldWorkForTimeseriesData() { + public void getFeatures_shouldReturnLastValue() { for (Granularity.Enum granularity : Granularity.Enum.values()) { if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { continue; } - FeatureSpec spec = - createFeatureSpec("feature_1", granularity); - - List featureSpecs = Collections.singletonList(spec); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - int limit = 5; - Pair featureSpecLimitPair = new Pair<>(spec, limit); - List result = - redisFs.getNLatestFeatureWithinTimestampRange( - entityName, entityIds, featureSpecLimitPair, tsRange); - redisPopulator.validateValueWithinTimerange( - result, entityIds, Collections.singletonList(featureSpecLimitPair), tsRange); - } - } - - @Test - public void getNLatestFeatureWithinTimerange_shouldNotReturnMissingEntity() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec = - createFeatureSpec("feature_1", granularity); - - List featureSpecs = Collections.singletonList(spec); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - int limit = 5; - Pair featureSpecLimitPair = new Pair<>(spec, limit); - - List entityIdsWithMissing = new ArrayList<>(entityIds); - entityIdsWithMissing.add("100"); // missing entity - List result = - redisFs.getNLatestFeatureWithinTimestampRange( - entityName, entityIdsWithMissing, featureSpecLimitPair, tsRange); - redisPopulator.validateValueWithinTimerange( - result, entityIds, Collections.singletonList(featureSpecLimitPair), tsRange); - } - } - - @Test - public void getNLatestFeatureWithinTimerange_shouldNotReturnMissingValueInCertainTimestamp() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec = - createFeatureSpec("feature_1", granularity); - - List featureSpecs = Collections.singletonList(spec); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - - Timestamp earlierStart = Timestamps.subtract(start, Durations.fromSeconds(86400)); - TimestampRange tsRange = - TimestampRange.newBuilder().setStart(earlierStart).setEnd(end).build(); - int limit = 5; - Pair featureSpecLimitPair = new Pair<>(spec, limit); - - List result = - redisFs.getNLatestFeatureWithinTimestampRange( - entityName, entityIds, featureSpecLimitPair, tsRange); - redisPopulator.validateValueWithinTimerange( - result, entityIds, Collections.singletonList(featureSpecLimitPair), tsRange); - } - } - - @Test - public void getNLatestFeatureWithinTimerange_shouldReturnEmptyListIfNoFeatureIsRetrieved() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec = - createFeatureSpec("feature_1", granularity); - - List featureSpecs = Collections.singletonList(spec); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - - Timestamp earlierStart = Timestamps.subtract(start, Durations.fromSeconds(86400)); - TimestampRange tsRange = - TimestampRange.newBuilder().setStart(earlierStart).setEnd(end).build(); - int limit = 5; - Pair featureSpecLimitPair = new Pair<>(spec, limit); - - String missingEntity = "missingEntity"; - List result = - redisFs.getNLatestFeatureWithinTimestampRange( - entityName, Arrays.asList(missingEntity), featureSpecLimitPair, tsRange); - assertThat(result.size(), equalTo(0)); - } - } - - - @Test - public void getNLatestFeaturesWithinTimerange_shouldWorkForTimeseriesData() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec1 = - createFeatureSpec("feature_1", granularity); - FeatureSpec spec2 = - createFeatureSpec("feature_2", granularity); + FeatureSpec spec1 = createFeatureSpec("feature_1", Enum.NONE); + FeatureSpec spec2 = createFeatureSpec("feature_2", granularity); List featureSpecs = Arrays.asList(spec1, spec2); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - int limit1 = 2; - int limit2 = 3; - Pair featureSpecLimitPair1 = new Pair<>(spec1, limit1); - Pair featureSpecLimitPair2 = new Pair<>(spec2, limit2); - List> featureSpecLimitPairs = Arrays.asList(featureSpecLimitPair1, featureSpecLimitPair2); - List result = - redisFs.getNLatestFeaturesWithinTimestampRange( - entityName, entityIds, featureSpecLimitPairs, tsRange); - redisPopulator.validateValueWithinTimerange( - result, entityIds, featureSpecLimitPairs, tsRange); - } - } - - @Test - public void getNLatestFeaturesWithinTimerange_shouldNotReturnMissingEntity() { - - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec1 = - createFeatureSpec("feature_1", granularity); - FeatureSpec spec2 = - createFeatureSpec("feature_2", granularity); - List featureSpecs = Arrays.asList(spec1, spec2); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); + redisPopulator.populate(entityName, entityIds, featureSpecs, now); - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - int limit1 = 5; - int limit2 = 1; - Pair featureSpecLimitPair1 = new Pair<>(spec1, limit1); - Pair featureSpecLimitPair2 = new Pair<>(spec2, limit2); - List> featureSpecLimitPairs = Arrays.asList(featureSpecLimitPair1, featureSpecLimitPair2); + List result = redisFs.getFeature(entityName, entityIds, featureSpecs, null); + List result2 = redisFs.getFeature(entityName, entityIds, featureSpecs, TimestampRange.getDefaultInstance()); - List entityIdsWithMissing = new ArrayList<>(entityIds); - entityIdsWithMissing.add("100"); // missing entity - List result = - redisFs.getNLatestFeaturesWithinTimestampRange( - entityName, entityIdsWithMissing, featureSpecLimitPairs, tsRange); - redisPopulator.validateValueWithinTimerange( - result, entityIds, featureSpecLimitPairs, tsRange); + redisPopulator.validate(result, entityIds, featureSpecs, null); + redisPopulator.validate(result2, entityIds, featureSpecs, null); } } @Test - public void getNLatestFeaturesWithinTimerange_shouldNotReturnMissingValueInCertainTimestamp() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec1 = - createFeatureSpec("feature_1", granularity); - FeatureSpec spec2 = - createFeatureSpec("feature_2", granularity); - List featureSpecs = Arrays.asList(spec1, spec2); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); + public void getFeatures_shouldReturnLastValueFilteredByTimestampIfSpecified() { + String entityIdWithOldData = "entity_old"; + Timestamp fiveMinutesAgo = Timestamps.subtract(now, Durations.fromSeconds(300)); - Timestamp earlierStart = Timestamps.subtract(start, Durations.fromSeconds(86400)); - TimestampRange tsRange = - TimestampRange.newBuilder().setStart(earlierStart).setEnd(end).build(); - int limit1 = 5; - int limit2 = 10; - Pair featureSpecLimitPair1 = new Pair<>(spec1, limit1); - Pair featureSpecLimitPair2 = new Pair<>(spec2, limit2); - List> featureSpecLimitPairs = Arrays.asList(featureSpecLimitPair1, featureSpecLimitPair2); + Timestamp start = Timestamps.subtract(now, Durations.fromSeconds(60)); + Timestamp end = now; + TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - List result = - redisFs.getNLatestFeaturesWithinTimestampRange( - entityName, entityIds, featureSpecLimitPairs, tsRange); - redisPopulator.validateValueWithinTimerange( - result, entityIds, featureSpecLimitPairs, tsRange); - } - } + Granularity.Enum granularity = Enum.SECOND; + FeatureSpec spec1 = createFeatureSpec("feature_1", granularity); + FeatureSpec spec2 = createFeatureSpec("feature_2", granularity); + List featureSpecs = Arrays.asList(spec1, spec2); - @Test - public void getNLatestFeaturesWithinTimerange_shouldReturnEmptyListIfNoFeatureIsRetrieved() { - for (Granularity.Enum granularity : Granularity.Enum.values()) { - if (granularity.equals(Enum.NONE) || granularity.equals(Enum.UNRECOGNIZED)) { - continue; - } - FeatureSpec spec1 = - createFeatureSpec("feature_1", granularity); - FeatureSpec spec2 = - createFeatureSpec("feature_2", granularity); - List featureSpecs = Arrays.asList(spec1, spec2); - redisPopulator.populate(entityName, entityIds, featureSpecs, start, end); + redisPopulator.populate(entityName, entityIds, featureSpecs, now); + redisPopulator.populate( + entityName, Collections.singletonList(entityIdWithOldData), featureSpecs, fiveMinutesAgo); - Timestamp earlierStart = Timestamps.subtract(start, Durations.fromSeconds(86400)); - TimestampRange tsRange = - TimestampRange.newBuilder().setStart(earlierStart).setEnd(end).build(); - int limit1 = 5; - int limit2 = 10; - Pair featureSpecLimitPair1 = new Pair<>(spec1, limit1); - Pair featureSpecLimitPair2 = new Pair<>(spec2, limit2); - List> featureSpecLimitPairs = Arrays.asList(featureSpecLimitPair1, featureSpecLimitPair2); + List allEntityIds = new ArrayList<>(entityIds); + allEntityIds.add(entityIdWithOldData); - String missingEntity = "missingEntity"; - List result = - redisFs.getNLatestFeaturesWithinTimestampRange( - entityName, Arrays.asList(missingEntity), featureSpecLimitPairs, tsRange); - assertThat(result.size(), equalTo(0)); - } + List result = redisFs.getFeature(entityName, allEntityIds, featureSpecs, tsRange); + redisPopulator.validate(result, entityIds, featureSpecs, tsRange); } private FeatureSpec createFeatureSpec(String featureName, Enum granularity) { - DataStore servingDatastoreSpec = - DataStore.newBuilder() - .putOptions( - RedisFeatureStorage.OPT_REDIS_BUCKET_SIZE, - Duration.standardSeconds(86400).toString()) - .build(); + DataStore servingDatastoreSpec = DataStore.newBuilder().setId("REDIS").build(); return createFeatureSpec(featureName, granularity, ValueType.Enum.STRING, servingDatastoreSpec); } - private FeatureSpec createFeatureSpec(String featureName, - Enum granularity, ValueType.Enum valType, DataStore dataStoreSpec) { + private FeatureSpec createFeatureSpec( + String featureName, Enum granularity, ValueType.Enum valType, DataStore dataStoreSpec) { String entityName = "entity"; String featureId = String.format("%s.%s.%s", entityName, granularity.toString().toLowerCase(), featureName); diff --git a/serving/src/test/java/feast/serving/testutil/BigTablePopulator.java b/serving/src/test/java/feast/serving/testutil/BigTablePopulator.java index c2317f040b..9f7b2292b3 100644 --- a/serving/src/test/java/feast/serving/testutil/BigTablePopulator.java +++ b/serving/src/test/java/feast/serving/testutil/BigTablePopulator.java @@ -17,58 +17,56 @@ package feast.serving.testutil; -import com.google.protobuf.Duration; +import static junit.framework.TestCase.fail; + import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Durations; import com.google.protobuf.util.Timestamps; -import org.apache.commons.codec.digest.DigestUtils; -import org.apache.hadoop.hbase.HTableDescriptor; -import org.apache.hadoop.hbase.TableExistsException; -import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.client.*; -import feast.serving.util.TimeUtil; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.storage.BigTableProto.BigTableRowKey; -import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.Value; -import feast.types.ValueProto.ValueType; - import java.io.IOException; -import java.util.ArrayList; import java.util.Collection; -import java.util.List; - -import static junit.framework.TestCase.fail; - -/** Helper class to populate a BigTable instance with fake data. */ +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.hadoop.hbase.HTableDescriptor; +import org.apache.hadoop.hbase.TableExistsException; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.client.TableDescriptor; + +/** + * Helper class to populate a BigTable instance with fake data. It mimic ingesting data to BigTable. + */ public class BigTablePopulator extends FeatureStoragePopulator { - private final Connection connection; private static final byte[] DEFAULT_COLUMN_FAMILY = "default".getBytes(); + private static final String LATEST_KEY = "0"; + private final Connection connection; public BigTablePopulator(Connection connection) { this.connection = connection; } /** - * Populate big table instance with fake data ranging between 'start' and 'end'. The data will be - * dense, meaning for each entity the data will be available for each granularity units. (except - * for granularity second, for which the data is available every 30 seconds) + * Populate big table instance with fake data. * - * @param entityIds - * @param featureSpecs - * @param start - * @param end + * @param entityName entity name of the feature + * @param entityIds collection of entity ID for which the feature should be populated + * @param featureSpecs collection of feature specs for which the feature should be populated + * @param timestamp timestamp of the features */ @Override public void populate( String entityName, Collection entityIds, Collection featureSpecs, - Timestamp start, - Timestamp end) { + Timestamp timestamp) { createTableIfNecessary(entityName); - populateTableWithFakeData(entityName, entityIds, featureSpecs, start, end); + populateTableWithFakeData(entityName, entityIds, featureSpecs, timestamp); } private void createTableIfNecessary(String entityName) { @@ -97,17 +95,15 @@ private void populateTableWithFakeData( String entityName, Collection entityIds, Collection featureSpecs, - Timestamp start, - Timestamp end) { + Timestamp timestamp) { TableName tableName = TableName.valueOf(entityName); try (Table table = connection.getTable(tableName)) { for (FeatureSpec featureSpec : featureSpecs) { for (String entityId : entityIds) { - Timestamp roundedStart = - TimeUtil.roundFloorTimestamp(start, featureSpec.getGranularity()); - Timestamp roundedEnd = TimeUtil.roundFloorTimestamp(end, featureSpec.getGranularity()); - List puts = makeMultiplePut(entityId, featureSpec, roundedStart, roundedEnd); - table.put(puts); + Timestamp roundedTimestamp = + TimeUtil.roundFloorTimestamp(timestamp, featureSpec.getGranularity()); + Put put = makePut(entityId, featureSpec, roundedTimestamp); + table.put(put); } } @@ -118,45 +114,11 @@ private void populateTableWithFakeData( } } - private List makeMultiplePut( - String entityId, FeatureSpec fs, Timestamp start, Timestamp end) { - String featureId = fs.getId(); - List puts = new ArrayList<>(); - if (fs.getGranularity().equals(Enum.NONE)) { - Value val = createValue(entityId, featureId, fs.getValueType()); - puts.add(makeSinglePutForGranularityNone(entityId, fs.getId(), val)); - return puts; - } - - Duration timesteps = getTimestep(fs.getGranularity()); - for (Timestamp iter = end; - Timestamps.compare(iter, start) > 0; - iter = Timestamps.subtract(iter, timesteps)) { - Timestamp roundedIter = TimeUtil.roundFloorTimestamp(iter, fs.getGranularity()); - if (Timestamps.compare(iter, end) == 0) { - Value value = createValue(entityId, featureId, iter, fs.getValueType()); - BigTableRowKey rowKey = - BigTableRowKey.newBuilder() - .setEntityKey(entityId) - .setReversedMillis("0") - .setSha1Prefix(DigestUtils.sha1Hex(entityId.getBytes()).substring(0, 7)) - .build(); - Put put = new Put(rowKey.toByteArray()); - long ts = Timestamps.toMillis(roundedIter); - put.addColumn(DEFAULT_COLUMN_FAMILY, fs.getId().getBytes(), ts, value.toByteArray()); - puts.add(put); - } - puts.add(makeSinglePut(entityId, fs, roundedIter)); - } - - return puts; - } - - private Put makeSinglePut(String entityId, FeatureSpec fs, Timestamp roundedTimestamp) { + private Put makePut(String entityId, FeatureSpec fs, Timestamp roundedTimestamp) { BigTableRowKey rowKey = BigTableRowKey.newBuilder() .setEntityKey(entityId) - .setReversedMillis(roundedReversedMillis(roundedTimestamp, fs.getGranularity())) + .setReversedMillis(LATEST_KEY) .setSha1Prefix(DigestUtils.sha1Hex(entityId.getBytes()).substring(0, 7)) .build(); Put put = new Put(rowKey.toByteArray()); @@ -165,41 +127,4 @@ private Put makeSinglePut(String entityId, FeatureSpec fs, Timestamp roundedTime put.addColumn(DEFAULT_COLUMN_FAMILY, fs.getId().getBytes(), timestamp, val.toByteArray()); return put; } - - private Put makeSinglePutForGranularityNone(String entityId, String featureId, Value value) { - BigTableRowKey rowKey = - BigTableRowKey.newBuilder() - .setEntityKey(entityId) - .setReversedMillis("0") - .setSha1Prefix(DigestUtils.sha1Hex(entityId.getBytes()).substring(0, 7)) - .build(); - Put put = new Put(rowKey.toByteArray()); - put.addColumn(DEFAULT_COLUMN_FAMILY, featureId.getBytes(), 0, value.toByteArray()); - return put; - } - - String roundedReversedMillis(Timestamp ts, Enum granularity) { - if (granularity.equals(Enum.NONE)) { - return "0"; - } - Timestamp roundedEnd = TimeUtil.roundFloorTimestamp(ts, granularity); - return String.valueOf(Long.MAX_VALUE - roundedEnd.getSeconds() * 1000); - } - - // use at least hourly time step since big table is slow - @Override - protected Duration getTimestep(Enum granularity) { - switch (granularity) { - case SECOND: - return Durations.fromSeconds(60 * 60); - case MINUTE: - return Durations.fromSeconds(60 * 60); - case HOUR: - return Durations.fromSeconds(60 * 60); - case DAY: - return Durations.fromSeconds(24 * 60 * 60); - default: - throw new IllegalArgumentException("unsupported granularity: " + granularity); - } - } } diff --git a/serving/src/test/java/feast/serving/testutil/FeatureStoragePopulator.java b/serving/src/test/java/feast/serving/testutil/FeatureStoragePopulator.java index 8cbea94886..4cd851b9e4 100644 --- a/serving/src/test/java/feast/serving/testutil/FeatureStoragePopulator.java +++ b/serving/src/test/java/feast/serving/testutil/FeatureStoragePopulator.java @@ -20,18 +20,13 @@ import static java.util.stream.Collectors.groupingBy; import static org.apache.hadoop.hbase.shaded.org.junit.Assert.assertNotNull; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.junit.Assert.assertThat; -import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Durations; import com.google.protobuf.util.Timestamps; import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.model.FeatureValue; -import feast.serving.model.Pair; -import feast.serving.util.TimeUtil; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.Value; @@ -49,15 +44,13 @@ public abstract class FeatureStoragePopulator { * @param entityName entity name. * @param entityIds collection of entity ID to be added. * @param featureSpecs collection of feature's spec for which the data should be added. - * @param start oldest timestamp of data that should be added. (exclusive) - * @param end newest timestamp of data that should be added. (inclusive) + * @param timestamp event timestamp of data */ public abstract void populate( String entityName, Collection entityIds, Collection featureSpecs, - Timestamp start, - Timestamp end); + Timestamp timestamp); /** * Create a historical feature value based on parameters. @@ -81,73 +74,11 @@ protected Value createValue( } } - /** - * Create single feature value based on parameters. - * - * @param entityId - * @param featureId - * @param valType - * @return - */ - protected Value createValue(String entityId, String featureId, ValueType.Enum valType) { - switch (valType) { - case INT64: - return Value.newBuilder().setInt64Val(Long.parseLong(entityId)).build(); - case STRING: - String value = String.format("%s_%s", entityId, featureId); - return Value.newBuilder().setStringVal(value).build(); - default: - throw new IllegalArgumentException("not yet supported"); - } - } - - /** - * Get the duration between 2 fake data based on granularity of a feature. - * - * @param granularity feature's granualarity. - * @return duration between two fake time-series data. - */ - protected Duration getTimestep(Enum granularity) { - switch (granularity) { - case SECOND: - return Durations.fromSeconds(30); - case MINUTE: - return Durations.fromSeconds(60); - case HOUR: - return Durations.fromSeconds(60 * 60); - case DAY: - return Durations.fromSeconds(24 * 60 * 60); - default: - throw new IllegalArgumentException("unsupported granularity: " + granularity); - } - } - - public void validateCurrentValueGranularityNone( - List result, List entityIds, List featureSpecs) { - Map>> entityMap = toEntityMap(result); - - assertNotNull(entityMap); - assertThat(entityMap.size(), equalTo(entityIds.size())); - for (String entityId : entityIds) { - Map> featureMap = entityMap.get(entityId); - assertNotNull(featureMap); - assertThat(featureMap.size(), equalTo(featureSpecs.size())); - for (FeatureSpec featureSpec : featureSpecs) { - List featureValueList = featureMap.get(featureSpec.getId()); - assertNotNull(featureValueList); - assertThat(featureValueList.size(), equalTo(1)); - - FeatureValue featureValue = featureValueList.get(0); - validateValue(featureValue, entityId, featureSpec); - } - } - } - - public void validateCurrentValueOtherGranularity( + public void validate( List result, List entityIds, List featureSpecs, - Timestamp lastTimestamp) { + TimestampRange tsRange) { Map>> entityMap = toEntityMap(result); assertNotNull(entityMap); @@ -165,48 +96,18 @@ public void validateCurrentValueOtherGranularity( FeatureValue featureValue = featureValueList.get(0); Timestamp timestamp = featureValue.getTimestamp(); validateValue(featureValue, entityId, featureSpec, timestamp); - lastTimestamp = TimeUtil.roundFloorTimestamp(lastTimestamp, featureSpec.getGranularity()); - assertThat(timestamp, equalTo(lastTimestamp)); - } - } - } - public void validateValueWithinTimerange( - List result, - List entityIds, - List> featureSpecLimitPairs, - TimestampRange tsRange) { - Map>> entityMap = toEntityMap(result); - - assertNotNull(entityMap); - assertThat(entityMap.size(), equalTo(entityIds.size())); - for (String entityId : entityIds) { - Map> featureMap = entityMap.get(entityId); - assertThat(featureMap.size(), equalTo(featureSpecLimitPairs.size())); - for (Pair featureSpecLimitPair : featureSpecLimitPairs) { - FeatureSpec featureSpec = featureSpecLimitPair.getLeft(); - int limit = featureSpecLimitPair.getRight(); - List featureValues = featureMap.get(featureSpec.getId()); - assertThat(featureValues.size(), lessThanOrEqualTo(limit)); + if (tsRange != null && !featureSpec.getGranularity().equals(Enum.NONE)) { + Timestamp start = tsRange.getStart(); + Timestamp end = tsRange.getEnd(); - for (FeatureValue featureValue : featureValues) { - Timestamp timestamp = featureValue.getTimestamp(); - validateValue(featureValue, entityId, featureSpec, timestamp); - - // check timestamp is within specified range - assertThat(Timestamps.compare(timestamp, tsRange.getEnd()), lessThanOrEqualTo(0)); - assertThat(Timestamps.compare(timestamp, tsRange.getStart()), greaterThanOrEqualTo(0)); + assertThat(Timestamps.compare(start, timestamp), lessThanOrEqualTo(0)); + assertThat(Timestamps.compare(timestamp, end), lessThanOrEqualTo(0)); } } } } - private void validateValue(FeatureValue featureValue, String entityId, FeatureSpec featureSpec) { - Value actualValue = featureValue.getValue(); - Value expectedValue = createValue(entityId, featureSpec.getId(), featureSpec.getValueType()); - assertThat(actualValue, equalTo(expectedValue)); - } - private void validateValue( FeatureValue featureValue, String entityId, FeatureSpec featureSpec, Timestamp timestamp) { Value actualValue = featureValue.getValue(); diff --git a/serving/src/test/java/feast/serving/testutil/RedisPopulator.java b/serving/src/test/java/feast/serving/testutil/RedisPopulator.java index b8260cb59d..68340405e6 100644 --- a/serving/src/test/java/feast/serving/testutil/RedisPopulator.java +++ b/serving/src/test/java/feast/serving/testutil/RedisPopulator.java @@ -17,20 +17,13 @@ package feast.serving.testutil; -import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; -import feast.serving.util.SpecUtil; -import feast.serving.util.TimeUtil; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.storage.RedisProto.RedisBucketKey; import feast.storage.RedisProto.RedisBucketValue; -import feast.types.GranularityProto.Granularity.Enum; +import java.util.Collection; import org.apache.commons.codec.digest.DigestUtils; import redis.clients.jedis.Jedis; -import redis.clients.jedis.Pipeline; - -import java.util.Collection; public class RedisPopulator extends FeatureStoragePopulator { private final Jedis jedis; @@ -44,76 +37,29 @@ public void populate( String entityName, Collection entityIds, Collection featureSpecs, - Timestamp start, - Timestamp end) { + Timestamp timestamp) { for (FeatureSpec fs : featureSpecs) { for (String entityId : entityIds) { - if (fs.getGranularity().equals(Enum.NONE)) { - addGranularityNoneData(entityId, fs); - } else { - start = TimeUtil.roundFloorTimestamp(start, fs.getGranularity()); - end = TimeUtil.roundFloorTimestamp(end, fs.getGranularity()); - addOtherGranularityData(entityId, fs, start, end); - } - } - } - } - - /** - * Add time series data for entityId within specified time range. - * - * @param entityId entity id of the data to be added. - * @param fs feature spec of the feature to be added. - * @param start start time of the available data (exclusive). - * @param end end time of the available data (inclusive). - */ - private void addOtherGranularityData( - String entityId, FeatureSpec fs, Timestamp start, Timestamp end) { - Duration dur = getTimestep(fs.getGranularity()); - long bucketSizeInSecond = SpecUtil.getBucketSize(fs).getStandardSeconds(); - String featureId = fs.getId(); - String featureIdSha1Prefix = getFeatureIdSha1Prefix(featureId); - Pipeline pipeline = jedis.pipelined(); - - for (Timestamp iter = end; - Timestamps.compare(iter, start) > 0; - iter = Timestamps.subtract(iter, dur)) { - long bucketId = - TimeUtil.roundFloorTimestamp(iter, fs.getGranularity()).getSeconds() / bucketSizeInSecond; - RedisBucketKey bucketKeyProto = createBucketKey(entityId, featureIdSha1Prefix, bucketId); - RedisBucketKey keyProto = createBucketKey(entityId, featureIdSha1Prefix, iter.getSeconds()); - - RedisBucketValue valueProto = - RedisBucketValue.newBuilder() - .setValue(createValue(entityId, featureId, iter, fs.getValueType())) - .setEventTimestamp(iter) - .build(); - - byte[] bucketKey = bucketKeyProto.toByteArray(); - byte[] key = keyProto.toByteArray(); - byte[] value = valueProto.toByteArray(); - pipeline.set(key, value); - pipeline.zadd(bucketKey, iter.getSeconds(), key); - - // add last value in bucket 0 - if (Timestamps.compare(iter, end) == 0) { - RedisBucketKey lastValueKey = createBucketKey(entityId, featureIdSha1Prefix, 0); - pipeline.set(lastValueKey.toByteArray(), value); + Timestamp roundedTimestamp = TimeUtil.roundFloorTimestamp(timestamp, fs.getGranularity()); + addData(entityId, fs, roundedTimestamp); } } - pipeline.sync(); } /** - * Add singular data as key value. + * Add feature value. * * @param entityId entityId of data to be added. * @param fs feature spec of the feature to be added. + * @param timestamp timestamp of data */ - private void addGranularityNoneData(String entityId, FeatureSpec fs) { + private void addData(String entityId, FeatureSpec fs, Timestamp timestamp) { RedisBucketKey bucketKey = createBucketKey(entityId, getFeatureIdSha1Prefix(fs.getId()), 0); RedisBucketValue bucketValue = - RedisBucketValue.newBuilder().setValue(createValue(entityId, fs.getId(), fs.getValueType())).build(); + RedisBucketValue.newBuilder() + .setValue(createValue(entityId, fs.getId(), timestamp, fs.getValueType())) + .setEventTimestamp(timestamp) + .build(); byte[] key = bucketKey.toByteArray(); byte[] value = bucketValue.toByteArray(); jedis.set(key, value); diff --git a/serving/src/test/java/feast/serving/testutil/TimeUtil.java b/serving/src/test/java/feast/serving/testutil/TimeUtil.java new file mode 100644 index 0000000000..77e4edea97 --- /dev/null +++ b/serving/src/test/java/feast/serving/testutil/TimeUtil.java @@ -0,0 +1,60 @@ +package feast.serving.testutil; + +import com.google.protobuf.Timestamp; +import feast.types.GranularityProto.Granularity.Enum; +import org.joda.time.DateTime; +import org.joda.time.DateTimeField; +import org.joda.time.DateTimeZone; +import org.joda.time.MutableDateTime; + +public class TimeUtil { + public static final int NANO_IN_MILLI = 1000000; + public static final int NANO_IN_MICRO = 1000; + public static final int MILLI_IN_SECOND = 1000; + + /** + * Round down timestamp to the nearest granularity. + * + * @param timestamp original timestamp. + * @param granularity granularity of the rounded timestamp. + * @return + */ + public static Timestamp roundFloorTimestamp(Timestamp timestamp, Enum granularity) { + MutableDateTime dt = new MutableDateTime(DateTimeZone.UTC); + DateTimeField roundingField; + switch (granularity) { + case DAY: + roundingField = dt.getChronology().dayOfMonth(); + break; + case HOUR: + roundingField = dt.getChronology().hourOfDay(); + break; + case MINUTE: + roundingField = dt.getChronology().minuteOfHour(); + break; + case SECOND: + roundingField = dt.getChronology().secondOfMinute(); + break; + case NONE: + return Timestamp.newBuilder().setSeconds(0).setNanos(0).build(); + default: + throw new RuntimeException("Unrecognised time series granularity"); + } + dt.setMillis(timestamp.getSeconds() * 1000 + timestamp.getNanos() / 1000000); + dt.setRounding(roundingField, MutableDateTime.ROUND_FLOOR); + return dateTimeToTimestamp(dt.toDateTime()); + } + + /** + * Convert {@link DateTime} into {@link Timestamp} + * + * @param dateTime + * @return + */ + private static Timestamp dateTimeToTimestamp(DateTime dateTime) { + return Timestamp.newBuilder() + .setSeconds(dateTime.getMillis() / MILLI_IN_SECOND) + .setNanos((int) (dateTime.getMillis() % MILLI_IN_SECOND) * NANO_IN_MILLI) + .build(); + } +} diff --git a/serving/src/test/java/feast/serving/util/EntityMapBuilderTest.java b/serving/src/test/java/feast/serving/util/EntityMapBuilderTest.java index 526cab17d3..e2cf193876 100644 --- a/serving/src/test/java/feast/serving/util/EntityMapBuilderTest.java +++ b/serving/src/test/java/feast/serving/util/EntityMapBuilderTest.java @@ -19,7 +19,6 @@ import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.lessThan; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertThat; @@ -27,16 +26,12 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.google.protobuf.Duration; import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Durations; import com.google.protobuf.util.Timestamps; +import feast.serving.ServingAPIProto; import feast.serving.ServingAPIProto.Entity; -import feast.serving.ServingAPIProto.FeatureValueList; import feast.serving.model.FeatureValue; -import feast.types.ValueProto.TimestampList; import feast.types.ValueProto.Value; -import feast.types.ValueProto.ValueList; import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -58,15 +53,14 @@ public void addFeatureValueList_shouldWorkForOneEntityIdAndOneFeatureId() { EntityMapBuilder builder = new EntityMapBuilder(); List entityIds = createEntityIds(1); List featureIds = createFeatureIds(1); - Timestamp start = Timestamps.fromSeconds(0); - Timestamp end = Timestamps.fromSeconds(10); - List featureValueList = createFeatureValues(entityIds, featureIds, start, end); + Timestamp timestamp = Timestamps.fromSeconds(0); + List featureValueList = createFeatureValues(entityIds, featureIds, timestamp); builder.addFeatureValueList(featureValueList); Map result = builder.toEntityMap(); - validate(result, entityIds, featureIds, start, end); + validate(result, entityIds, featureIds, timestamp); } @Test @@ -74,14 +68,13 @@ public void addFeatureValueList_shouldWorkForSeveralEntityIdAndOneFeatureId() { EntityMapBuilder builder = new EntityMapBuilder(); List entityIds = createEntityIds(10); List featureIds = createFeatureIds(1); - Timestamp start = Timestamps.fromSeconds(0); - Timestamp end = Timestamps.fromSeconds(10); - List featureValueList = createFeatureValues(entityIds, featureIds, start, end); + Timestamp timestamp = Timestamps.fromSeconds(0); + List featureValueList = createFeatureValues(entityIds, featureIds, timestamp); builder.addFeatureValueList(featureValueList); Map result = builder.toEntityMap(); - validate(result, entityIds, featureIds, start, end); + validate(result, entityIds, featureIds, timestamp); } @Test @@ -89,15 +82,14 @@ public void addFeatureValueList_shouldWorkForSeveralEntityIdAndSeveralFeatureId( EntityMapBuilder builder = new EntityMapBuilder(); List entityIds = createEntityIds(10); List featureIds = createFeatureIds(10); - Timestamp start = Timestamps.fromSeconds(0); - Timestamp end = Timestamps.fromSeconds(10); - List featureValueList = createFeatureValues(entityIds, featureIds, start, end); + Timestamp timestamp = Timestamps.fromSeconds(0); + List featureValueList = createFeatureValues(entityIds, featureIds, timestamp); builder.addFeatureValueList(featureValueList); Map result = builder.toEntityMap(); - validate(result, entityIds, featureIds, start, end); + validate(result, entityIds, featureIds, timestamp); } @Test @@ -105,19 +97,18 @@ public void addFeatureValueList_shouldWorkForMultipleCallOfDifferentEntityId() { EntityMapBuilder builder = new EntityMapBuilder(); List entityIds = createEntityIds(20); List featureIds = createFeatureIds(20); - Timestamp start = Timestamps.fromSeconds(0); - Timestamp end = Timestamps.fromSeconds(10); + Timestamp timestamp = Timestamps.fromSeconds(0); List featureValueList1 = - createFeatureValues(entityIds.subList(0, 10), featureIds, start, end); + createFeatureValues(entityIds.subList(0, 10), featureIds, timestamp); List featureValueList2 = - createFeatureValues(entityIds.subList(10, 20), featureIds, start, end); + createFeatureValues(entityIds.subList(10, 20), featureIds, timestamp); builder.addFeatureValueList(featureValueList1); builder.addFeatureValueList(featureValueList2); Map result = builder.toEntityMap(); - validate(result, entityIds, featureIds, start, end); + validate(result, entityIds, featureIds, timestamp); } @Test @@ -125,19 +116,18 @@ public void addFeatureValueList_shouldWorkForMultipleCallOfDifferentFeature() { EntityMapBuilder builder = new EntityMapBuilder(); List entityIds = createEntityIds(20); List featureIds = createFeatureIds(20); - Timestamp start = Timestamps.fromSeconds(0); - Timestamp end = Timestamps.fromSeconds(10); + Timestamp timestamp = Timestamps.fromSeconds(0); List featureValueList1 = - createFeatureValues(entityIds, featureIds.subList(0, 10), start, end); + createFeatureValues(entityIds, featureIds.subList(0, 10), timestamp); List featureValueList2 = - createFeatureValues(entityIds, featureIds.subList(10, 20), start, end); + createFeatureValues(entityIds, featureIds.subList(10, 20), timestamp); builder.addFeatureValueList(featureValueList1); builder.addFeatureValueList(featureValueList2); Map result = builder.toEntityMap(); - validate(result, entityIds, featureIds, start, end); + validate(result, entityIds, featureIds, timestamp); } @Test @@ -153,8 +143,7 @@ public void shouldBeThreadSafe() throws Exception { EntityMapBuilder builder = new EntityMapBuilder(); List entityIds = createEntityIds(20); List featureIds = createFeatureIds(featurePerThread * nbThread); - Timestamp start = Timestamps.fromSeconds(0); - Timestamp end = Timestamps.fromSeconds(10); + Timestamp timestamp = Timestamps.fromSeconds(0); List> featureValueList = new ArrayList<>(); for (int i = 0; i < nbThread; i++) { @@ -162,8 +151,7 @@ public void shouldBeThreadSafe() throws Exception { createFeatureValues( entityIds, featureIds.subList(i * featurePerThread, (i + 1) * featurePerThread), - start, - end)); + timestamp)); } List> futures = new ArrayList<>(); @@ -192,7 +180,7 @@ public void shouldBeThreadSafe() throws Exception { all.get(); - validate(builder.toEntityMap(), entityIds, featureIds, start, end); + validate(builder.toEntityMap(), entityIds, featureIds, timestamp); assertThat(overlaps.get(), greaterThan(0)); } @@ -200,34 +188,19 @@ private void validate( Map result, List entityIds, List featureIds, - Timestamp start, - Timestamp end) { + Timestamp timestamp) { assertThat(result.size(), equalTo(entityIds.size())); for (String entityId : entityIds) { Entity entity = result.get(entityId); assertNotNull(entity); - Map featureValueListMap = entity.getFeaturesMap(); - assertNotNull(featureValueListMap); - assertThat(featureValueListMap.size(), equalTo(featureIds.size())); + Map featureValueMap = entity.getFeaturesMap(); + assertNotNull(featureValueMap); + assertThat(featureValueMap.size(), equalTo(featureIds.size())); for (String featureId : featureIds) { - FeatureValueList featureValueList = featureValueListMap.get(featureId); - Duration duration = Timestamps.between(start, end); - int count = (int) duration.getSeconds(); - - ValueList valueList = featureValueList.getValueList(); - TimestampList tsList = featureValueList.getTimestampList(); - assertThat(valueList.getInt64List().getValCount(), equalTo(count)); - assertThat(tsList.getValCount(), equalTo(count)); - - // check ordering - for (int i = 0; i < count; i++) { - if (i == 0) { - continue; - } - - // compare that timestamp is ordered in descending order. - assertThat(Timestamps.compare(tsList.getVal(i), tsList.getVal(i - 1)), lessThan(0)); - } + ServingAPIProto.FeatureValue featureValue = featureValueMap.get(featureId); + + assertThat(featureValue.getTimestamp(), equalTo(timestamp)); + assertThat(timestamp.getSeconds(), equalTo(featureValue.getValue().getInt64Val())); } } } @@ -249,20 +222,16 @@ private List createEntityIds(int count) { } private List createFeatureValues( - List entityIds, List featureIds, Timestamp start, Timestamp end) { + List entityIds, List featureIds, Timestamp timestamp) { List featureValues = new ArrayList<>(); for (String entityId : entityIds) { for (String featureId : featureIds) { - for (Timestamp iter = start; - Timestamps.compare(iter, end) < 0; - iter = Timestamps.add(iter, Durations.fromSeconds(1))) { - featureValues.add( - new FeatureValue( - featureId, - entityId, - Value.newBuilder().setInt64Val(iter.getSeconds()).build(), - iter)); - } + featureValues.add( + new FeatureValue( + featureId, + entityId, + Value.newBuilder().setInt64Val(timestamp.getSeconds()).build(), + timestamp)); } } return featureValues; diff --git a/serving/src/test/java/feast/serving/util/FeatureBuilderTest.java b/serving/src/test/java/feast/serving/util/FeatureBuilderTest.java deleted file mode 100644 index c7442b43cc..0000000000 --- a/serving/src/test/java/feast/serving/util/FeatureBuilderTest.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright 2018 The Feast Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package feast.serving.util; - -import com.google.protobuf.ByteString; -import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.FeatureValueList; -import feast.types.ValueProto.Value; -import feast.types.ValueProto.ValueList; -import feast.types.ValueProto.ValueType; -import feast.types.ValueProto.ValueType.Enum; -import org.junit.Test; - -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertThat; - -public class FeatureBuilderTest { - - private static final Random random = new Random(); - - @Test - public void shouldBeAbleToStoreValueWithoutTimestamp() { - for (ValueType.Enum type : ValueType.Enum.values()) { - if (type.equals(Enum.UNKNOWN) || type.equals(Enum.UNRECOGNIZED)) { - continue; - } - - Value val = createRandomValue(type); - - FeatureValueListBuilder featureBuilder = new FeatureValueListBuilder(); - featureBuilder.addValue(val, null); - - FeatureValueList feature = featureBuilder.build(); - assertThat(getValueList(feature.getValueList()).get(0), equalTo(getValue(val))); - } - } - - @Test - public void shouldBeAbleToStoreValueWithTimestamp() { - for (ValueType.Enum type : ValueType.Enum.values()) { - if (type.equals(Enum.UNKNOWN) || type.equals(Enum.UNRECOGNIZED)) { - continue; - } - - Value val = createRandomValue(type); - - FeatureValueListBuilder featureBuilder = new FeatureValueListBuilder(); - Timestamp ts = Timestamp.newBuilder().setSeconds(1234).build(); - - featureBuilder.addValue(val, ts); - - FeatureValueList feature = featureBuilder.build(); - assertThat(getValueList(feature.getValueList()).get(0), equalTo(getValue(val))); - assertThat(feature.getTimestampList().getVal(0), equalTo(ts)); - } - } - - @Test - public void shouldBeAbleToStoreMultipleValueWithTimestamp() { - - for (ValueType.Enum type : ValueType.Enum.values()) { - if (type.equals(Enum.UNKNOWN) || type.equals(Enum.UNRECOGNIZED)) { - continue; - } - - FeatureValueListBuilder fb = new FeatureValueListBuilder(); - List values = new ArrayList<>(); - List timestamps = new ArrayList<>(); - for (int i = 0; i < 10; i++) { - Value val = createRandomValue(type); - Timestamp ts = Timestamp.newBuilder().setSeconds(i).build(); - fb.addValue(val, ts); - - values.add(val); - timestamps.add(ts); - } - - FeatureValueList feature = fb.build(); - for (int i = 0; i < 10; i++) { - assertThat(getValueList(feature.getValueList()).get(i), equalTo(getValue(values.get(i)))); - assertThat(feature.getTimestampList().getVal(i), equalTo(timestamps.get(i))); - } - } - } - - private Value createRandomValue(ValueType.Enum type) { - Value.Builder valBuilder = Value.newBuilder(); - switch (type) { - case INT32: - valBuilder.setInt32Val(random.nextInt()); - break; - case INT64: - valBuilder.setInt64Val(random.nextLong()); - break; - case BYTES: - byte[] randomBytes = new byte[5]; - random.nextBytes(randomBytes); - valBuilder.setBytesVal(ByteString.copyFrom(randomBytes)); - break; - case STRING: - valBuilder.setStringVal("value_" + random.nextInt()); - break; - case BOOL: - valBuilder.setBoolVal(true); - break; - case FLOAT: - valBuilder.setFloatVal(random.nextFloat()); - break; - case DOUBLE: - valBuilder.setDoubleVal(random.nextDouble()); - break; - case TIMESTAMP: - valBuilder.setTimestampVal(Timestamps.fromSeconds(random.nextInt())); - break; - default: - throw new IllegalArgumentException("unknown type: " + type); - } - - return valBuilder.build(); - } - - private Object getValue(Value value) { - switch (value.getValCase()) { - case FLOATVAL: - return value.getFloatVal(); - case BOOLVAL: - return value.getBoolVal(); - case BYTESVAL: - return value.getBytesVal(); - case INT32VAL: - return value.getInt32Val(); - case INT64VAL: - return value.getInt64Val(); - case STRINGVAL: - return value.getStringVal(); - case DOUBLEVAL: - return value.getDoubleVal(); - case TIMESTAMPVAL: - return value.getTimestampVal(); - default: - throw new IllegalArgumentException("Unknown type"); - } - } - - private List getValueList(ValueList valueList) { - switch (valueList.getValueListCase()) { - case BOOLLIST: - return valueList.getBoolList().getValList(); - case BYTESLIST: - return valueList.getBytesList().getValList(); - case FLOATLIST: - return valueList.getFloatList().getValList(); - case INT32LIST: - return valueList.getInt32List().getValList(); - case INT64LIST: - return valueList.getInt64List().getValList(); - case DOUBLELIST: - return valueList.getDoubleList().getValList(); - case STRINGLIST: - return valueList.getStringList().getValList(); - case TIMESTAMPLIST: - return valueList.getTimestampList().getValList(); - default: - throw new IllegalArgumentException("Unknown type"); - } - } -} From 66852ab8445bffc0d2e92bf61339ad12e6e30fed Mon Sep 17 00:00:00 2001 From: Pradithya Aria Date: Sat, 19 Jan 2019 00:21:52 +0800 Subject: [PATCH 2/8] Update golang generated code --- protos/feast/serving/Serving.proto | 1 + .../generated/go/feast/core/CoreService.pb.go | 90 ++-- .../generated/go/feast/core/JobService.pb.go | 60 ++- .../generated/go/feast/core/UIService.pb.go | 108 +++-- .../generated/go/feast/serving/Serving.pb.go | 398 ++++++------------ .../generated/go/feast/specs/EntitySpec.pb.go | 12 +- .../go/feast/specs/FeatureGroupSpec.pb.go | 12 +- .../go/feast/specs/FeatureSpec.pb.go | 24 +- .../go/feast/specs/StorageSpec.pb.go | 12 +- protos/generated/go/feast/storage/Redis.pb.go | 22 +- .../go/feast/types/FeatureRowExtended.pb.go | 22 +- .../go/feast/types/Granularity.pb.go | 16 +- protos/generated/go/feast/types/Value.pb.go | 74 ++-- 13 files changed, 355 insertions(+), 496 deletions(-) diff --git a/protos/feast/serving/Serving.proto b/protos/feast/serving/Serving.proto index a05a142ae7..5ded4becf2 100644 --- a/protos/feast/serving/Serving.proto +++ b/protos/feast/serving/Serving.proto @@ -26,6 +26,7 @@ option java_outer_classname = "ServingAPIProto"; option go_package = "github.com/gojek/feast/protos/generated/go/feast/serving"; service ServingAPI { + // Query features from Feast serving storage rpc QueryFeatures (QueryFeaturesRequest) returns (QueryFeaturesResponse) {}; } diff --git a/protos/generated/go/feast/core/CoreService.pb.go b/protos/generated/go/feast/core/CoreService.pb.go index 4037da58af..d8224c13a9 100644 --- a/protos/generated/go/feast/core/CoreService.pb.go +++ b/protos/generated/go/feast/core/CoreService.pb.go @@ -35,7 +35,7 @@ func (m *CoreServiceTypes) Reset() { *m = CoreServiceTypes{} } func (m *CoreServiceTypes) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes) ProtoMessage() {} func (*CoreServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0} + return fileDescriptor_d9be266444105411, []int{0} } func (m *CoreServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes.Unmarshal(m, b) @@ -43,8 +43,8 @@ func (m *CoreServiceTypes) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes.Merge(dst, src) +func (m *CoreServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes.Merge(m, src) } func (m *CoreServiceTypes) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes.Size(m) @@ -66,7 +66,7 @@ func (m *CoreServiceTypes_GetEntitiesRequest) Reset() { *m = CoreService func (m *CoreServiceTypes_GetEntitiesRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetEntitiesRequest) ProtoMessage() {} func (*CoreServiceTypes_GetEntitiesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 0} + return fileDescriptor_d9be266444105411, []int{0, 0} } func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Unmarshal(m, b) @@ -74,8 +74,8 @@ func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetEntitiesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Merge(dst, src) +func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Merge(m, src) } func (m *CoreServiceTypes_GetEntitiesRequest) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetEntitiesRequest.Size(m) @@ -104,7 +104,7 @@ func (m *CoreServiceTypes_GetEntitiesResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_GetEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetEntitiesResponse) ProtoMessage() {} func (*CoreServiceTypes_GetEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 1} + return fileDescriptor_d9be266444105411, []int{0, 1} } func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Unmarshal(m, b) @@ -112,8 +112,8 @@ func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetEntitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Merge(dst, src) +func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Merge(m, src) } func (m *CoreServiceTypes_GetEntitiesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetEntitiesResponse.Size(m) @@ -142,7 +142,7 @@ func (m *CoreServiceTypes_ListEntitiesResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ListEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListEntitiesResponse) ProtoMessage() {} func (*CoreServiceTypes_ListEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 2} + return fileDescriptor_d9be266444105411, []int{0, 2} } func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Unmarshal(m, b) @@ -150,8 +150,8 @@ func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Merge(dst, src) +func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Merge(m, src) } func (m *CoreServiceTypes_ListEntitiesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ListEntitiesResponse.Size(m) @@ -181,7 +181,7 @@ func (m *CoreServiceTypes_GetFeaturesRequest) Reset() { *m = CoreService func (m *CoreServiceTypes_GetFeaturesRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetFeaturesRequest) ProtoMessage() {} func (*CoreServiceTypes_GetFeaturesRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 3} + return fileDescriptor_d9be266444105411, []int{0, 3} } func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Unmarshal(m, b) @@ -189,8 +189,8 @@ func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetFeaturesRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Merge(dst, src) +func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Merge(m, src) } func (m *CoreServiceTypes_GetFeaturesRequest) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetFeaturesRequest.Size(m) @@ -219,7 +219,7 @@ func (m *CoreServiceTypes_GetFeaturesResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_GetFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetFeaturesResponse) ProtoMessage() {} func (*CoreServiceTypes_GetFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 4} + return fileDescriptor_d9be266444105411, []int{0, 4} } func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Unmarshal(m, b) @@ -227,8 +227,8 @@ func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetFeaturesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Merge(dst, src) +func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Merge(m, src) } func (m *CoreServiceTypes_GetFeaturesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetFeaturesResponse.Size(m) @@ -257,7 +257,7 @@ func (m *CoreServiceTypes_ListFeaturesResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ListFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListFeaturesResponse) ProtoMessage() {} func (*CoreServiceTypes_ListFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 5} + return fileDescriptor_d9be266444105411, []int{0, 5} } func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Unmarshal(m, b) @@ -265,8 +265,8 @@ func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Merge(dst, src) +func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Merge(m, src) } func (m *CoreServiceTypes_ListFeaturesResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ListFeaturesResponse.Size(m) @@ -296,7 +296,7 @@ func (m *CoreServiceTypes_GetStorageRequest) Reset() { *m = CoreServiceT func (m *CoreServiceTypes_GetStorageRequest) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetStorageRequest) ProtoMessage() {} func (*CoreServiceTypes_GetStorageRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 6} + return fileDescriptor_d9be266444105411, []int{0, 6} } func (m *CoreServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Unmarshal(m, b) @@ -304,8 +304,8 @@ func (m *CoreServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetStorageRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Merge(dst, src) +func (m *CoreServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Merge(m, src) } func (m *CoreServiceTypes_GetStorageRequest) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetStorageRequest.Size(m) @@ -334,7 +334,7 @@ func (m *CoreServiceTypes_GetStorageResponse) Reset() { *m = CoreService func (m *CoreServiceTypes_GetStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_GetStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_GetStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 7} + return fileDescriptor_d9be266444105411, []int{0, 7} } func (m *CoreServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Unmarshal(m, b) @@ -342,8 +342,8 @@ func (m *CoreServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_GetStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Merge(dst, src) +func (m *CoreServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Merge(m, src) } func (m *CoreServiceTypes_GetStorageResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_GetStorageResponse.Size(m) @@ -372,7 +372,7 @@ func (m *CoreServiceTypes_ListStorageResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_ListStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ListStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_ListStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 8} + return fileDescriptor_d9be266444105411, []int{0, 8} } func (m *CoreServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Unmarshal(m, b) @@ -380,8 +380,8 @@ func (m *CoreServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ListStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Merge(dst, src) +func (m *CoreServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Merge(m, src) } func (m *CoreServiceTypes_ListStorageResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ListStorageResponse.Size(m) @@ -411,7 +411,7 @@ func (m *CoreServiceTypes_ApplyEntityResponse) Reset() { *m = CoreServic func (m *CoreServiceTypes_ApplyEntityResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ApplyEntityResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyEntityResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 9} + return fileDescriptor_d9be266444105411, []int{0, 9} } func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Unmarshal(m, b) @@ -419,8 +419,8 @@ func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ApplyEntityResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Merge(dst, src) +func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Merge(m, src) } func (m *CoreServiceTypes_ApplyEntityResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyEntityResponse.Size(m) @@ -450,7 +450,7 @@ func (m *CoreServiceTypes_ApplyFeatureResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ApplyFeatureResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ApplyFeatureResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyFeatureResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 10} + return fileDescriptor_d9be266444105411, []int{0, 10} } func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Unmarshal(m, b) @@ -458,8 +458,8 @@ func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ApplyFeatureResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Merge(dst, src) +func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Merge(m, src) } func (m *CoreServiceTypes_ApplyFeatureResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureResponse.Size(m) @@ -493,7 +493,7 @@ func (m *CoreServiceTypes_ApplyFeatureGroupResponse) String() string { } func (*CoreServiceTypes_ApplyFeatureGroupResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyFeatureGroupResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 11} + return fileDescriptor_d9be266444105411, []int{0, 11} } func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Unmarshal(m, b) @@ -501,8 +501,8 @@ func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Unmarshal(b []byte) err func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Merge(dst, src) +func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Merge(m, src) } func (m *CoreServiceTypes_ApplyFeatureGroupResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyFeatureGroupResponse.Size(m) @@ -532,7 +532,7 @@ func (m *CoreServiceTypes_ApplyStorageResponse) Reset() { *m = CoreServi func (m *CoreServiceTypes_ApplyStorageResponse) String() string { return proto.CompactTextString(m) } func (*CoreServiceTypes_ApplyStorageResponse) ProtoMessage() {} func (*CoreServiceTypes_ApplyStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_CoreService_a0e9a1504f969203, []int{0, 12} + return fileDescriptor_d9be266444105411, []int{0, 12} } func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Unmarshal(m, b) @@ -540,8 +540,8 @@ func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Unmarshal(b []byte) error { func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Marshal(b, m, deterministic) } -func (dst *CoreServiceTypes_ApplyStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Merge(dst, src) +func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Merge(m, src) } func (m *CoreServiceTypes_ApplyStorageResponse) XXX_Size() int { return xxx_messageInfo_CoreServiceTypes_ApplyStorageResponse.Size(m) @@ -1005,11 +1005,9 @@ var _CoreService_serviceDesc = grpc.ServiceDesc{ Metadata: "feast/core/CoreService.proto", } -func init() { - proto.RegisterFile("feast/core/CoreService.proto", fileDescriptor_CoreService_a0e9a1504f969203) -} +func init() { proto.RegisterFile("feast/core/CoreService.proto", fileDescriptor_d9be266444105411) } -var fileDescriptor_CoreService_a0e9a1504f969203 = []byte{ +var fileDescriptor_d9be266444105411 = []byte{ // 602 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x5b, 0x6f, 0x12, 0x41, 0x14, 0x86, 0x34, 0x69, 0xe0, 0x40, 0x4c, 0x3b, 0x34, 0x8a, 0x23, 0x35, 0xcd, 0x26, 0x36, 0x7d, diff --git a/protos/generated/go/feast/core/JobService.pb.go b/protos/generated/go/feast/core/JobService.pb.go index 6a3a4e6dc0..699d2b4d32 100644 --- a/protos/generated/go/feast/core/JobService.pb.go +++ b/protos/generated/go/feast/core/JobService.pb.go @@ -36,7 +36,7 @@ func (m *JobServiceTypes) Reset() { *m = JobServiceTypes{} } func (m *JobServiceTypes) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes) ProtoMessage() {} func (*JobServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0} + return fileDescriptor_7115affcfc7885c4, []int{0} } func (m *JobServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes.Unmarshal(m, b) @@ -44,8 +44,8 @@ func (m *JobServiceTypes) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes.Merge(dst, src) +func (m *JobServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes.Merge(m, src) } func (m *JobServiceTypes) XXX_Size() int { return xxx_messageInfo_JobServiceTypes.Size(m) @@ -70,7 +70,7 @@ func (m *JobServiceTypes_SubmitImportJobRequest) Reset() { func (m *JobServiceTypes_SubmitImportJobRequest) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_SubmitImportJobRequest) ProtoMessage() {} func (*JobServiceTypes_SubmitImportJobRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 0} + return fileDescriptor_7115affcfc7885c4, []int{0, 0} } func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Unmarshal(m, b) @@ -78,8 +78,8 @@ func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_SubmitImportJobRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Merge(dst, src) +func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Merge(m, src) } func (m *JobServiceTypes_SubmitImportJobRequest) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_SubmitImportJobRequest.Size(m) @@ -117,7 +117,7 @@ func (m *JobServiceTypes_SubmitImportJobResponse) Reset() { func (m *JobServiceTypes_SubmitImportJobResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_SubmitImportJobResponse) ProtoMessage() {} func (*JobServiceTypes_SubmitImportJobResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 1} + return fileDescriptor_7115affcfc7885c4, []int{0, 1} } func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Unmarshal(m, b) @@ -125,8 +125,8 @@ func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Unmarshal(b []byte) error func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_SubmitImportJobResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Merge(dst, src) +func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Merge(m, src) } func (m *JobServiceTypes_SubmitImportJobResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_SubmitImportJobResponse.Size(m) @@ -155,7 +155,7 @@ func (m *JobServiceTypes_ListJobsResponse) Reset() { *m = JobServiceType func (m *JobServiceTypes_ListJobsResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_ListJobsResponse) ProtoMessage() {} func (*JobServiceTypes_ListJobsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 2} + return fileDescriptor_7115affcfc7885c4, []int{0, 2} } func (m *JobServiceTypes_ListJobsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_ListJobsResponse.Unmarshal(m, b) @@ -163,8 +163,8 @@ func (m *JobServiceTypes_ListJobsResponse) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_ListJobsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_ListJobsResponse.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_ListJobsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_ListJobsResponse.Merge(dst, src) +func (m *JobServiceTypes_ListJobsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_ListJobsResponse.Merge(m, src) } func (m *JobServiceTypes_ListJobsResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_ListJobsResponse.Size(m) @@ -193,7 +193,7 @@ func (m *JobServiceTypes_GetJobRequest) Reset() { *m = JobServiceTypes_G func (m *JobServiceTypes_GetJobRequest) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_GetJobRequest) ProtoMessage() {} func (*JobServiceTypes_GetJobRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 3} + return fileDescriptor_7115affcfc7885c4, []int{0, 3} } func (m *JobServiceTypes_GetJobRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_GetJobRequest.Unmarshal(m, b) @@ -201,8 +201,8 @@ func (m *JobServiceTypes_GetJobRequest) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_GetJobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_GetJobRequest.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_GetJobRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_GetJobRequest.Merge(dst, src) +func (m *JobServiceTypes_GetJobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_GetJobRequest.Merge(m, src) } func (m *JobServiceTypes_GetJobRequest) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_GetJobRequest.Size(m) @@ -231,7 +231,7 @@ func (m *JobServiceTypes_GetJobResponse) Reset() { *m = JobServiceTypes_ func (m *JobServiceTypes_GetJobResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_GetJobResponse) ProtoMessage() {} func (*JobServiceTypes_GetJobResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 4} + return fileDescriptor_7115affcfc7885c4, []int{0, 4} } func (m *JobServiceTypes_GetJobResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_GetJobResponse.Unmarshal(m, b) @@ -239,8 +239,8 @@ func (m *JobServiceTypes_GetJobResponse) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_GetJobResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_GetJobResponse.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_GetJobResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_GetJobResponse.Merge(dst, src) +func (m *JobServiceTypes_GetJobResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_GetJobResponse.Merge(m, src) } func (m *JobServiceTypes_GetJobResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_GetJobResponse.Size(m) @@ -269,7 +269,7 @@ func (m *JobServiceTypes_AbortJobRequest) Reset() { *m = JobServiceTypes func (m *JobServiceTypes_AbortJobRequest) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_AbortJobRequest) ProtoMessage() {} func (*JobServiceTypes_AbortJobRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 5} + return fileDescriptor_7115affcfc7885c4, []int{0, 5} } func (m *JobServiceTypes_AbortJobRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_AbortJobRequest.Unmarshal(m, b) @@ -277,8 +277,8 @@ func (m *JobServiceTypes_AbortJobRequest) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_AbortJobRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_AbortJobRequest.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_AbortJobRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_AbortJobRequest.Merge(dst, src) +func (m *JobServiceTypes_AbortJobRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_AbortJobRequest.Merge(m, src) } func (m *JobServiceTypes_AbortJobRequest) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_AbortJobRequest.Size(m) @@ -307,7 +307,7 @@ func (m *JobServiceTypes_AbortJobResponse) Reset() { *m = JobServiceType func (m *JobServiceTypes_AbortJobResponse) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_AbortJobResponse) ProtoMessage() {} func (*JobServiceTypes_AbortJobResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 6} + return fileDescriptor_7115affcfc7885c4, []int{0, 6} } func (m *JobServiceTypes_AbortJobResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_AbortJobResponse.Unmarshal(m, b) @@ -315,8 +315,8 @@ func (m *JobServiceTypes_AbortJobResponse) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_AbortJobResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_AbortJobResponse.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_AbortJobResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_AbortJobResponse.Merge(dst, src) +func (m *JobServiceTypes_AbortJobResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_AbortJobResponse.Merge(m, src) } func (m *JobServiceTypes_AbortJobResponse) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_AbortJobResponse.Size(m) @@ -356,7 +356,7 @@ func (m *JobServiceTypes_JobDetail) Reset() { *m = JobServiceTypes_JobDe func (m *JobServiceTypes_JobDetail) String() string { return proto.CompactTextString(m) } func (*JobServiceTypes_JobDetail) ProtoMessage() {} func (*JobServiceTypes_JobDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_JobService_edcd183b773c9f62, []int{0, 7} + return fileDescriptor_7115affcfc7885c4, []int{0, 7} } func (m *JobServiceTypes_JobDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JobServiceTypes_JobDetail.Unmarshal(m, b) @@ -364,8 +364,8 @@ func (m *JobServiceTypes_JobDetail) XXX_Unmarshal(b []byte) error { func (m *JobServiceTypes_JobDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_JobServiceTypes_JobDetail.Marshal(b, m, deterministic) } -func (dst *JobServiceTypes_JobDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_JobServiceTypes_JobDetail.Merge(dst, src) +func (m *JobServiceTypes_JobDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_JobServiceTypes_JobDetail.Merge(m, src) } func (m *JobServiceTypes_JobDetail) XXX_Size() int { return xxx_messageInfo_JobServiceTypes_JobDetail.Size(m) @@ -638,11 +638,9 @@ var _JobService_serviceDesc = grpc.ServiceDesc{ Metadata: "feast/core/JobService.proto", } -func init() { - proto.RegisterFile("feast/core/JobService.proto", fileDescriptor_JobService_edcd183b773c9f62) -} +func init() { proto.RegisterFile("feast/core/JobService.proto", fileDescriptor_7115affcfc7885c4) } -var fileDescriptor_JobService_edcd183b773c9f62 = []byte{ +var fileDescriptor_7115affcfc7885c4 = []byte{ // 621 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0xdd, 0x4e, 0xdb, 0x4c, 0x10, 0x55, 0x62, 0x08, 0xf1, 0xf0, 0x7d, 0x80, 0x56, 0x15, 0x58, 0x4b, 0x25, 0x52, 0xa4, 0x4a, diff --git a/protos/generated/go/feast/core/UIService.pb.go b/protos/generated/go/feast/core/UIService.pb.go index 772c548c87..87adc7210e 100644 --- a/protos/generated/go/feast/core/UIService.pb.go +++ b/protos/generated/go/feast/core/UIService.pb.go @@ -36,7 +36,7 @@ func (m *UIServiceTypes) Reset() { *m = UIServiceTypes{} } func (m *UIServiceTypes) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes) ProtoMessage() {} func (*UIServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0} + return fileDescriptor_c13b3edb35d457e8, []int{0} } func (m *UIServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes.Unmarshal(m, b) @@ -44,8 +44,8 @@ func (m *UIServiceTypes) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes.Merge(dst, src) +func (m *UIServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes.Merge(m, src) } func (m *UIServiceTypes) XXX_Size() int { return xxx_messageInfo_UIServiceTypes.Size(m) @@ -70,7 +70,7 @@ func (m *UIServiceTypes_EntityDetail) Reset() { *m = UIServiceTypes_Enti func (m *UIServiceTypes_EntityDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_EntityDetail) ProtoMessage() {} func (*UIServiceTypes_EntityDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 0} + return fileDescriptor_c13b3edb35d457e8, []int{0, 0} } func (m *UIServiceTypes_EntityDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_EntityDetail.Unmarshal(m, b) @@ -78,8 +78,8 @@ func (m *UIServiceTypes_EntityDetail) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_EntityDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_EntityDetail.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_EntityDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_EntityDetail.Merge(dst, src) +func (m *UIServiceTypes_EntityDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_EntityDetail.Merge(m, src) } func (m *UIServiceTypes_EntityDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_EntityDetail.Size(m) @@ -122,7 +122,7 @@ func (m *UIServiceTypes_GetEntityRequest) Reset() { *m = UIServiceTypes_ func (m *UIServiceTypes_GetEntityRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetEntityRequest) ProtoMessage() {} func (*UIServiceTypes_GetEntityRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 1} + return fileDescriptor_c13b3edb35d457e8, []int{0, 1} } func (m *UIServiceTypes_GetEntityRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetEntityRequest.Unmarshal(m, b) @@ -130,8 +130,8 @@ func (m *UIServiceTypes_GetEntityRequest) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetEntityRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetEntityRequest.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetEntityRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetEntityRequest.Merge(dst, src) +func (m *UIServiceTypes_GetEntityRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetEntityRequest.Merge(m, src) } func (m *UIServiceTypes_GetEntityRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetEntityRequest.Size(m) @@ -160,7 +160,7 @@ func (m *UIServiceTypes_GetEntityResponse) Reset() { *m = UIServiceTypes func (m *UIServiceTypes_GetEntityResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetEntityResponse) ProtoMessage() {} func (*UIServiceTypes_GetEntityResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 2} + return fileDescriptor_c13b3edb35d457e8, []int{0, 2} } func (m *UIServiceTypes_GetEntityResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetEntityResponse.Unmarshal(m, b) @@ -168,8 +168,8 @@ func (m *UIServiceTypes_GetEntityResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetEntityResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetEntityResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetEntityResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetEntityResponse.Merge(dst, src) +func (m *UIServiceTypes_GetEntityResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetEntityResponse.Merge(m, src) } func (m *UIServiceTypes_GetEntityResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetEntityResponse.Size(m) @@ -198,7 +198,7 @@ func (m *UIServiceTypes_ListEntitiesResponse) Reset() { *m = UIServiceTy func (m *UIServiceTypes_ListEntitiesResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListEntitiesResponse) ProtoMessage() {} func (*UIServiceTypes_ListEntitiesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 3} + return fileDescriptor_c13b3edb35d457e8, []int{0, 3} } func (m *UIServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Unmarshal(m, b) @@ -206,8 +206,8 @@ func (m *UIServiceTypes_ListEntitiesResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_ListEntitiesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Merge(dst, src) +func (m *UIServiceTypes_ListEntitiesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Merge(m, src) } func (m *UIServiceTypes_ListEntitiesResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListEntitiesResponse.Size(m) @@ -242,7 +242,7 @@ func (m *UIServiceTypes_FeatureDetail) Reset() { *m = UIServiceTypes_Fea func (m *UIServiceTypes_FeatureDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_FeatureDetail) ProtoMessage() {} func (*UIServiceTypes_FeatureDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 4} + return fileDescriptor_c13b3edb35d457e8, []int{0, 4} } func (m *UIServiceTypes_FeatureDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_FeatureDetail.Unmarshal(m, b) @@ -250,8 +250,8 @@ func (m *UIServiceTypes_FeatureDetail) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_FeatureDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_FeatureDetail.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_FeatureDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_FeatureDetail.Merge(dst, src) +func (m *UIServiceTypes_FeatureDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_FeatureDetail.Merge(m, src) } func (m *UIServiceTypes_FeatureDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_FeatureDetail.Size(m) @@ -315,7 +315,7 @@ func (m *UIServiceTypes_GetFeatureRequest) Reset() { *m = UIServiceTypes func (m *UIServiceTypes_GetFeatureRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureRequest) ProtoMessage() {} func (*UIServiceTypes_GetFeatureRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 5} + return fileDescriptor_c13b3edb35d457e8, []int{0, 5} } func (m *UIServiceTypes_GetFeatureRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Unmarshal(m, b) @@ -323,8 +323,8 @@ func (m *UIServiceTypes_GetFeatureRequest) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetFeatureRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetFeatureRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Merge(dst, src) +func (m *UIServiceTypes_GetFeatureRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Merge(m, src) } func (m *UIServiceTypes_GetFeatureRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureRequest.Size(m) @@ -354,7 +354,7 @@ func (m *UIServiceTypes_GetFeatureResponse) Reset() { *m = UIServiceType func (m *UIServiceTypes_GetFeatureResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureResponse) ProtoMessage() {} func (*UIServiceTypes_GetFeatureResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 6} + return fileDescriptor_c13b3edb35d457e8, []int{0, 6} } func (m *UIServiceTypes_GetFeatureResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Unmarshal(m, b) @@ -362,8 +362,8 @@ func (m *UIServiceTypes_GetFeatureResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetFeatureResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetFeatureResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Merge(dst, src) +func (m *UIServiceTypes_GetFeatureResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Merge(m, src) } func (m *UIServiceTypes_GetFeatureResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureResponse.Size(m) @@ -399,7 +399,7 @@ func (m *UIServiceTypes_ListFeaturesResponse) Reset() { *m = UIServiceTy func (m *UIServiceTypes_ListFeaturesResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListFeaturesResponse) ProtoMessage() {} func (*UIServiceTypes_ListFeaturesResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 7} + return fileDescriptor_c13b3edb35d457e8, []int{0, 7} } func (m *UIServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Unmarshal(m, b) @@ -407,8 +407,8 @@ func (m *UIServiceTypes_ListFeaturesResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_ListFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Merge(dst, src) +func (m *UIServiceTypes_ListFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Merge(m, src) } func (m *UIServiceTypes_ListFeaturesResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListFeaturesResponse.Size(m) @@ -439,7 +439,7 @@ func (m *UIServiceTypes_FeatureGroupDetail) Reset() { *m = UIServiceType func (m *UIServiceTypes_FeatureGroupDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_FeatureGroupDetail) ProtoMessage() {} func (*UIServiceTypes_FeatureGroupDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 8} + return fileDescriptor_c13b3edb35d457e8, []int{0, 8} } func (m *UIServiceTypes_FeatureGroupDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Unmarshal(m, b) @@ -447,8 +447,8 @@ func (m *UIServiceTypes_FeatureGroupDetail) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_FeatureGroupDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_FeatureGroupDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Merge(dst, src) +func (m *UIServiceTypes_FeatureGroupDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Merge(m, src) } func (m *UIServiceTypes_FeatureGroupDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_FeatureGroupDetail.Size(m) @@ -484,7 +484,7 @@ func (m *UIServiceTypes_GetFeatureGroupRequest) Reset() { *m = UIService func (m *UIServiceTypes_GetFeatureGroupRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureGroupRequest) ProtoMessage() {} func (*UIServiceTypes_GetFeatureGroupRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 9} + return fileDescriptor_c13b3edb35d457e8, []int{0, 9} } func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Unmarshal(m, b) @@ -492,8 +492,8 @@ func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetFeatureGroupRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Merge(dst, src) +func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Merge(m, src) } func (m *UIServiceTypes_GetFeatureGroupRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupRequest.Size(m) @@ -524,7 +524,7 @@ func (m *UIServiceTypes_GetFeatureGroupResponse) Reset() { func (m *UIServiceTypes_GetFeatureGroupResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetFeatureGroupResponse) ProtoMessage() {} func (*UIServiceTypes_GetFeatureGroupResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 10} + return fileDescriptor_c13b3edb35d457e8, []int{0, 10} } func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Unmarshal(m, b) @@ -532,8 +532,8 @@ func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetFeatureGroupResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Merge(dst, src) +func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Merge(m, src) } func (m *UIServiceTypes_GetFeatureGroupResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetFeatureGroupResponse.Size(m) @@ -564,7 +564,7 @@ func (m *UIServiceTypes_ListFeatureGroupsResponse) Reset() { func (m *UIServiceTypes_ListFeatureGroupsResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListFeatureGroupsResponse) ProtoMessage() {} func (*UIServiceTypes_ListFeatureGroupsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 11} + return fileDescriptor_c13b3edb35d457e8, []int{0, 11} } func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Unmarshal(m, b) @@ -572,8 +572,8 @@ func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Unmarshal(b []byte) error func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_ListFeatureGroupsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Merge(dst, src) +func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Merge(m, src) } func (m *UIServiceTypes_ListFeatureGroupsResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListFeatureGroupsResponse.Size(m) @@ -604,7 +604,7 @@ func (m *UIServiceTypes_StorageDetail) Reset() { *m = UIServiceTypes_Sto func (m *UIServiceTypes_StorageDetail) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_StorageDetail) ProtoMessage() {} func (*UIServiceTypes_StorageDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 12} + return fileDescriptor_c13b3edb35d457e8, []int{0, 12} } func (m *UIServiceTypes_StorageDetail) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_StorageDetail.Unmarshal(m, b) @@ -612,8 +612,8 @@ func (m *UIServiceTypes_StorageDetail) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_StorageDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_StorageDetail.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_StorageDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_StorageDetail.Merge(dst, src) +func (m *UIServiceTypes_StorageDetail) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_StorageDetail.Merge(m, src) } func (m *UIServiceTypes_StorageDetail) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_StorageDetail.Size(m) @@ -649,7 +649,7 @@ func (m *UIServiceTypes_GetStorageRequest) Reset() { *m = UIServiceTypes func (m *UIServiceTypes_GetStorageRequest) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetStorageRequest) ProtoMessage() {} func (*UIServiceTypes_GetStorageRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 13} + return fileDescriptor_c13b3edb35d457e8, []int{0, 13} } func (m *UIServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetStorageRequest.Unmarshal(m, b) @@ -657,8 +657,8 @@ func (m *UIServiceTypes_GetStorageRequest) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetStorageRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetStorageRequest.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetStorageRequest.Merge(dst, src) +func (m *UIServiceTypes_GetStorageRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetStorageRequest.Merge(m, src) } func (m *UIServiceTypes_GetStorageRequest) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetStorageRequest.Size(m) @@ -687,7 +687,7 @@ func (m *UIServiceTypes_GetStorageResponse) Reset() { *m = UIServiceType func (m *UIServiceTypes_GetStorageResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_GetStorageResponse) ProtoMessage() {} func (*UIServiceTypes_GetStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 14} + return fileDescriptor_c13b3edb35d457e8, []int{0, 14} } func (m *UIServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_GetStorageResponse.Unmarshal(m, b) @@ -695,8 +695,8 @@ func (m *UIServiceTypes_GetStorageResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_GetStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_GetStorageResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_GetStorageResponse.Merge(dst, src) +func (m *UIServiceTypes_GetStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_GetStorageResponse.Merge(m, src) } func (m *UIServiceTypes_GetStorageResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_GetStorageResponse.Size(m) @@ -725,7 +725,7 @@ func (m *UIServiceTypes_ListStorageResponse) Reset() { *m = UIServiceTyp func (m *UIServiceTypes_ListStorageResponse) String() string { return proto.CompactTextString(m) } func (*UIServiceTypes_ListStorageResponse) ProtoMessage() {} func (*UIServiceTypes_ListStorageResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_UIService_04866529701c634c, []int{0, 15} + return fileDescriptor_c13b3edb35d457e8, []int{0, 15} } func (m *UIServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_UIServiceTypes_ListStorageResponse.Unmarshal(m, b) @@ -733,8 +733,8 @@ func (m *UIServiceTypes_ListStorageResponse) XXX_Unmarshal(b []byte) error { func (m *UIServiceTypes_ListStorageResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_UIServiceTypes_ListStorageResponse.Marshal(b, m, deterministic) } -func (dst *UIServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_UIServiceTypes_ListStorageResponse.Merge(dst, src) +func (m *UIServiceTypes_ListStorageResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_UIServiceTypes_ListStorageResponse.Merge(m, src) } func (m *UIServiceTypes_ListStorageResponse) XXX_Size() int { return xxx_messageInfo_UIServiceTypes_ListStorageResponse.Size(m) @@ -1119,11 +1119,9 @@ var _UIService_serviceDesc = grpc.ServiceDesc{ Metadata: "feast/core/UIService.proto", } -func init() { - proto.RegisterFile("feast/core/UIService.proto", fileDescriptor_UIService_04866529701c634c) -} +func init() { proto.RegisterFile("feast/core/UIService.proto", fileDescriptor_c13b3edb35d457e8) } -var fileDescriptor_UIService_04866529701c634c = []byte{ +var fileDescriptor_c13b3edb35d457e8 = []byte{ // 784 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0x4d, 0x6f, 0xd3, 0x40, 0x10, 0xcd, 0x47, 0x49, 0x9a, 0x69, 0x5a, 0xe8, 0x82, 0xda, 0xb0, 0x50, 0x51, 0x99, 0x03, 0x91, diff --git a/protos/generated/go/feast/serving/Serving.pb.go b/protos/generated/go/feast/serving/Serving.pb.go index 30048a75bc..60ee7d3663 100644 --- a/protos/generated/go/feast/serving/Serving.pb.go +++ b/protos/generated/go/feast/serving/Serving.pb.go @@ -25,129 +25,77 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package -type RequestType int32 - -const ( - // LAST : return only the latest value based on timestamp. (default) - RequestType_LAST RequestType = 0 - // LIST : return list of historical data sorted by timestamp. - RequestType_LIST RequestType = 1 -) - -var RequestType_name = map[int32]string{ - 0: "LAST", - 1: "LIST", -} -var RequestType_value = map[string]int32{ - "LAST": 0, - "LIST": 1, -} - -func (x RequestType) String() string { - return proto.EnumName(RequestType_name, int32(x)) -} -func (RequestType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{0} -} - -type QueryFeatures struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *QueryFeatures) Reset() { *m = QueryFeatures{} } -func (m *QueryFeatures) String() string { return proto.CompactTextString(m) } -func (*QueryFeatures) ProtoMessage() {} -func (*QueryFeatures) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{0} -} -func (m *QueryFeatures) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_QueryFeatures.Unmarshal(m, b) -} -func (m *QueryFeatures) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_QueryFeatures.Marshal(b, m, deterministic) -} -func (dst *QueryFeatures) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryFeatures.Merge(dst, src) -} -func (m *QueryFeatures) XXX_Size() int { - return xxx_messageInfo_QueryFeatures.Size(m) -} -func (m *QueryFeatures) XXX_DiscardUnknown() { - xxx_messageInfo_QueryFeatures.DiscardUnknown(m) -} - -var xxx_messageInfo_QueryFeatures proto.InternalMessageInfo - -type QueryFeatures_Request struct { +type QueryFeaturesRequest struct { // e.g. "driver", "customer", "city". EntityName string `protobuf:"bytes,1,opt,name=entityName,proto3" json:"entityName,omitempty"` // List of entity ID. EntityId []string `protobuf:"bytes,2,rep,name=entityId,proto3" json:"entityId,omitempty"` - // List of request details, contains: featureId, type of query, and limit. - RequestDetails []*RequestDetail `protobuf:"bytes,3,rep,name=requestDetails,proto3" json:"requestDetails,omitempty"` - // Filter specifying only to retrieve features having timestamp within this range. - TimestampRange *TimestampRange `protobuf:"bytes,4,opt,name=timestampRange,proto3" json:"timestampRange,omitempty"` + // List of feature ID. + // feature ID is in the form of [entity_name].[granularity].[feature_name] + // e.g: "driver.day.total_accepted_booking" + // all requested feature ID shall have same entity name. + FeatureId []string `protobuf:"bytes,3,rep,name=featureId,proto3" json:"featureId,omitempty"` + // [optional] time range filter. If not specified all feature with any timestamp will be returned. + TimeRange *TimestampRange `protobuf:"bytes,4,opt,name=timeRange,proto3" json:"timeRange,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *QueryFeatures_Request) Reset() { *m = QueryFeatures_Request{} } -func (m *QueryFeatures_Request) String() string { return proto.CompactTextString(m) } -func (*QueryFeatures_Request) ProtoMessage() {} -func (*QueryFeatures_Request) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{0, 0} +func (m *QueryFeaturesRequest) Reset() { *m = QueryFeaturesRequest{} } +func (m *QueryFeaturesRequest) String() string { return proto.CompactTextString(m) } +func (*QueryFeaturesRequest) ProtoMessage() {} +func (*QueryFeaturesRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_7609de6de542e6f0, []int{0} } -func (m *QueryFeatures_Request) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_QueryFeatures_Request.Unmarshal(m, b) +func (m *QueryFeaturesRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_QueryFeaturesRequest.Unmarshal(m, b) } -func (m *QueryFeatures_Request) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_QueryFeatures_Request.Marshal(b, m, deterministic) +func (m *QueryFeaturesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_QueryFeaturesRequest.Marshal(b, m, deterministic) } -func (dst *QueryFeatures_Request) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryFeatures_Request.Merge(dst, src) +func (m *QueryFeaturesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryFeaturesRequest.Merge(m, src) } -func (m *QueryFeatures_Request) XXX_Size() int { - return xxx_messageInfo_QueryFeatures_Request.Size(m) +func (m *QueryFeaturesRequest) XXX_Size() int { + return xxx_messageInfo_QueryFeaturesRequest.Size(m) } -func (m *QueryFeatures_Request) XXX_DiscardUnknown() { - xxx_messageInfo_QueryFeatures_Request.DiscardUnknown(m) +func (m *QueryFeaturesRequest) XXX_DiscardUnknown() { + xxx_messageInfo_QueryFeaturesRequest.DiscardUnknown(m) } -var xxx_messageInfo_QueryFeatures_Request proto.InternalMessageInfo +var xxx_messageInfo_QueryFeaturesRequest proto.InternalMessageInfo -func (m *QueryFeatures_Request) GetEntityName() string { +func (m *QueryFeaturesRequest) GetEntityName() string { if m != nil { return m.EntityName } return "" } -func (m *QueryFeatures_Request) GetEntityId() []string { +func (m *QueryFeaturesRequest) GetEntityId() []string { if m != nil { return m.EntityId } return nil } -func (m *QueryFeatures_Request) GetRequestDetails() []*RequestDetail { +func (m *QueryFeaturesRequest) GetFeatureId() []string { if m != nil { - return m.RequestDetails + return m.FeatureId } return nil } -func (m *QueryFeatures_Request) GetTimestampRange() *TimestampRange { +func (m *QueryFeaturesRequest) GetTimeRange() *TimestampRange { if m != nil { - return m.TimestampRange + return m.TimeRange } return nil } -type QueryFeatures_Response struct { - // e.g. "driver", "customer", "city". +type QueryFeaturesResponse struct { + // Entity name of the response EntityName string `protobuf:"bytes,1,opt,name=entityName,proto3" json:"entityName,omitempty"` // map of entity ID and its entity's properties. Entities map[string]*Entity `protobuf:"bytes,2,rep,name=entities,proto3" json:"entities,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` @@ -156,108 +104,44 @@ type QueryFeatures_Response struct { XXX_sizecache int32 `json:"-"` } -func (m *QueryFeatures_Response) Reset() { *m = QueryFeatures_Response{} } -func (m *QueryFeatures_Response) String() string { return proto.CompactTextString(m) } -func (*QueryFeatures_Response) ProtoMessage() {} -func (*QueryFeatures_Response) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{0, 1} +func (m *QueryFeaturesResponse) Reset() { *m = QueryFeaturesResponse{} } +func (m *QueryFeaturesResponse) String() string { return proto.CompactTextString(m) } +func (*QueryFeaturesResponse) ProtoMessage() {} +func (*QueryFeaturesResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_7609de6de542e6f0, []int{1} } -func (m *QueryFeatures_Response) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_QueryFeatures_Response.Unmarshal(m, b) +func (m *QueryFeaturesResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_QueryFeaturesResponse.Unmarshal(m, b) } -func (m *QueryFeatures_Response) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_QueryFeatures_Response.Marshal(b, m, deterministic) +func (m *QueryFeaturesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_QueryFeaturesResponse.Marshal(b, m, deterministic) } -func (dst *QueryFeatures_Response) XXX_Merge(src proto.Message) { - xxx_messageInfo_QueryFeatures_Response.Merge(dst, src) +func (m *QueryFeaturesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryFeaturesResponse.Merge(m, src) } -func (m *QueryFeatures_Response) XXX_Size() int { - return xxx_messageInfo_QueryFeatures_Response.Size(m) +func (m *QueryFeaturesResponse) XXX_Size() int { + return xxx_messageInfo_QueryFeaturesResponse.Size(m) } -func (m *QueryFeatures_Response) XXX_DiscardUnknown() { - xxx_messageInfo_QueryFeatures_Response.DiscardUnknown(m) +func (m *QueryFeaturesResponse) XXX_DiscardUnknown() { + xxx_messageInfo_QueryFeaturesResponse.DiscardUnknown(m) } -var xxx_messageInfo_QueryFeatures_Response proto.InternalMessageInfo +var xxx_messageInfo_QueryFeaturesResponse proto.InternalMessageInfo -func (m *QueryFeatures_Response) GetEntityName() string { +func (m *QueryFeaturesResponse) GetEntityName() string { if m != nil { return m.EntityName } return "" } -func (m *QueryFeatures_Response) GetEntities() map[string]*Entity { +func (m *QueryFeaturesResponse) GetEntities() map[string]*Entity { if m != nil { return m.Entities } return nil } -type RequestDetail struct { - // feature ID to be included in the query. - // feature ID is in the form of [entity_name].[granularity].[feature_name] - // e.g: "driver.day.total_accepted_booking" - // all requested feature ID shall have same entity name. - FeatureId string `protobuf:"bytes,1,opt,name=featureId,proto3" json:"featureId,omitempty"` - // request type either LAST or LIST. - // LAST : return only the latest value based on timestamp. - // LIST : return list of historical data sorted by timestamp. - Type RequestType `protobuf:"varint,2,opt,name=type,proto3,enum=feast.serving.RequestType" json:"type,omitempty"` - // only applicable to LIST. - // length of the returned list <= limit. - // default = 0 - Limit int32 `protobuf:"varint,3,opt,name=limit,proto3" json:"limit,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *RequestDetail) Reset() { *m = RequestDetail{} } -func (m *RequestDetail) String() string { return proto.CompactTextString(m) } -func (*RequestDetail) ProtoMessage() {} -func (*RequestDetail) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{1} -} -func (m *RequestDetail) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_RequestDetail.Unmarshal(m, b) -} -func (m *RequestDetail) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_RequestDetail.Marshal(b, m, deterministic) -} -func (dst *RequestDetail) XXX_Merge(src proto.Message) { - xxx_messageInfo_RequestDetail.Merge(dst, src) -} -func (m *RequestDetail) XXX_Size() int { - return xxx_messageInfo_RequestDetail.Size(m) -} -func (m *RequestDetail) XXX_DiscardUnknown() { - xxx_messageInfo_RequestDetail.DiscardUnknown(m) -} - -var xxx_messageInfo_RequestDetail proto.InternalMessageInfo - -func (m *RequestDetail) GetFeatureId() string { - if m != nil { - return m.FeatureId - } - return "" -} - -func (m *RequestDetail) GetType() RequestType { - if m != nil { - return m.Type - } - return RequestType_LAST -} - -func (m *RequestDetail) GetLimit() int32 { - if m != nil { - return m.Limit - } - return 0 -} - // range of timestamp for querying // valid timestamp range is having start <= end type TimestampRange struct { @@ -274,7 +158,7 @@ func (m *TimestampRange) Reset() { *m = TimestampRange{} } func (m *TimestampRange) String() string { return proto.CompactTextString(m) } func (*TimestampRange) ProtoMessage() {} func (*TimestampRange) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{2} + return fileDescriptor_7609de6de542e6f0, []int{2} } func (m *TimestampRange) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TimestampRange.Unmarshal(m, b) @@ -282,8 +166,8 @@ func (m *TimestampRange) XXX_Unmarshal(b []byte) error { func (m *TimestampRange) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_TimestampRange.Marshal(b, m, deterministic) } -func (dst *TimestampRange) XXX_Merge(src proto.Message) { - xxx_messageInfo_TimestampRange.Merge(dst, src) +func (m *TimestampRange) XXX_Merge(src proto.Message) { + xxx_messageInfo_TimestampRange.Merge(m, src) } func (m *TimestampRange) XXX_Size() int { return xxx_messageInfo_TimestampRange.Size(m) @@ -310,17 +194,17 @@ func (m *TimestampRange) GetEnd() *timestamp.Timestamp { type Entity struct { // map of feature ID and its feature value. - Features map[string]*FeatureValueList `protobuf:"bytes,1,rep,name=features,proto3" json:"features,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Features map[string]*FeatureValue `protobuf:"bytes,1,rep,name=features,proto3" json:"features,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *Entity) Reset() { *m = Entity{} } func (m *Entity) String() string { return proto.CompactTextString(m) } func (*Entity) ProtoMessage() {} func (*Entity) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{3} + return fileDescriptor_7609de6de542e6f0, []int{3} } func (m *Entity) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Entity.Unmarshal(m, b) @@ -328,8 +212,8 @@ func (m *Entity) XXX_Unmarshal(b []byte) error { func (m *Entity) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Entity.Marshal(b, m, deterministic) } -func (dst *Entity) XXX_Merge(src proto.Message) { - xxx_messageInfo_Entity.Merge(dst, src) +func (m *Entity) XXX_Merge(src proto.Message) { + xxx_messageInfo_Entity.Merge(m, src) } func (m *Entity) XXX_Size() int { return xxx_messageInfo_Entity.Size(m) @@ -340,75 +224,69 @@ func (m *Entity) XXX_DiscardUnknown() { var xxx_messageInfo_Entity proto.InternalMessageInfo -func (m *Entity) GetFeatures() map[string]*FeatureValueList { +func (m *Entity) GetFeatures() map[string]*FeatureValue { if m != nil { return m.Features } return nil } -type FeatureValueList struct { - // list of feature value - // if "type" in "requestDetail" is "LAST", then the length will always be 1. - // if "type" in "requestDetail" is "LIST", then the length is <= "limit". - ValueList *types.ValueList `protobuf:"bytes,1,opt,name=valueList,proto3" json:"valueList,omitempty"` - // list of timestamp of the value. - // the i-th timestamps correspond to the i-th value. - TimestampList *types.TimestampList `protobuf:"bytes,2,opt,name=timestampList,proto3" json:"timestampList,omitempty"` +type FeatureValue struct { + // value of feature + Value *types.Value `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` + // timestamp of the feature + Timestamp *timestamp.Timestamp `protobuf:"bytes,2,opt,name=timestamp,proto3" json:"timestamp,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } -func (m *FeatureValueList) Reset() { *m = FeatureValueList{} } -func (m *FeatureValueList) String() string { return proto.CompactTextString(m) } -func (*FeatureValueList) ProtoMessage() {} -func (*FeatureValueList) Descriptor() ([]byte, []int) { - return fileDescriptor_Serving_3801c4d5794dbe13, []int{4} +func (m *FeatureValue) Reset() { *m = FeatureValue{} } +func (m *FeatureValue) String() string { return proto.CompactTextString(m) } +func (*FeatureValue) ProtoMessage() {} +func (*FeatureValue) Descriptor() ([]byte, []int) { + return fileDescriptor_7609de6de542e6f0, []int{4} } -func (m *FeatureValueList) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_FeatureValueList.Unmarshal(m, b) +func (m *FeatureValue) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_FeatureValue.Unmarshal(m, b) } -func (m *FeatureValueList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_FeatureValueList.Marshal(b, m, deterministic) +func (m *FeatureValue) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_FeatureValue.Marshal(b, m, deterministic) } -func (dst *FeatureValueList) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureValueList.Merge(dst, src) +func (m *FeatureValue) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureValue.Merge(m, src) } -func (m *FeatureValueList) XXX_Size() int { - return xxx_messageInfo_FeatureValueList.Size(m) +func (m *FeatureValue) XXX_Size() int { + return xxx_messageInfo_FeatureValue.Size(m) } -func (m *FeatureValueList) XXX_DiscardUnknown() { - xxx_messageInfo_FeatureValueList.DiscardUnknown(m) +func (m *FeatureValue) XXX_DiscardUnknown() { + xxx_messageInfo_FeatureValue.DiscardUnknown(m) } -var xxx_messageInfo_FeatureValueList proto.InternalMessageInfo +var xxx_messageInfo_FeatureValue proto.InternalMessageInfo -func (m *FeatureValueList) GetValueList() *types.ValueList { +func (m *FeatureValue) GetValue() *types.Value { if m != nil { - return m.ValueList + return m.Value } return nil } -func (m *FeatureValueList) GetTimestampList() *types.TimestampList { +func (m *FeatureValue) GetTimestamp() *timestamp.Timestamp { if m != nil { - return m.TimestampList + return m.Timestamp } return nil } func init() { - proto.RegisterType((*QueryFeatures)(nil), "feast.serving.QueryFeatures") - proto.RegisterType((*QueryFeatures_Request)(nil), "feast.serving.QueryFeatures.Request") - proto.RegisterType((*QueryFeatures_Response)(nil), "feast.serving.QueryFeatures.Response") - proto.RegisterMapType((map[string]*Entity)(nil), "feast.serving.QueryFeatures.Response.EntitiesEntry") - proto.RegisterType((*RequestDetail)(nil), "feast.serving.RequestDetail") + proto.RegisterType((*QueryFeaturesRequest)(nil), "feast.serving.QueryFeaturesRequest") + proto.RegisterType((*QueryFeaturesResponse)(nil), "feast.serving.QueryFeaturesResponse") + proto.RegisterMapType((map[string]*Entity)(nil), "feast.serving.QueryFeaturesResponse.EntitiesEntry") proto.RegisterType((*TimestampRange)(nil), "feast.serving.TimestampRange") proto.RegisterType((*Entity)(nil), "feast.serving.Entity") - proto.RegisterMapType((map[string]*FeatureValueList)(nil), "feast.serving.Entity.FeaturesEntry") - proto.RegisterType((*FeatureValueList)(nil), "feast.serving.FeatureValueList") - proto.RegisterEnum("feast.serving.RequestType", RequestType_name, RequestType_value) + proto.RegisterMapType((map[string]*FeatureValue)(nil), "feast.serving.Entity.FeaturesEntry") + proto.RegisterType((*FeatureValue)(nil), "feast.serving.FeatureValue") } // Reference imports to suppress errors if they are not otherwise used. @@ -423,8 +301,8 @@ const _ = grpc.SupportPackageIsVersion4 // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type ServingAPIClient interface { - // Query features from Feast - QueryFeatures(ctx context.Context, in *QueryFeatures_Request, opts ...grpc.CallOption) (*QueryFeatures_Response, error) + // Query features from Feast serving storage + QueryFeatures(ctx context.Context, in *QueryFeaturesRequest, opts ...grpc.CallOption) (*QueryFeaturesResponse, error) } type servingAPIClient struct { @@ -435,8 +313,8 @@ func NewServingAPIClient(cc *grpc.ClientConn) ServingAPIClient { return &servingAPIClient{cc} } -func (c *servingAPIClient) QueryFeatures(ctx context.Context, in *QueryFeatures_Request, opts ...grpc.CallOption) (*QueryFeatures_Response, error) { - out := new(QueryFeatures_Response) +func (c *servingAPIClient) QueryFeatures(ctx context.Context, in *QueryFeaturesRequest, opts ...grpc.CallOption) (*QueryFeaturesResponse, error) { + out := new(QueryFeaturesResponse) err := c.cc.Invoke(ctx, "/feast.serving.ServingAPI/QueryFeatures", in, out, opts...) if err != nil { return nil, err @@ -446,8 +324,8 @@ func (c *servingAPIClient) QueryFeatures(ctx context.Context, in *QueryFeatures_ // ServingAPIServer is the server API for ServingAPI service. type ServingAPIServer interface { - // Query features from Feast - QueryFeatures(context.Context, *QueryFeatures_Request) (*QueryFeatures_Response, error) + // Query features from Feast serving storage + QueryFeatures(context.Context, *QueryFeaturesRequest) (*QueryFeaturesResponse, error) } func RegisterServingAPIServer(s *grpc.Server, srv ServingAPIServer) { @@ -455,7 +333,7 @@ func RegisterServingAPIServer(s *grpc.Server, srv ServingAPIServer) { } func _ServingAPI_QueryFeatures_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(QueryFeatures_Request) + in := new(QueryFeaturesRequest) if err := dec(in); err != nil { return nil, err } @@ -467,7 +345,7 @@ func _ServingAPI_QueryFeatures_Handler(srv interface{}, ctx context.Context, dec FullMethod: "/feast.serving.ServingAPI/QueryFeatures", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(ServingAPIServer).QueryFeatures(ctx, req.(*QueryFeatures_Request)) + return srv.(ServingAPIServer).QueryFeatures(ctx, req.(*QueryFeaturesRequest)) } return interceptor(ctx, in, info, handler) } @@ -485,47 +363,39 @@ var _ServingAPI_serviceDesc = grpc.ServiceDesc{ Metadata: "feast/serving/Serving.proto", } -func init() { - proto.RegisterFile("feast/serving/Serving.proto", fileDescriptor_Serving_3801c4d5794dbe13) -} - -var fileDescriptor_Serving_3801c4d5794dbe13 = []byte{ - // 591 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xdd, 0x6e, 0xd3, 0x4c, - 0x10, 0xad, 0xeb, 0xa6, 0x5f, 0x32, 0x91, 0xf3, 0x45, 0x2b, 0x7e, 0x2c, 0x53, 0x68, 0x08, 0x20, - 0x45, 0x80, 0xd6, 0xc8, 0x05, 0xa9, 0xe2, 0x06, 0x5a, 0xb5, 0x48, 0x91, 0x2a, 0x28, 0xdb, 0x88, - 0x8b, 0x0a, 0x21, 0x39, 0x64, 0x62, 0x4c, 0x13, 0xdb, 0x78, 0xd7, 0x91, 0xfc, 0x0a, 0xbc, 0x08, - 0x6f, 0xc0, 0x6b, 0x20, 0xf1, 0x44, 0x28, 0xbb, 0xeb, 0xc4, 0xb6, 0x22, 0xca, 0x95, 0xbd, 0x9e, - 0x73, 0xce, 0x78, 0xe6, 0xcc, 0x0e, 0xdc, 0x99, 0xa2, 0xcf, 0x85, 0xcb, 0x31, 0x5d, 0x84, 0x51, - 0xe0, 0x5e, 0xa8, 0x27, 0x4d, 0xd2, 0x58, 0xc4, 0xc4, 0x92, 0x41, 0xaa, 0x83, 0xce, 0x7e, 0x10, - 0xc7, 0xc1, 0x0c, 0x5d, 0x19, 0x1c, 0x67, 0x53, 0x57, 0x84, 0x73, 0xe4, 0xc2, 0x9f, 0x27, 0x0a, - 0xef, 0xdc, 0x56, 0x62, 0x22, 0x4f, 0x90, 0xbb, 0x1f, 0xfc, 0x59, 0x86, 0x2a, 0xd0, 0xff, 0x69, - 0x82, 0xf5, 0x3e, 0xc3, 0x34, 0x7f, 0x83, 0xbe, 0xc8, 0x52, 0xe4, 0xce, 0x6f, 0x03, 0xfe, 0x63, - 0xf8, 0x2d, 0x43, 0x2e, 0xc8, 0x3d, 0x00, 0x8c, 0x44, 0x28, 0xf2, 0xb7, 0xfe, 0x1c, 0x6d, 0xa3, - 0x67, 0x0c, 0x5a, 0xac, 0xf4, 0x85, 0x38, 0xd0, 0x54, 0xa7, 0xe1, 0xc4, 0xde, 0xee, 0x99, 0x83, - 0x16, 0x5b, 0x9d, 0xc9, 0x09, 0x74, 0x52, 0x25, 0x73, 0x82, 0xc2, 0x0f, 0x67, 0xdc, 0x36, 0x7b, - 0xe6, 0xa0, 0xed, 0xed, 0xd1, 0xca, 0xbf, 0x53, 0x56, 0x06, 0xb1, 0x1a, 0x87, 0x9c, 0x42, 0x67, - 0x55, 0x0b, 0xf3, 0xa3, 0x00, 0xed, 0x9d, 0x9e, 0x31, 0x68, 0x7b, 0x77, 0x6b, 0x2a, 0xa3, 0x0a, - 0x88, 0xd5, 0x48, 0xce, 0x2f, 0x03, 0x9a, 0x0c, 0x79, 0x12, 0x47, 0x1c, 0xaf, 0xad, 0xea, 0x9d, - 0xae, 0x2a, 0x44, 0x2e, 0xab, 0x6a, 0x7b, 0x07, 0xb5, 0x6c, 0x95, 0x8e, 0xd1, 0x42, 0x98, 0x9e, - 0x6a, 0xd6, 0x69, 0x24, 0xd2, 0x9c, 0xad, 0x44, 0x1c, 0x06, 0x56, 0x25, 0x44, 0xba, 0x60, 0x5e, - 0x61, 0xae, 0x53, 0x2f, 0x5f, 0xc9, 0x13, 0x68, 0x2c, 0x96, 0xb6, 0xd8, 0xdb, 0xb2, 0xbc, 0x9b, - 0xb5, 0x84, 0x92, 0x9e, 0x33, 0x85, 0x79, 0xb9, 0x7d, 0x68, 0xf4, 0x39, 0x58, 0x95, 0xce, 0x91, - 0x3d, 0x68, 0x4d, 0xd5, 0x1f, 0x0d, 0x27, 0x5a, 0x79, 0xfd, 0x81, 0x50, 0xd8, 0x59, 0x9a, 0x2f, - 0xe5, 0x3b, 0x9e, 0xb3, 0xd9, 0x83, 0x51, 0x9e, 0x20, 0x93, 0x38, 0x72, 0x03, 0x1a, 0xb3, 0x70, - 0x1e, 0x0a, 0xdb, 0xec, 0x19, 0x83, 0x06, 0x53, 0x87, 0x7e, 0x02, 0x9d, 0x6a, 0xa3, 0xc9, 0x33, - 0x68, 0x70, 0xe1, 0xa7, 0x42, 0x66, 0x6c, 0x7b, 0x0e, 0x55, 0x93, 0x48, 0x8b, 0x49, 0x2c, 0x19, - 0xa3, 0x80, 0xe4, 0x29, 0x98, 0x18, 0x4d, 0x74, 0x9d, 0x7f, 0xc3, 0x2f, 0x61, 0xfd, 0x1f, 0x06, - 0xec, 0xaa, 0xe2, 0xc9, 0x2b, 0x68, 0xea, 0x7a, 0xb8, 0x6d, 0x48, 0x5b, 0x1e, 0x6c, 0xec, 0x12, - 0x2d, 0x8c, 0xd1, 0x36, 0x14, 0x24, 0xe7, 0x23, 0x58, 0x95, 0xd0, 0x06, 0x1b, 0x5e, 0x54, 0x6d, - 0xd8, 0xaf, 0x25, 0xd0, 0x74, 0x79, 0x81, 0xce, 0x42, 0x2e, 0xca, 0x86, 0x7c, 0x37, 0xa0, 0x5b, - 0x8f, 0x93, 0xe7, 0xd0, 0x5a, 0x14, 0x07, 0xdd, 0xa2, 0x5b, 0x5a, 0x53, 0xde, 0x45, 0xba, 0x96, - 0x5a, 0x03, 0xc9, 0x6b, 0xb0, 0x56, 0xf3, 0x2b, 0x99, 0x45, 0xb3, 0xca, 0xcc, 0x51, 0x19, 0xc1, - 0xaa, 0x84, 0xc7, 0xf7, 0xa1, 0x5d, 0xf2, 0x94, 0x34, 0x61, 0xe7, 0xec, 0xe8, 0x62, 0xd4, 0xdd, - 0x92, 0x6f, 0xc3, 0x8b, 0x51, 0xd7, 0xf0, 0x66, 0x00, 0x7a, 0xa7, 0x1c, 0x9d, 0x0f, 0xc9, 0xa7, - 0xda, 0x1a, 0x20, 0x0f, 0xaf, 0x19, 0x79, 0x29, 0xee, 0x3c, 0xfa, 0xa7, 0x8b, 0xd1, 0xdf, 0x3a, - 0xbe, 0x84, 0xea, 0xca, 0x3a, 0xfe, 0x7f, 0x9d, 0xfc, 0x7c, 0xe9, 0xfd, 0xe5, 0x61, 0x10, 0x8a, - 0x2f, 0xd9, 0x98, 0x7e, 0x8e, 0xe7, 0x6e, 0x10, 0x7f, 0xc5, 0x2b, 0x57, 0xed, 0x2c, 0x39, 0x19, - 0xdc, 0x0d, 0x30, 0xc2, 0xd4, 0x17, 0x38, 0x71, 0x83, 0xd8, 0xad, 0xac, 0xc6, 0xf1, 0xae, 0x84, - 0x1c, 0xfc, 0x09, 0x00, 0x00, 0xff, 0xff, 0x17, 0x94, 0x4b, 0x84, 0x32, 0x05, 0x00, 0x00, +func init() { proto.RegisterFile("feast/serving/Serving.proto", fileDescriptor_7609de6de542e6f0) } + +var fileDescriptor_7609de6de542e6f0 = []byte{ + // 486 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4d, 0x6f, 0xd3, 0x40, + 0x10, 0xc5, 0x09, 0xad, 0x9a, 0x09, 0x01, 0xb4, 0xa2, 0xc2, 0x72, 0xf9, 0x88, 0x5c, 0x0e, 0x91, + 0x40, 0xbb, 0x60, 0x2e, 0x11, 0x1c, 0x10, 0x95, 0x8a, 0x94, 0x4b, 0x55, 0x0c, 0x42, 0xa8, 0xe2, + 0xe2, 0x90, 0x89, 0x71, 0xdb, 0x78, 0xcd, 0xee, 0xba, 0x92, 0x7f, 0x0f, 0x67, 0x7e, 0x11, 0x7f, + 0x06, 0x79, 0x77, 0xdd, 0x78, 0xad, 0xa8, 0xcd, 0x29, 0xd9, 0x7d, 0x6f, 0x66, 0xde, 0xbc, 0x9d, + 0x31, 0x1c, 0x2c, 0x31, 0x91, 0x8a, 0x49, 0x14, 0x57, 0x59, 0x9e, 0xb2, 0x2f, 0xe6, 0x97, 0x16, + 0x82, 0x2b, 0x4e, 0x46, 0x1a, 0xa4, 0x16, 0x0c, 0x9e, 0xa7, 0x9c, 0xa7, 0x97, 0xc8, 0x34, 0x38, + 0x2f, 0x97, 0x4c, 0x65, 0x2b, 0x94, 0x2a, 0x59, 0x15, 0x86, 0x1f, 0x3c, 0x36, 0xc9, 0x54, 0x55, + 0xa0, 0x64, 0xdf, 0x92, 0xcb, 0x12, 0x0d, 0x10, 0xfe, 0xf5, 0xe0, 0xd1, 0xe7, 0x12, 0x45, 0xf5, + 0x09, 0x13, 0x55, 0x0a, 0x94, 0x31, 0xfe, 0x2e, 0x51, 0x2a, 0xf2, 0x0c, 0x00, 0x73, 0x95, 0xa9, + 0xea, 0x24, 0x59, 0xa1, 0xef, 0x8d, 0xbd, 0xc9, 0x20, 0x6e, 0xdd, 0x90, 0x00, 0xf6, 0xcc, 0x69, + 0xb6, 0xf0, 0x7b, 0xe3, 0xfe, 0x64, 0x10, 0x5f, 0x9f, 0xc9, 0x13, 0x18, 0x2c, 0x4d, 0xba, 0xd9, + 0xc2, 0xef, 0x6b, 0x70, 0x7d, 0x41, 0xde, 0xc3, 0xa0, 0x96, 0x17, 0x27, 0x79, 0x8a, 0xfe, 0xdd, + 0xb1, 0x37, 0x19, 0x46, 0x4f, 0xa9, 0xd3, 0x0f, 0xfd, 0xda, 0xc8, 0xd7, 0xa4, 0x78, 0xcd, 0x0f, + 0xff, 0x79, 0xb0, 0xdf, 0xd1, 0x2b, 0x0b, 0x9e, 0x4b, 0xbc, 0x55, 0xf0, 0x89, 0x15, 0x9c, 0xa1, + 0xd4, 0x82, 0x87, 0x51, 0xd4, 0xa9, 0xba, 0x31, 0x2f, 0x3d, 0xb6, 0x41, 0xc7, 0xb9, 0x12, 0x55, + 0x7c, 0x9d, 0x23, 0x88, 0x61, 0xe4, 0x40, 0xe4, 0x21, 0xf4, 0x2f, 0xb0, 0xb2, 0x95, 0xeb, 0xbf, + 0xe4, 0x25, 0xec, 0x5c, 0xd5, 0x5e, 0xfb, 0x3d, 0xdd, 0xe5, 0x7e, 0xa7, 0x9e, 0x0e, 0xaf, 0x62, + 0xc3, 0x79, 0xd7, 0x9b, 0x7a, 0x61, 0x01, 0xf7, 0xdd, 0xd6, 0xc9, 0x6b, 0xd8, 0x91, 0x2a, 0x11, + 0x4a, 0xa7, 0x1d, 0x46, 0x01, 0x35, 0x2f, 0x4d, 0x9b, 0x97, 0x6e, 0x59, 0x65, 0x88, 0xe4, 0x15, + 0xf4, 0x31, 0x5f, 0xd8, 0x92, 0x37, 0xf1, 0x6b, 0x5a, 0xf8, 0xc7, 0x83, 0x5d, 0xa3, 0x83, 0x7c, + 0x80, 0x3d, 0xfb, 0x48, 0xd2, 0xf7, 0xb4, 0x41, 0x87, 0x1b, 0x05, 0xd3, 0xc6, 0x22, 0xeb, 0x48, + 0x13, 0x14, 0x7c, 0x87, 0x91, 0x03, 0x6d, 0x70, 0xe4, 0x8d, 0xeb, 0xc8, 0x41, 0xa7, 0x80, 0x0d, + 0xd7, 0x03, 0xda, 0xf6, 0x45, 0xc0, 0xbd, 0x36, 0x44, 0x26, 0x4d, 0x1a, 0xe3, 0x0a, 0xb1, 0x69, + 0xf4, 0x78, 0xd3, 0x76, 0x34, 0x99, 0x9a, 0x61, 0xd3, 0x1d, 0x6f, 0xe1, 0xc9, 0x9a, 0x1c, 0x9d, + 0x03, 0xd8, 0x9d, 0xfb, 0x78, 0x3a, 0x23, 0x3f, 0x60, 0xe4, 0x8c, 0x07, 0x39, 0xbc, 0x79, 0x78, + 0xf4, 0x12, 0x05, 0x2f, 0xb6, 0x99, 0xb0, 0xf0, 0xce, 0xd1, 0x19, 0xb8, 0x0b, 0x7d, 0xf4, 0x60, + 0x5d, 0xfa, 0xb4, 0x56, 0x79, 0x36, 0x4d, 0x33, 0xf5, 0xab, 0x9c, 0xd3, 0x9f, 0x7c, 0xc5, 0x52, + 0x7e, 0x8e, 0x17, 0xcc, 0x6c, 0xb4, 0xee, 0x41, 0xb2, 0x14, 0x73, 0x14, 0x89, 0xc2, 0x05, 0x4b, + 0x39, 0x73, 0x3e, 0x1c, 0xf3, 0x5d, 0x4d, 0x79, 0xfb, 0x3f, 0x00, 0x00, 0xff, 0xff, 0x14, 0x04, + 0x54, 0x19, 0x50, 0x04, 0x00, 0x00, } diff --git a/protos/generated/go/feast/specs/EntitySpec.pb.go b/protos/generated/go/feast/specs/EntitySpec.pb.go index 0f4374c679..6b2bbca109 100644 --- a/protos/generated/go/feast/specs/EntitySpec.pb.go +++ b/protos/generated/go/feast/specs/EntitySpec.pb.go @@ -31,7 +31,7 @@ func (m *EntitySpec) Reset() { *m = EntitySpec{} } func (m *EntitySpec) String() string { return proto.CompactTextString(m) } func (*EntitySpec) ProtoMessage() {} func (*EntitySpec) Descriptor() ([]byte, []int) { - return fileDescriptor_EntitySpec_b8950ded39b854cb, []int{0} + return fileDescriptor_a3230229c2278d5e, []int{0} } func (m *EntitySpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_EntitySpec.Unmarshal(m, b) @@ -39,8 +39,8 @@ func (m *EntitySpec) XXX_Unmarshal(b []byte) error { func (m *EntitySpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_EntitySpec.Marshal(b, m, deterministic) } -func (dst *EntitySpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_EntitySpec.Merge(dst, src) +func (m *EntitySpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_EntitySpec.Merge(m, src) } func (m *EntitySpec) XXX_Size() int { return xxx_messageInfo_EntitySpec.Size(m) @@ -76,11 +76,9 @@ func init() { proto.RegisterType((*EntitySpec)(nil), "feast.specs.EntitySpec") } -func init() { - proto.RegisterFile("feast/specs/EntitySpec.proto", fileDescriptor_EntitySpec_b8950ded39b854cb) -} +func init() { proto.RegisterFile("feast/specs/EntitySpec.proto", fileDescriptor_a3230229c2278d5e) } -var fileDescriptor_EntitySpec_b8950ded39b854cb = []byte{ +var fileDescriptor_a3230229c2278d5e = []byte{ // 177 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x49, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0x2e, 0x48, 0x4d, 0x2e, 0xd6, 0x77, 0xcd, 0x2b, 0xc9, 0x2c, 0xa9, 0x0c, 0x2e, diff --git a/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go b/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go index 8349e5a53a..43bb2bb36d 100644 --- a/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go +++ b/protos/generated/go/feast/specs/FeatureGroupSpec.pb.go @@ -31,7 +31,7 @@ func (m *FeatureGroupSpec) Reset() { *m = FeatureGroupSpec{} } func (m *FeatureGroupSpec) String() string { return proto.CompactTextString(m) } func (*FeatureGroupSpec) ProtoMessage() {} func (*FeatureGroupSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureGroupSpec_21c67ee01edd412c, []int{0} + return fileDescriptor_df3b6a9e736b5719, []int{0} } func (m *FeatureGroupSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureGroupSpec.Unmarshal(m, b) @@ -39,8 +39,8 @@ func (m *FeatureGroupSpec) XXX_Unmarshal(b []byte) error { func (m *FeatureGroupSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureGroupSpec.Marshal(b, m, deterministic) } -func (dst *FeatureGroupSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureGroupSpec.Merge(dst, src) +func (m *FeatureGroupSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureGroupSpec.Merge(m, src) } func (m *FeatureGroupSpec) XXX_Size() int { return xxx_messageInfo_FeatureGroupSpec.Size(m) @@ -76,11 +76,9 @@ func init() { proto.RegisterType((*FeatureGroupSpec)(nil), "feast.specs.FeatureGroupSpec") } -func init() { - proto.RegisterFile("feast/specs/FeatureGroupSpec.proto", fileDescriptor_FeatureGroupSpec_21c67ee01edd412c) -} +func init() { proto.RegisterFile("feast/specs/FeatureGroupSpec.proto", fileDescriptor_df3b6a9e736b5719) } -var fileDescriptor_FeatureGroupSpec_21c67ee01edd412c = []byte{ +var fileDescriptor_df3b6a9e736b5719 = []byte{ // 203 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x52, 0x4a, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0x2e, 0x48, 0x4d, 0x2e, 0xd6, 0x77, 0x4b, 0x4d, 0x2c, 0x29, 0x2d, 0x4a, 0x75, diff --git a/protos/generated/go/feast/specs/FeatureSpec.pb.go b/protos/generated/go/feast/specs/FeatureSpec.pb.go index 4b8a3d9505..35e4c5adcb 100644 --- a/protos/generated/go/feast/specs/FeatureSpec.pb.go +++ b/protos/generated/go/feast/specs/FeatureSpec.pb.go @@ -41,7 +41,7 @@ func (m *FeatureSpec) Reset() { *m = FeatureSpec{} } func (m *FeatureSpec) String() string { return proto.CompactTextString(m) } func (*FeatureSpec) ProtoMessage() {} func (*FeatureSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureSpec_eccd5917f38e2b1f, []int{0} + return fileDescriptor_b8f1468f11147fbe, []int{0} } func (m *FeatureSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureSpec.Unmarshal(m, b) @@ -49,8 +49,8 @@ func (m *FeatureSpec) XXX_Unmarshal(b []byte) error { func (m *FeatureSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureSpec.Marshal(b, m, deterministic) } -func (dst *FeatureSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureSpec.Merge(dst, src) +func (m *FeatureSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureSpec.Merge(m, src) } func (m *FeatureSpec) XXX_Size() int { return xxx_messageInfo_FeatureSpec.Size(m) @@ -157,7 +157,7 @@ func (m *DataStores) Reset() { *m = DataStores{} } func (m *DataStores) String() string { return proto.CompactTextString(m) } func (*DataStores) ProtoMessage() {} func (*DataStores) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureSpec_eccd5917f38e2b1f, []int{1} + return fileDescriptor_b8f1468f11147fbe, []int{1} } func (m *DataStores) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DataStores.Unmarshal(m, b) @@ -165,8 +165,8 @@ func (m *DataStores) XXX_Unmarshal(b []byte) error { func (m *DataStores) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DataStores.Marshal(b, m, deterministic) } -func (dst *DataStores) XXX_Merge(src proto.Message) { - xxx_messageInfo_DataStores.Merge(dst, src) +func (m *DataStores) XXX_Merge(src proto.Message) { + xxx_messageInfo_DataStores.Merge(m, src) } func (m *DataStores) XXX_Size() int { return xxx_messageInfo_DataStores.Size(m) @@ -203,7 +203,7 @@ func (m *DataStore) Reset() { *m = DataStore{} } func (m *DataStore) String() string { return proto.CompactTextString(m) } func (*DataStore) ProtoMessage() {} func (*DataStore) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureSpec_eccd5917f38e2b1f, []int{2} + return fileDescriptor_b8f1468f11147fbe, []int{2} } func (m *DataStore) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DataStore.Unmarshal(m, b) @@ -211,8 +211,8 @@ func (m *DataStore) XXX_Unmarshal(b []byte) error { func (m *DataStore) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DataStore.Marshal(b, m, deterministic) } -func (dst *DataStore) XXX_Merge(src proto.Message) { - xxx_messageInfo_DataStore.Merge(dst, src) +func (m *DataStore) XXX_Merge(src proto.Message) { + xxx_messageInfo_DataStore.Merge(m, src) } func (m *DataStore) XXX_Size() int { return xxx_messageInfo_DataStore.Size(m) @@ -245,11 +245,9 @@ func init() { proto.RegisterMapType((map[string]string)(nil), "feast.specs.DataStore.OptionsEntry") } -func init() { - proto.RegisterFile("feast/specs/FeatureSpec.proto", fileDescriptor_FeatureSpec_eccd5917f38e2b1f) -} +func init() { proto.RegisterFile("feast/specs/FeatureSpec.proto", fileDescriptor_b8f1468f11147fbe) } -var fileDescriptor_FeatureSpec_eccd5917f38e2b1f = []byte{ +var fileDescriptor_b8f1468f11147fbe = []byte{ // 479 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x53, 0x4d, 0x6b, 0xdb, 0x40, 0x10, 0x45, 0x56, 0x62, 0x57, 0xa3, 0x10, 0xc2, 0x52, 0x92, 0xc5, 0x6d, 0x40, 0xb8, 0x14, 0x7c, diff --git a/protos/generated/go/feast/specs/StorageSpec.pb.go b/protos/generated/go/feast/specs/StorageSpec.pb.go index fefe56a1be..924e1f154f 100644 --- a/protos/generated/go/feast/specs/StorageSpec.pb.go +++ b/protos/generated/go/feast/specs/StorageSpec.pb.go @@ -36,7 +36,7 @@ func (m *StorageSpec) Reset() { *m = StorageSpec{} } func (m *StorageSpec) String() string { return proto.CompactTextString(m) } func (*StorageSpec) ProtoMessage() {} func (*StorageSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_StorageSpec_bfb8a5e5cf34de95, []int{0} + return fileDescriptor_7783b72a0d689614, []int{0} } func (m *StorageSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StorageSpec.Unmarshal(m, b) @@ -44,8 +44,8 @@ func (m *StorageSpec) XXX_Unmarshal(b []byte) error { func (m *StorageSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_StorageSpec.Marshal(b, m, deterministic) } -func (dst *StorageSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_StorageSpec.Merge(dst, src) +func (m *StorageSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_StorageSpec.Merge(m, src) } func (m *StorageSpec) XXX_Size() int { return xxx_messageInfo_StorageSpec.Size(m) @@ -82,11 +82,9 @@ func init() { proto.RegisterMapType((map[string]string)(nil), "feast.specs.StorageSpec.OptionsEntry") } -func init() { - proto.RegisterFile("feast/specs/StorageSpec.proto", fileDescriptor_StorageSpec_bfb8a5e5cf34de95) -} +func init() { proto.RegisterFile("feast/specs/StorageSpec.proto", fileDescriptor_7783b72a0d689614) } -var fileDescriptor_StorageSpec_bfb8a5e5cf34de95 = []byte{ +var fileDescriptor_7783b72a0d689614 = []byte{ // 227 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4d, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0x2e, 0x48, 0x4d, 0x2e, 0xd6, 0x0f, 0x2e, 0xc9, 0x2f, 0x4a, 0x4c, 0x4f, 0x0d, diff --git a/protos/generated/go/feast/storage/Redis.pb.go b/protos/generated/go/feast/storage/Redis.pb.go index 49f80a061e..f42a1f618d 100644 --- a/protos/generated/go/feast/storage/Redis.pb.go +++ b/protos/generated/go/feast/storage/Redis.pb.go @@ -43,7 +43,7 @@ func (m *RedisBucketKey) Reset() { *m = RedisBucketKey{} } func (m *RedisBucketKey) String() string { return proto.CompactTextString(m) } func (*RedisBucketKey) ProtoMessage() {} func (*RedisBucketKey) Descriptor() ([]byte, []int) { - return fileDescriptor_Redis_cef62c817c1622ce, []int{0} + return fileDescriptor_64e898a359fc9e5d, []int{0} } func (m *RedisBucketKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RedisBucketKey.Unmarshal(m, b) @@ -51,8 +51,8 @@ func (m *RedisBucketKey) XXX_Unmarshal(b []byte) error { func (m *RedisBucketKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RedisBucketKey.Marshal(b, m, deterministic) } -func (dst *RedisBucketKey) XXX_Merge(src proto.Message) { - xxx_messageInfo_RedisBucketKey.Merge(dst, src) +func (m *RedisBucketKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_RedisBucketKey.Merge(m, src) } func (m *RedisBucketKey) XXX_Size() int { return xxx_messageInfo_RedisBucketKey.Size(m) @@ -99,7 +99,7 @@ func (m *RedisBucketValue) Reset() { *m = RedisBucketValue{} } func (m *RedisBucketValue) String() string { return proto.CompactTextString(m) } func (*RedisBucketValue) ProtoMessage() {} func (*RedisBucketValue) Descriptor() ([]byte, []int) { - return fileDescriptor_Redis_cef62c817c1622ce, []int{1} + return fileDescriptor_64e898a359fc9e5d, []int{1} } func (m *RedisBucketValue) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RedisBucketValue.Unmarshal(m, b) @@ -107,8 +107,8 @@ func (m *RedisBucketValue) XXX_Unmarshal(b []byte) error { func (m *RedisBucketValue) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RedisBucketValue.Marshal(b, m, deterministic) } -func (dst *RedisBucketValue) XXX_Merge(src proto.Message) { - xxx_messageInfo_RedisBucketValue.Merge(dst, src) +func (m *RedisBucketValue) XXX_Merge(src proto.Message) { + xxx_messageInfo_RedisBucketValue.Merge(m, src) } func (m *RedisBucketValue) XXX_Size() int { return xxx_messageInfo_RedisBucketValue.Size(m) @@ -147,7 +147,7 @@ func (m *RedisBucketValueList) Reset() { *m = RedisBucketValueList{} } func (m *RedisBucketValueList) String() string { return proto.CompactTextString(m) } func (*RedisBucketValueList) ProtoMessage() {} func (*RedisBucketValueList) Descriptor() ([]byte, []int) { - return fileDescriptor_Redis_cef62c817c1622ce, []int{2} + return fileDescriptor_64e898a359fc9e5d, []int{2} } func (m *RedisBucketValueList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_RedisBucketValueList.Unmarshal(m, b) @@ -155,8 +155,8 @@ func (m *RedisBucketValueList) XXX_Unmarshal(b []byte) error { func (m *RedisBucketValueList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_RedisBucketValueList.Marshal(b, m, deterministic) } -func (dst *RedisBucketValueList) XXX_Merge(src proto.Message) { - xxx_messageInfo_RedisBucketValueList.Merge(dst, src) +func (m *RedisBucketValueList) XXX_Merge(src proto.Message) { + xxx_messageInfo_RedisBucketValueList.Merge(m, src) } func (m *RedisBucketValueList) XXX_Size() int { return xxx_messageInfo_RedisBucketValueList.Size(m) @@ -180,9 +180,9 @@ func init() { proto.RegisterType((*RedisBucketValueList)(nil), "feast.storage.RedisBucketValueList") } -func init() { proto.RegisterFile("feast/storage/Redis.proto", fileDescriptor_Redis_cef62c817c1622ce) } +func init() { proto.RegisterFile("feast/storage/Redis.proto", fileDescriptor_64e898a359fc9e5d) } -var fileDescriptor_Redis_cef62c817c1622ce = []byte{ +var fileDescriptor_64e898a359fc9e5d = []byte{ // 325 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xcd, 0x4f, 0xf2, 0x40, 0x10, 0xc6, 0xd3, 0x97, 0x57, 0x22, 0x4b, 0x24, 0x66, 0x35, 0xb1, 0x36, 0x26, 0x34, 0x9c, 0x7a, diff --git a/protos/generated/go/feast/types/FeatureRowExtended.pb.go b/protos/generated/go/feast/types/FeatureRowExtended.pb.go index ad89f27db2..9aab57b189 100644 --- a/protos/generated/go/feast/types/FeatureRowExtended.pb.go +++ b/protos/generated/go/feast/types/FeatureRowExtended.pb.go @@ -33,7 +33,7 @@ func (m *Error) Reset() { *m = Error{} } func (m *Error) String() string { return proto.CompactTextString(m) } func (*Error) ProtoMessage() {} func (*Error) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRowExtended_bfd3c37956d1a040, []int{0} + return fileDescriptor_7823aa2c72575793, []int{0} } func (m *Error) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Error.Unmarshal(m, b) @@ -41,8 +41,8 @@ func (m *Error) XXX_Unmarshal(b []byte) error { func (m *Error) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Error.Marshal(b, m, deterministic) } -func (dst *Error) XXX_Merge(src proto.Message) { - xxx_messageInfo_Error.Merge(dst, src) +func (m *Error) XXX_Merge(src proto.Message) { + xxx_messageInfo_Error.Merge(m, src) } func (m *Error) XXX_Size() int { return xxx_messageInfo_Error.Size(m) @@ -93,7 +93,7 @@ func (m *Attempt) Reset() { *m = Attempt{} } func (m *Attempt) String() string { return proto.CompactTextString(m) } func (*Attempt) ProtoMessage() {} func (*Attempt) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRowExtended_bfd3c37956d1a040, []int{1} + return fileDescriptor_7823aa2c72575793, []int{1} } func (m *Attempt) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Attempt.Unmarshal(m, b) @@ -101,8 +101,8 @@ func (m *Attempt) XXX_Unmarshal(b []byte) error { func (m *Attempt) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Attempt.Marshal(b, m, deterministic) } -func (dst *Attempt) XXX_Merge(src proto.Message) { - xxx_messageInfo_Attempt.Merge(dst, src) +func (m *Attempt) XXX_Merge(src proto.Message) { + xxx_messageInfo_Attempt.Merge(m, src) } func (m *Attempt) XXX_Size() int { return xxx_messageInfo_Attempt.Size(m) @@ -140,7 +140,7 @@ func (m *FeatureRowExtended) Reset() { *m = FeatureRowExtended{} } func (m *FeatureRowExtended) String() string { return proto.CompactTextString(m) } func (*FeatureRowExtended) ProtoMessage() {} func (*FeatureRowExtended) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRowExtended_bfd3c37956d1a040, []int{2} + return fileDescriptor_7823aa2c72575793, []int{2} } func (m *FeatureRowExtended) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRowExtended.Unmarshal(m, b) @@ -148,8 +148,8 @@ func (m *FeatureRowExtended) XXX_Unmarshal(b []byte) error { func (m *FeatureRowExtended) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureRowExtended.Marshal(b, m, deterministic) } -func (dst *FeatureRowExtended) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureRowExtended.Merge(dst, src) +func (m *FeatureRowExtended) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureRowExtended.Merge(m, src) } func (m *FeatureRowExtended) XXX_Size() int { return xxx_messageInfo_FeatureRowExtended.Size(m) @@ -188,10 +188,10 @@ func init() { } func init() { - proto.RegisterFile("feast/types/FeatureRowExtended.proto", fileDescriptor_FeatureRowExtended_bfd3c37956d1a040) + proto.RegisterFile("feast/types/FeatureRowExtended.proto", fileDescriptor_7823aa2c72575793) } -var fileDescriptor_FeatureRowExtended_bfd3c37956d1a040 = []byte{ +var fileDescriptor_7823aa2c72575793 = []byte{ // 338 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0xc1, 0x6b, 0xea, 0x40, 0x10, 0xc6, 0xf1, 0xf9, 0xf2, 0x7c, 0x4e, 0x6e, 0x8b, 0x60, 0x08, 0xd2, 0x16, 0xe9, 0xc1, 0x5e, diff --git a/protos/generated/go/feast/types/Granularity.pb.go b/protos/generated/go/feast/types/Granularity.pb.go index 5166f7bbc2..99aae3d90f 100644 --- a/protos/generated/go/feast/types/Granularity.pb.go +++ b/protos/generated/go/feast/types/Granularity.pb.go @@ -35,6 +35,7 @@ var Granularity_Enum_name = map[int32]string{ 3: "MINUTE", 4: "SECOND", } + var Granularity_Enum_value = map[string]int32{ "NONE": 0, "DAY": 1, @@ -46,8 +47,9 @@ var Granularity_Enum_value = map[string]int32{ func (x Granularity_Enum) String() string { return proto.EnumName(Granularity_Enum_name, int32(x)) } + func (Granularity_Enum) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_Granularity_4b57756f7a751fdb, []int{0, 0} + return fileDescriptor_846024d9f96d5fa2, []int{0, 0} } type Granularity struct { @@ -60,7 +62,7 @@ func (m *Granularity) Reset() { *m = Granularity{} } func (m *Granularity) String() string { return proto.CompactTextString(m) } func (*Granularity) ProtoMessage() {} func (*Granularity) Descriptor() ([]byte, []int) { - return fileDescriptor_Granularity_4b57756f7a751fdb, []int{0} + return fileDescriptor_846024d9f96d5fa2, []int{0} } func (m *Granularity) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Granularity.Unmarshal(m, b) @@ -68,8 +70,8 @@ func (m *Granularity) XXX_Unmarshal(b []byte) error { func (m *Granularity) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Granularity.Marshal(b, m, deterministic) } -func (dst *Granularity) XXX_Merge(src proto.Message) { - xxx_messageInfo_Granularity.Merge(dst, src) +func (m *Granularity) XXX_Merge(src proto.Message) { + xxx_messageInfo_Granularity.Merge(m, src) } func (m *Granularity) XXX_Size() int { return xxx_messageInfo_Granularity.Size(m) @@ -85,11 +87,9 @@ func init() { proto.RegisterEnum("feast.types.Granularity_Enum", Granularity_Enum_name, Granularity_Enum_value) } -func init() { - proto.RegisterFile("feast/types/Granularity.proto", fileDescriptor_Granularity_4b57756f7a751fdb) -} +func init() { proto.RegisterFile("feast/types/Granularity.proto", fileDescriptor_846024d9f96d5fa2) } -var fileDescriptor_Granularity_4b57756f7a751fdb = []byte{ +var fileDescriptor_846024d9f96d5fa2 = []byte{ // 183 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4d, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x77, 0x2f, 0x4a, 0xcc, 0x2b, 0xcd, 0x49, 0x2c, diff --git a/protos/generated/go/feast/types/Value.pb.go b/protos/generated/go/feast/types/Value.pb.go index b9c26f8c4b..4b0121e326 100644 --- a/protos/generated/go/feast/types/Value.pb.go +++ b/protos/generated/go/feast/types/Value.pb.go @@ -44,6 +44,7 @@ var ValueType_Enum_name = map[int32]string{ 7: "BOOL", 8: "TIMESTAMP", } + var ValueType_Enum_value = map[string]int32{ "UNKNOWN": 0, "BYTES": 1, @@ -59,8 +60,9 @@ var ValueType_Enum_value = map[string]int32{ func (x ValueType_Enum) String() string { return proto.EnumName(ValueType_Enum_name, int32(x)) } + func (ValueType_Enum) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{0, 0} + return fileDescriptor_47c504407d284ecc, []int{0, 0} } type ValueType struct { @@ -73,7 +75,7 @@ func (m *ValueType) Reset() { *m = ValueType{} } func (m *ValueType) String() string { return proto.CompactTextString(m) } func (*ValueType) ProtoMessage() {} func (*ValueType) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{0} + return fileDescriptor_47c504407d284ecc, []int{0} } func (m *ValueType) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValueType.Unmarshal(m, b) @@ -81,8 +83,8 @@ func (m *ValueType) XXX_Unmarshal(b []byte) error { func (m *ValueType) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ValueType.Marshal(b, m, deterministic) } -func (dst *ValueType) XXX_Merge(src proto.Message) { - xxx_messageInfo_ValueType.Merge(dst, src) +func (m *ValueType) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValueType.Merge(m, src) } func (m *ValueType) XXX_Size() int { return xxx_messageInfo_ValueType.Size(m) @@ -113,7 +115,7 @@ func (m *Value) Reset() { *m = Value{} } func (m *Value) String() string { return proto.CompactTextString(m) } func (*Value) ProtoMessage() {} func (*Value) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{1} + return fileDescriptor_47c504407d284ecc, []int{1} } func (m *Value) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Value.Unmarshal(m, b) @@ -121,8 +123,8 @@ func (m *Value) XXX_Unmarshal(b []byte) error { func (m *Value) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Value.Marshal(b, m, deterministic) } -func (dst *Value) XXX_Merge(src proto.Message) { - xxx_messageInfo_Value.Merge(dst, src) +func (m *Value) XXX_Merge(src proto.Message) { + xxx_messageInfo_Value.Merge(m, src) } func (m *Value) XXX_Size() int { return xxx_messageInfo_Value.Size(m) @@ -427,7 +429,7 @@ func (m *ValueList) Reset() { *m = ValueList{} } func (m *ValueList) String() string { return proto.CompactTextString(m) } func (*ValueList) ProtoMessage() {} func (*ValueList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{2} + return fileDescriptor_47c504407d284ecc, []int{2} } func (m *ValueList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ValueList.Unmarshal(m, b) @@ -435,8 +437,8 @@ func (m *ValueList) XXX_Unmarshal(b []byte) error { func (m *ValueList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ValueList.Marshal(b, m, deterministic) } -func (dst *ValueList) XXX_Merge(src proto.Message) { - xxx_messageInfo_ValueList.Merge(dst, src) +func (m *ValueList) XXX_Merge(src proto.Message) { + xxx_messageInfo_ValueList.Merge(m, src) } func (m *ValueList) XXX_Size() int { return xxx_messageInfo_ValueList.Size(m) @@ -761,7 +763,7 @@ func (m *BytesList) Reset() { *m = BytesList{} } func (m *BytesList) String() string { return proto.CompactTextString(m) } func (*BytesList) ProtoMessage() {} func (*BytesList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{3} + return fileDescriptor_47c504407d284ecc, []int{3} } func (m *BytesList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BytesList.Unmarshal(m, b) @@ -769,8 +771,8 @@ func (m *BytesList) XXX_Unmarshal(b []byte) error { func (m *BytesList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BytesList.Marshal(b, m, deterministic) } -func (dst *BytesList) XXX_Merge(src proto.Message) { - xxx_messageInfo_BytesList.Merge(dst, src) +func (m *BytesList) XXX_Merge(src proto.Message) { + xxx_messageInfo_BytesList.Merge(m, src) } func (m *BytesList) XXX_Size() int { return xxx_messageInfo_BytesList.Size(m) @@ -799,7 +801,7 @@ func (m *StringList) Reset() { *m = StringList{} } func (m *StringList) String() string { return proto.CompactTextString(m) } func (*StringList) ProtoMessage() {} func (*StringList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{4} + return fileDescriptor_47c504407d284ecc, []int{4} } func (m *StringList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_StringList.Unmarshal(m, b) @@ -807,8 +809,8 @@ func (m *StringList) XXX_Unmarshal(b []byte) error { func (m *StringList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_StringList.Marshal(b, m, deterministic) } -func (dst *StringList) XXX_Merge(src proto.Message) { - xxx_messageInfo_StringList.Merge(dst, src) +func (m *StringList) XXX_Merge(src proto.Message) { + xxx_messageInfo_StringList.Merge(m, src) } func (m *StringList) XXX_Size() int { return xxx_messageInfo_StringList.Size(m) @@ -837,7 +839,7 @@ func (m *Int32List) Reset() { *m = Int32List{} } func (m *Int32List) String() string { return proto.CompactTextString(m) } func (*Int32List) ProtoMessage() {} func (*Int32List) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{5} + return fileDescriptor_47c504407d284ecc, []int{5} } func (m *Int32List) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Int32List.Unmarshal(m, b) @@ -845,8 +847,8 @@ func (m *Int32List) XXX_Unmarshal(b []byte) error { func (m *Int32List) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Int32List.Marshal(b, m, deterministic) } -func (dst *Int32List) XXX_Merge(src proto.Message) { - xxx_messageInfo_Int32List.Merge(dst, src) +func (m *Int32List) XXX_Merge(src proto.Message) { + xxx_messageInfo_Int32List.Merge(m, src) } func (m *Int32List) XXX_Size() int { return xxx_messageInfo_Int32List.Size(m) @@ -875,7 +877,7 @@ func (m *Int64List) Reset() { *m = Int64List{} } func (m *Int64List) String() string { return proto.CompactTextString(m) } func (*Int64List) ProtoMessage() {} func (*Int64List) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{6} + return fileDescriptor_47c504407d284ecc, []int{6} } func (m *Int64List) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Int64List.Unmarshal(m, b) @@ -883,8 +885,8 @@ func (m *Int64List) XXX_Unmarshal(b []byte) error { func (m *Int64List) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Int64List.Marshal(b, m, deterministic) } -func (dst *Int64List) XXX_Merge(src proto.Message) { - xxx_messageInfo_Int64List.Merge(dst, src) +func (m *Int64List) XXX_Merge(src proto.Message) { + xxx_messageInfo_Int64List.Merge(m, src) } func (m *Int64List) XXX_Size() int { return xxx_messageInfo_Int64List.Size(m) @@ -913,7 +915,7 @@ func (m *DoubleList) Reset() { *m = DoubleList{} } func (m *DoubleList) String() string { return proto.CompactTextString(m) } func (*DoubleList) ProtoMessage() {} func (*DoubleList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{7} + return fileDescriptor_47c504407d284ecc, []int{7} } func (m *DoubleList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DoubleList.Unmarshal(m, b) @@ -921,8 +923,8 @@ func (m *DoubleList) XXX_Unmarshal(b []byte) error { func (m *DoubleList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DoubleList.Marshal(b, m, deterministic) } -func (dst *DoubleList) XXX_Merge(src proto.Message) { - xxx_messageInfo_DoubleList.Merge(dst, src) +func (m *DoubleList) XXX_Merge(src proto.Message) { + xxx_messageInfo_DoubleList.Merge(m, src) } func (m *DoubleList) XXX_Size() int { return xxx_messageInfo_DoubleList.Size(m) @@ -951,7 +953,7 @@ func (m *FloatList) Reset() { *m = FloatList{} } func (m *FloatList) String() string { return proto.CompactTextString(m) } func (*FloatList) ProtoMessage() {} func (*FloatList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{8} + return fileDescriptor_47c504407d284ecc, []int{8} } func (m *FloatList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FloatList.Unmarshal(m, b) @@ -959,8 +961,8 @@ func (m *FloatList) XXX_Unmarshal(b []byte) error { func (m *FloatList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FloatList.Marshal(b, m, deterministic) } -func (dst *FloatList) XXX_Merge(src proto.Message) { - xxx_messageInfo_FloatList.Merge(dst, src) +func (m *FloatList) XXX_Merge(src proto.Message) { + xxx_messageInfo_FloatList.Merge(m, src) } func (m *FloatList) XXX_Size() int { return xxx_messageInfo_FloatList.Size(m) @@ -989,7 +991,7 @@ func (m *BoolList) Reset() { *m = BoolList{} } func (m *BoolList) String() string { return proto.CompactTextString(m) } func (*BoolList) ProtoMessage() {} func (*BoolList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{9} + return fileDescriptor_47c504407d284ecc, []int{9} } func (m *BoolList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BoolList.Unmarshal(m, b) @@ -997,8 +999,8 @@ func (m *BoolList) XXX_Unmarshal(b []byte) error { func (m *BoolList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BoolList.Marshal(b, m, deterministic) } -func (dst *BoolList) XXX_Merge(src proto.Message) { - xxx_messageInfo_BoolList.Merge(dst, src) +func (m *BoolList) XXX_Merge(src proto.Message) { + xxx_messageInfo_BoolList.Merge(m, src) } func (m *BoolList) XXX_Size() int { return xxx_messageInfo_BoolList.Size(m) @@ -1027,7 +1029,7 @@ func (m *TimestampList) Reset() { *m = TimestampList{} } func (m *TimestampList) String() string { return proto.CompactTextString(m) } func (*TimestampList) ProtoMessage() {} func (*TimestampList) Descriptor() ([]byte, []int) { - return fileDescriptor_Value_0680a2f024df1112, []int{10} + return fileDescriptor_47c504407d284ecc, []int{10} } func (m *TimestampList) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_TimestampList.Unmarshal(m, b) @@ -1035,8 +1037,8 @@ func (m *TimestampList) XXX_Unmarshal(b []byte) error { func (m *TimestampList) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_TimestampList.Marshal(b, m, deterministic) } -func (dst *TimestampList) XXX_Merge(src proto.Message) { - xxx_messageInfo_TimestampList.Merge(dst, src) +func (m *TimestampList) XXX_Merge(src proto.Message) { + xxx_messageInfo_TimestampList.Merge(m, src) } func (m *TimestampList) XXX_Size() int { return xxx_messageInfo_TimestampList.Size(m) @@ -1069,9 +1071,9 @@ func init() { proto.RegisterEnum("feast.types.ValueType_Enum", ValueType_Enum_name, ValueType_Enum_value) } -func init() { proto.RegisterFile("feast/types/Value.proto", fileDescriptor_Value_0680a2f024df1112) } +func init() { proto.RegisterFile("feast/types/Value.proto", fileDescriptor_47c504407d284ecc) } -var fileDescriptor_Value_0680a2f024df1112 = []byte{ +var fileDescriptor_47c504407d284ecc = []byte{ // 626 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x94, 0xd1, 0x6f, 0x9a, 0x50, 0x14, 0xc6, 0xb9, 0x22, 0x0a, 0xc7, 0x36, 0x21, 0x37, 0xd9, 0xda, 0x34, 0x6d, 0x47, 0x7c, 0xe2, From e6f60ca68b22864506e1e99807e762ef3f7dba57 Mon Sep 17 00:00:00 2001 From: Pradithya Aria Date: Mon, 21 Jan 2019 16:51:55 +0800 Subject: [PATCH 3/8] Update python SDK --- sdk/python/feast/sdk/client.py | 86 ++----- sdk/python/feast/serving/Serving_pb2.py | 258 ++++++------------- sdk/python/feast/serving/Serving_pb2_grpc.py | 10 +- sdk/python/tests/sdk/test_client.py | 146 ++++------- 4 files changed, 156 insertions(+), 344 deletions(-) diff --git a/sdk/python/feast/sdk/client.py b/sdk/python/feast/sdk/client.py index d1b09ebf22..a2155893f0 100644 --- a/sdk/python/feast/sdk/client.py +++ b/sdk/python/feast/sdk/client.py @@ -16,13 +16,11 @@ Main interface for users to interact with the Core API. """ -import enum import os from datetime import datetime import grpc import pandas as pd -import dateutil.parser from google.protobuf.timestamp_pb2 import Timestamp from feast.core.CoreService_pb2_grpc import CoreServiceStub @@ -38,21 +36,10 @@ from feast.sdk.resources.storage import Storage from feast.sdk.utils.bq_util import TableDownloader from feast.sdk.utils.print_utils import spec_to_yaml -from feast.serving.Serving_pb2 import QueryFeatures, RequestDetail, \ - TimestampRange +from feast.serving.Serving_pb2 import QueryFeaturesRequest, TimestampRange from feast.serving.Serving_pb2_grpc import ServingAPIStub -class ServingRequestType(enum.Enum): - """ - Request type for serving api - """ - LAST = 0 - """ Get last value of a feature """ - LIST = 1 - """ Get list of value of a feature """ - - class Client: def __init__(self, core_url=None, serving_url=None, verbose=False): """Create an instance of Feast client which is connected to feast @@ -222,12 +209,8 @@ def create_dataset(self, feature_set, start_date, end_date, resp.datasetInfo.tableUrl)) return DatasetInfo(resp.datasetInfo.name, resp.datasetInfo.tableUrl) - def get_serving_data(self, feature_set, entity_keys, - request_type=ServingRequestType.LAST, - ts_range=[], limit=10): - """Get data from the feast serving layer. You can either retrieve the - the latest value, or a list of the latest values, up to a provided - limit. + def get_serving_data(self, feature_set, entity_keys, ts_range=None): + """Get feature value from feast serving API. If server_url is not provided, the value stored in the environment variable FEAST_SERVING_URL is used to connect to the serving server instead. @@ -236,23 +219,16 @@ def get_serving_data(self, feature_set, entity_keys, feature_set (feast.sdk.resources.feature_set.FeatureSet): feature set representing the data wanted entity_keys (:obj: `list` of :obj: `str): list of entity keys - request_type (feast.sdk.utils.types.ServingRequestType): - (default: feast.sdk.utils.types.ServingRequestType.LAST) type of - request: one of [LIST, LAST] - ts_range (:obj: `list` of str, optional): size 2 list of start - timestamp and end timestamp, in ISO 8601 format. Only required if - request_type is set to LIST - limit (int, optional): (default: 10) number of values to get. Only - required if request_type is set to LIST + ts_range (:obj: `list` of str, optional): size 2 list of start + timestamp and end timestamp, in ISO 8601 format. It will + filter out any feature value having event timestamp outside + of the ts_range. Returns: pandas.DataFrame: DataFrame of results """ - - ts_range = [_timestamp_from_datetime(dateutil.parser.parse(dt)) - for dt in ts_range] request = self._build_serving_request(feature_set, entity_keys, - request_type, ts_range, limit) + ts_range) self._connect_serving() return self._response_to_df(feature_set, self._serving_service_stub .QueryFeatures(request)) @@ -317,20 +293,21 @@ def _connect_serving(self): self.__serving_channel = grpc.insecure_channel(self.serving_url) self._serving_service_stub = ServingAPIStub(self.__serving_channel) - def _build_serving_request(self, feature_set, entity_keys, request_type, - ts_range, limit): + def _build_serving_request(self, feature_set, entity_keys, ts_range): """Helper function to build serving service request.""" - request = QueryFeatures.Request(entityName=feature_set.entity, - entityId=entity_keys) - features = [RequestDetail(featureId=feat_id, type=request_type.value) - for feat_id in feature_set.features] - - if request_type == ServingRequestType.LIST: - ts_range = TimestampRange(start=ts_range[0], end=ts_range[1]) - request.timestampRange.CopyFrom(ts_range) - for feature in features: - feature.limit = limit - request.requestDetails.extend(features) + if ts_range is not None: + if len(ts_range) != 2: + raise ValueError("ts_range must have len 2") + + start = Timestamp() + end = Timestamp() + start.FromJsonString(ts_range[0]) + end.FromJsonString(ts_range[1]) + ts_range = TimestampRange(start=start, end=end) + request = QueryFeaturesRequest(entityName=feature_set.entity, + entityId=entity_keys, + featureId=feature_set.features, + timeRange=ts_range) return request def _response_to_df(self, feature_set, response): @@ -339,25 +316,18 @@ def _response_to_df(self, feature_set, response): feature_tables = [] features = response.entities[entity_key].features for feature_name in features: - rows = [] - v_list = features[feature_name].valueList - v_list = getattr(v_list, v_list.WhichOneof("valueList")).val - for idx in range(len(v_list)): - row = {response.entityName: entity_key, - feature_name: v_list[idx]} - if features[feature_name].HasField("timestampList"): - ts_seconds = \ - features[feature_name].timestampList.val[idx].seconds - row["timestamp"] = datetime.fromtimestamp(ts_seconds) - rows.append(row) - feature_tables.append(pd.DataFrame(rows)) + v = features[feature_name].value + v = getattr(v, v.WhichOneof("val")) + row = {response.entityName: entity_key, + feature_name: v} + feature_tables.append(pd.DataFrame(row, index=[0])) entity_table = feature_tables[0] for idx in range(1, len(feature_tables)): entity_table = pd.merge(left=entity_table, right=feature_tables[idx], how='outer') entity_tables.append(entity_table) if len(entity_tables) == 0: - return pd.DataFrame(columns=[feature_set.entity, "timestamp"] + + return pd.DataFrame(columns=[feature_set.entity] + feature_set.features) df = pd.concat(entity_tables) return df.reset_index(drop=True) diff --git a/sdk/python/feast/serving/Serving_pb2.py b/sdk/python/feast/serving/Serving_pb2.py index 9393a39ca7..9aab187c2b 100644 --- a/sdk/python/feast/serving/Serving_pb2.py +++ b/sdk/python/feast/serving/Serving_pb2.py @@ -3,7 +3,6 @@ import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf.internal import enum_type_wrapper from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -22,68 +21,43 @@ package='feast.serving', syntax='proto3', serialized_options=_b('\n\rfeast.servingB\017ServingAPIProtoZ8github.com/gojek/feast/protos/generated/go/feast/serving'), - serialized_pb=_b('\n\x1b\x66\x65\x61st/serving/Serving.proto\x12\rfeast.serving\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17\x66\x65\x61st/types/Value.proto\"\xde\x02\n\rQueryFeatures\x1a\x9c\x01\n\x07Request\x12\x12\n\nentityName\x18\x01 \x01(\t\x12\x10\n\x08\x65ntityId\x18\x02 \x03(\t\x12\x34\n\x0erequestDetails\x18\x03 \x03(\x0b\x32\x1c.feast.serving.RequestDetail\x12\x35\n\x0etimestampRange\x18\x04 \x01(\x0b\x32\x1d.feast.serving.TimestampRange\x1a\xad\x01\n\x08Response\x12\x12\n\nentityName\x18\x01 \x01(\t\x12\x45\n\x08\x65ntities\x18\x02 \x03(\x0b\x32\x33.feast.serving.QueryFeatures.Response.EntitiesEntry\x1a\x46\n\rEntitiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.feast.serving.Entity:\x02\x38\x01\"[\n\rRequestDetail\x12\x11\n\tfeatureId\x18\x01 \x01(\t\x12(\n\x04type\x18\x02 \x01(\x0e\x32\x1a.feast.serving.RequestType\x12\r\n\x05limit\x18\x03 \x01(\x05\"d\n\x0eTimestampRange\x12)\n\x05start\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\'\n\x03\x65nd\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\x91\x01\n\x06\x45ntity\x12\x35\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32#.feast.serving.Entity.FeaturesEntry\x1aP\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12.\n\x05value\x18\x02 \x01(\x0b\x32\x1f.feast.serving.FeatureValueList:\x02\x38\x01\"p\n\x10\x46\x65\x61tureValueList\x12)\n\tvalueList\x18\x01 \x01(\x0b\x32\x16.feast.types.ValueList\x12\x31\n\rtimestampList\x18\x02 \x01(\x0b\x32\x1a.feast.types.TimestampList*!\n\x0bRequestType\x12\x08\n\x04LAST\x10\x00\x12\x08\n\x04LIST\x10\x01\x32l\n\nServingAPI\x12^\n\rQueryFeatures\x12$.feast.serving.QueryFeatures.Request\x1a%.feast.serving.QueryFeatures.Response\"\x00\x42Z\n\rfeast.servingB\x0fServingAPIProtoZ8github.com/gojek/feast/protos/generated/go/feast/servingb\x06proto3') + serialized_pb=_b('\n\x1b\x66\x65\x61st/serving/Serving.proto\x12\rfeast.serving\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17\x66\x65\x61st/types/Value.proto\"\x81\x01\n\x14QueryFeaturesRequest\x12\x12\n\nentityName\x18\x01 \x01(\t\x12\x10\n\x08\x65ntityId\x18\x02 \x03(\t\x12\x11\n\tfeatureId\x18\x03 \x03(\t\x12\x30\n\ttimeRange\x18\x04 \x01(\x0b\x32\x1d.feast.serving.TimestampRange\"\xb9\x01\n\x15QueryFeaturesResponse\x12\x12\n\nentityName\x18\x01 \x01(\t\x12\x44\n\x08\x65ntities\x18\x02 \x03(\x0b\x32\x32.feast.serving.QueryFeaturesResponse.EntitiesEntry\x1a\x46\n\rEntitiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.feast.serving.Entity:\x02\x38\x01\"d\n\x0eTimestampRange\x12)\n\x05start\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\'\n\x03\x65nd\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\x8d\x01\n\x06\x45ntity\x12\x35\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32#.feast.serving.Entity.FeaturesEntry\x1aL\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.feast.serving.FeatureValue:\x02\x38\x01\"`\n\x0c\x46\x65\x61tureValue\x12!\n\x05value\x18\x01 \x01(\x0b\x32\x12.feast.types.Value\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp2j\n\nServingAPI\x12\\\n\rQueryFeatures\x12#.feast.serving.QueryFeaturesRequest\x1a$.feast.serving.QueryFeaturesResponse\"\x00\x42Z\n\rfeast.servingB\x0fServingAPIProtoZ8github.com/gojek/feast/protos/generated/go/feast/servingb\x06proto3') , dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,feast_dot_types_dot_Value__pb2.DESCRIPTOR,]) -_REQUESTTYPE = _descriptor.EnumDescriptor( - name='RequestType', - full_name='feast.serving.RequestType', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='LAST', index=0, number=0, - serialized_options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LIST', index=1, number=1, - serialized_options=None, - type=None), - ], - containing_type=None, - serialized_options=None, - serialized_start=914, - serialized_end=947, -) -_sym_db.RegisterEnumDescriptor(_REQUESTTYPE) - -RequestType = enum_type_wrapper.EnumTypeWrapper(_REQUESTTYPE) -LAST = 0 -LIST = 1 -_QUERYFEATURES_REQUEST = _descriptor.Descriptor( - name='Request', - full_name='feast.serving.QueryFeatures.Request', +_QUERYFEATURESREQUEST = _descriptor.Descriptor( + name='QueryFeaturesRequest', + full_name='feast.serving.QueryFeaturesRequest', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='entityName', full_name='feast.serving.QueryFeatures.Request.entityName', index=0, + name='entityName', full_name='feast.serving.QueryFeaturesRequest.entityName', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='entityId', full_name='feast.serving.QueryFeatures.Request.entityId', index=1, + name='entityId', full_name='feast.serving.QueryFeaturesRequest.entityId', index=1, number=2, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='requestDetails', full_name='feast.serving.QueryFeatures.Request.requestDetails', index=2, - number=3, type=11, cpp_type=10, label=3, + name='featureId', full_name='feast.serving.QueryFeaturesRequest.featureId', index=2, + number=3, type=9, cpp_type=9, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='timestampRange', full_name='feast.serving.QueryFeatures.Request.timestampRange', index=3, + name='timeRange', full_name='feast.serving.QueryFeaturesRequest.timeRange', index=3, number=4, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, @@ -101,26 +75,27 @@ extension_ranges=[], oneofs=[ ], - serialized_start=123, - serialized_end=279, + serialized_start=105, + serialized_end=234, ) -_QUERYFEATURES_RESPONSE_ENTITIESENTRY = _descriptor.Descriptor( + +_QUERYFEATURESRESPONSE_ENTITIESENTRY = _descriptor.Descriptor( name='EntitiesEntry', - full_name='feast.serving.QueryFeatures.Response.EntitiesEntry', + full_name='feast.serving.QueryFeaturesResponse.EntitiesEntry', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='key', full_name='feast.serving.QueryFeatures.Response.EntitiesEntry.key', index=0, + name='key', full_name='feast.serving.QueryFeaturesResponse.EntitiesEntry.key', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='value', full_name='feast.serving.QueryFeatures.Response.EntitiesEntry.value', index=1, + name='value', full_name='feast.serving.QueryFeaturesResponse.EntitiesEntry.value', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, @@ -138,26 +113,26 @@ extension_ranges=[], oneofs=[ ], - serialized_start=385, - serialized_end=455, + serialized_start=352, + serialized_end=422, ) -_QUERYFEATURES_RESPONSE = _descriptor.Descriptor( - name='Response', - full_name='feast.serving.QueryFeatures.Response', +_QUERYFEATURESRESPONSE = _descriptor.Descriptor( + name='QueryFeaturesResponse', + full_name='feast.serving.QueryFeaturesResponse', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='entityName', full_name='feast.serving.QueryFeatures.Response.entityName', index=0, + name='entityName', full_name='feast.serving.QueryFeaturesResponse.entityName', index=0, number=1, type=9, cpp_type=9, label=1, has_default_value=False, default_value=_b("").decode('utf-8'), message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='entities', full_name='feast.serving.QueryFeatures.Response.entities', index=1, + name='entities', full_name='feast.serving.QueryFeaturesResponse.entities', index=1, number=2, type=11, cpp_type=10, label=3, has_default_value=False, default_value=[], message_type=None, enum_type=None, containing_type=None, @@ -166,7 +141,7 @@ ], extensions=[ ], - nested_types=[_QUERYFEATURES_RESPONSE_ENTITIESENTRY, ], + nested_types=[_QUERYFEATURESRESPONSE_ENTITIESENTRY, ], enum_types=[ ], serialized_options=None, @@ -175,76 +150,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=282, - serialized_end=455, -) - -_QUERYFEATURES = _descriptor.Descriptor( - name='QueryFeatures', - full_name='feast.serving.QueryFeatures', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - ], - extensions=[ - ], - nested_types=[_QUERYFEATURES_REQUEST, _QUERYFEATURES_RESPONSE, ], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=105, - serialized_end=455, -) - - -_REQUESTDETAIL = _descriptor.Descriptor( - name='RequestDetail', - full_name='feast.serving.RequestDetail', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='featureId', full_name='feast.serving.RequestDetail.featureId', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=_b("").decode('utf-8'), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='type', full_name='feast.serving.RequestDetail.type', index=1, - number=2, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='limit', full_name='feast.serving.RequestDetail.limit', index=2, - number=3, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=457, - serialized_end=548, + serialized_start=237, + serialized_end=422, ) @@ -281,8 +188,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=550, - serialized_end=650, + serialized_start=424, + serialized_end=524, ) @@ -319,8 +226,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=718, - serialized_end=798, + serialized_start=592, + serialized_end=668, ) _ENTITY = _descriptor.Descriptor( @@ -349,27 +256,27 @@ extension_ranges=[], oneofs=[ ], - serialized_start=653, - serialized_end=798, + serialized_start=527, + serialized_end=668, ) -_FEATUREVALUELIST = _descriptor.Descriptor( - name='FeatureValueList', - full_name='feast.serving.FeatureValueList', +_FEATUREVALUE = _descriptor.Descriptor( + name='FeatureValue', + full_name='feast.serving.FeatureValue', filename=None, file=DESCRIPTOR, containing_type=None, fields=[ _descriptor.FieldDescriptor( - name='valueList', full_name='feast.serving.FeatureValueList.valueList', index=0, + name='value', full_name='feast.serving.FeatureValue.value', index=0, number=1, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), _descriptor.FieldDescriptor( - name='timestampList', full_name='feast.serving.FeatureValueList.timestampList', index=1, + name='timestamp', full_name='feast.serving.FeatureValue.timestamp', index=1, number=2, type=11, cpp_type=10, label=1, has_default_value=False, default_value=None, message_type=None, enum_type=None, containing_type=None, @@ -387,70 +294,49 @@ extension_ranges=[], oneofs=[ ], - serialized_start=800, - serialized_end=912, + serialized_start=670, + serialized_end=766, ) -_QUERYFEATURES_REQUEST.fields_by_name['requestDetails'].message_type = _REQUESTDETAIL -_QUERYFEATURES_REQUEST.fields_by_name['timestampRange'].message_type = _TIMESTAMPRANGE -_QUERYFEATURES_REQUEST.containing_type = _QUERYFEATURES -_QUERYFEATURES_RESPONSE_ENTITIESENTRY.fields_by_name['value'].message_type = _ENTITY -_QUERYFEATURES_RESPONSE_ENTITIESENTRY.containing_type = _QUERYFEATURES_RESPONSE -_QUERYFEATURES_RESPONSE.fields_by_name['entities'].message_type = _QUERYFEATURES_RESPONSE_ENTITIESENTRY -_QUERYFEATURES_RESPONSE.containing_type = _QUERYFEATURES -_REQUESTDETAIL.fields_by_name['type'].enum_type = _REQUESTTYPE +_QUERYFEATURESREQUEST.fields_by_name['timeRange'].message_type = _TIMESTAMPRANGE +_QUERYFEATURESRESPONSE_ENTITIESENTRY.fields_by_name['value'].message_type = _ENTITY +_QUERYFEATURESRESPONSE_ENTITIESENTRY.containing_type = _QUERYFEATURESRESPONSE +_QUERYFEATURESRESPONSE.fields_by_name['entities'].message_type = _QUERYFEATURESRESPONSE_ENTITIESENTRY _TIMESTAMPRANGE.fields_by_name['start'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP _TIMESTAMPRANGE.fields_by_name['end'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP -_ENTITY_FEATURESENTRY.fields_by_name['value'].message_type = _FEATUREVALUELIST +_ENTITY_FEATURESENTRY.fields_by_name['value'].message_type = _FEATUREVALUE _ENTITY_FEATURESENTRY.containing_type = _ENTITY _ENTITY.fields_by_name['features'].message_type = _ENTITY_FEATURESENTRY -_FEATUREVALUELIST.fields_by_name['valueList'].message_type = feast_dot_types_dot_Value__pb2._VALUELIST -_FEATUREVALUELIST.fields_by_name['timestampList'].message_type = feast_dot_types_dot_Value__pb2._TIMESTAMPLIST -DESCRIPTOR.message_types_by_name['QueryFeatures'] = _QUERYFEATURES -DESCRIPTOR.message_types_by_name['RequestDetail'] = _REQUESTDETAIL +_FEATUREVALUE.fields_by_name['value'].message_type = feast_dot_types_dot_Value__pb2._VALUE +_FEATUREVALUE.fields_by_name['timestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP +DESCRIPTOR.message_types_by_name['QueryFeaturesRequest'] = _QUERYFEATURESREQUEST +DESCRIPTOR.message_types_by_name['QueryFeaturesResponse'] = _QUERYFEATURESRESPONSE DESCRIPTOR.message_types_by_name['TimestampRange'] = _TIMESTAMPRANGE DESCRIPTOR.message_types_by_name['Entity'] = _ENTITY -DESCRIPTOR.message_types_by_name['FeatureValueList'] = _FEATUREVALUELIST -DESCRIPTOR.enum_types_by_name['RequestType'] = _REQUESTTYPE +DESCRIPTOR.message_types_by_name['FeatureValue'] = _FEATUREVALUE _sym_db.RegisterFileDescriptor(DESCRIPTOR) -QueryFeatures = _reflection.GeneratedProtocolMessageType('QueryFeatures', (_message.Message,), dict( - - Request = _reflection.GeneratedProtocolMessageType('Request', (_message.Message,), dict( - DESCRIPTOR = _QUERYFEATURES_REQUEST, - __module__ = 'feast.serving.Serving_pb2' - # @@protoc_insertion_point(class_scope:feast.serving.QueryFeatures.Request) - )) - , +QueryFeaturesRequest = _reflection.GeneratedProtocolMessageType('QueryFeaturesRequest', (_message.Message,), dict( + DESCRIPTOR = _QUERYFEATURESREQUEST, + __module__ = 'feast.serving.Serving_pb2' + # @@protoc_insertion_point(class_scope:feast.serving.QueryFeaturesRequest) + )) +_sym_db.RegisterMessage(QueryFeaturesRequest) - Response = _reflection.GeneratedProtocolMessageType('Response', (_message.Message,), dict( +QueryFeaturesResponse = _reflection.GeneratedProtocolMessageType('QueryFeaturesResponse', (_message.Message,), dict( - EntitiesEntry = _reflection.GeneratedProtocolMessageType('EntitiesEntry', (_message.Message,), dict( - DESCRIPTOR = _QUERYFEATURES_RESPONSE_ENTITIESENTRY, - __module__ = 'feast.serving.Serving_pb2' - # @@protoc_insertion_point(class_scope:feast.serving.QueryFeatures.Response.EntitiesEntry) - )) - , - DESCRIPTOR = _QUERYFEATURES_RESPONSE, + EntitiesEntry = _reflection.GeneratedProtocolMessageType('EntitiesEntry', (_message.Message,), dict( + DESCRIPTOR = _QUERYFEATURESRESPONSE_ENTITIESENTRY, __module__ = 'feast.serving.Serving_pb2' - # @@protoc_insertion_point(class_scope:feast.serving.QueryFeatures.Response) + # @@protoc_insertion_point(class_scope:feast.serving.QueryFeaturesResponse.EntitiesEntry) )) , - DESCRIPTOR = _QUERYFEATURES, - __module__ = 'feast.serving.Serving_pb2' - # @@protoc_insertion_point(class_scope:feast.serving.QueryFeatures) - )) -_sym_db.RegisterMessage(QueryFeatures) -_sym_db.RegisterMessage(QueryFeatures.Request) -_sym_db.RegisterMessage(QueryFeatures.Response) -_sym_db.RegisterMessage(QueryFeatures.Response.EntitiesEntry) - -RequestDetail = _reflection.GeneratedProtocolMessageType('RequestDetail', (_message.Message,), dict( - DESCRIPTOR = _REQUESTDETAIL, + DESCRIPTOR = _QUERYFEATURESRESPONSE, __module__ = 'feast.serving.Serving_pb2' - # @@protoc_insertion_point(class_scope:feast.serving.RequestDetail) + # @@protoc_insertion_point(class_scope:feast.serving.QueryFeaturesResponse) )) -_sym_db.RegisterMessage(RequestDetail) +_sym_db.RegisterMessage(QueryFeaturesResponse) +_sym_db.RegisterMessage(QueryFeaturesResponse.EntitiesEntry) TimestampRange = _reflection.GeneratedProtocolMessageType('TimestampRange', (_message.Message,), dict( DESCRIPTOR = _TIMESTAMPRANGE, @@ -474,16 +360,16 @@ _sym_db.RegisterMessage(Entity) _sym_db.RegisterMessage(Entity.FeaturesEntry) -FeatureValueList = _reflection.GeneratedProtocolMessageType('FeatureValueList', (_message.Message,), dict( - DESCRIPTOR = _FEATUREVALUELIST, +FeatureValue = _reflection.GeneratedProtocolMessageType('FeatureValue', (_message.Message,), dict( + DESCRIPTOR = _FEATUREVALUE, __module__ = 'feast.serving.Serving_pb2' - # @@protoc_insertion_point(class_scope:feast.serving.FeatureValueList) + # @@protoc_insertion_point(class_scope:feast.serving.FeatureValue) )) -_sym_db.RegisterMessage(FeatureValueList) +_sym_db.RegisterMessage(FeatureValue) DESCRIPTOR._options = None -_QUERYFEATURES_RESPONSE_ENTITIESENTRY._options = None +_QUERYFEATURESRESPONSE_ENTITIESENTRY._options = None _ENTITY_FEATURESENTRY._options = None _SERVINGAPI = _descriptor.ServiceDescriptor( @@ -492,16 +378,16 @@ file=DESCRIPTOR, index=0, serialized_options=None, - serialized_start=949, - serialized_end=1057, + serialized_start=768, + serialized_end=874, methods=[ _descriptor.MethodDescriptor( name='QueryFeatures', full_name='feast.serving.ServingAPI.QueryFeatures', index=0, containing_service=None, - input_type=_QUERYFEATURES_REQUEST, - output_type=_QUERYFEATURES_RESPONSE, + input_type=_QUERYFEATURESREQUEST, + output_type=_QUERYFEATURESRESPONSE, serialized_options=None, ), ]) diff --git a/sdk/python/feast/serving/Serving_pb2_grpc.py b/sdk/python/feast/serving/Serving_pb2_grpc.py index a17c7b6305..d3a8da4b4d 100644 --- a/sdk/python/feast/serving/Serving_pb2_grpc.py +++ b/sdk/python/feast/serving/Serving_pb2_grpc.py @@ -16,8 +16,8 @@ def __init__(self, channel): """ self.QueryFeatures = channel.unary_unary( '/feast.serving.ServingAPI/QueryFeatures', - request_serializer=feast_dot_serving_dot_Serving__pb2.QueryFeatures.Request.SerializeToString, - response_deserializer=feast_dot_serving_dot_Serving__pb2.QueryFeatures.Response.FromString, + request_serializer=feast_dot_serving_dot_Serving__pb2.QueryFeaturesRequest.SerializeToString, + response_deserializer=feast_dot_serving_dot_Serving__pb2.QueryFeaturesResponse.FromString, ) @@ -26,7 +26,7 @@ class ServingAPIServicer(object): pass def QueryFeatures(self, request, context): - """Query features from Feast + """Query features from Feast serving storage """ context.set_code(grpc.StatusCode.UNIMPLEMENTED) context.set_details('Method not implemented!') @@ -37,8 +37,8 @@ def add_ServingAPIServicer_to_server(servicer, server): rpc_method_handlers = { 'QueryFeatures': grpc.unary_unary_rpc_method_handler( servicer.QueryFeatures, - request_deserializer=feast_dot_serving_dot_Serving__pb2.QueryFeatures.Request.FromString, - response_serializer=feast_dot_serving_dot_Serving__pb2.QueryFeatures.Response.SerializeToString, + request_deserializer=feast_dot_serving_dot_Serving__pb2.QueryFeaturesRequest.FromString, + response_serializer=feast_dot_serving_dot_Serving__pb2.QueryFeaturesResponse.SerializeToString, ), } generic_handler = grpc.method_handlers_generic_handler( diff --git a/sdk/python/tests/sdk/test_client.py b/sdk/python/tests/sdk/test_client.py index 8190934924..5dc8b18374 100644 --- a/sdk/python/tests/sdk/test_client.py +++ b/sdk/python/tests/sdk/test_client.py @@ -29,7 +29,6 @@ from feast.core.DatasetService_pb2 import DatasetServiceTypes from feast.core.DatasetService_pb2 import DatasetInfo as DatasetInfo_pb from feast.sdk.client import Client, _parse_date, _timestamp_from_datetime -from feast.sdk.client import ServingRequestType from feast.sdk.importer import Importer from feast.sdk.resources.entity import Entity from feast.sdk.resources.feature import Feature, Granularity @@ -37,12 +36,12 @@ from feast.sdk.resources.feature_set import FeatureSet, DatasetInfo, FileType from feast.sdk.resources.storage import Storage from feast.sdk.utils.bq_util import TableDownloader -from feast.serving.Serving_pb2 import QueryFeatures, RequestDetail, \ - TimestampRange, FeatureValueList +from feast.serving.Serving_pb2 import TimestampRange, \ + QueryFeaturesRequest, QueryFeaturesResponse, FeatureValue from feast.specs.FeatureSpec_pb2 import FeatureSpec, DataStores, DataStore from feast.specs.ImportSpec_pb2 import ImportSpec from feast.specs.StorageSpec_pb2 import StorageSpec -from feast.types.Value_pb2 import ValueList, TimestampList, Int32List +from feast.types.Value_pb2 import Value class TestClient(object): @@ -146,26 +145,26 @@ def test_create_dataset_invalid_args(self, client): with pytest.raises(ValueError, match="feature set is empty"): inv_feature_set = FeatureSet("entity", []) client.create_dataset(inv_feature_set, "2018-12-01", - "2018-12-02") + "2018-12-02") # invalid start date with pytest.raises(ValueError, match="Incorrect date format, should be YYYY-MM-DD"): client.create_dataset(feature_set, "20181201", - "2018-12-02") + "2018-12-02") # invalid end date with pytest.raises(ValueError, match="Incorrect date format, should be YYYY-MM-DD"): client.create_dataset(feature_set, "2018-12-01", - "20181202") + "20181202") # start date > end date with pytest.raises(ValueError, match="end_date is before start_date"): client.create_dataset(feature_set, "2018-12-02", - "2018-12-01") + "2018-12-01") # invalid limit with pytest.raises(ValueError, match="limit is not a positive integer"): client.create_dataset(feature_set, "2018-12-01", - "2018-12-02", -1) + "2018-12-02", -1) def test_create_dataset(self, client, mocker): entity_name = "myentity" @@ -175,7 +174,7 @@ def test_create_dataset(self, client, mocker): end_date = "2018-12-31" ds_pb = DatasetInfo_pb(name="dataset_name", - tableUrl="project.dataset.table") + tableUrl="project.dataset.table") mock_trn_stub = training.DatasetServiceStub(grpc.insecure_channel("")) mocker.patch.object(mock_trn_stub, "CreateDataset", @@ -206,7 +205,7 @@ def test_create_dataset_with_limit(self, client, mocker): limit = 100 ds_pb = DatasetInfo_pb(name="dataset_name", - tableUrl="project.dataset.table") + tableUrl="project.dataset.table") mock_trn_stub = training.DatasetServiceStub(grpc.insecure_channel("")) mocker.patch.object(mock_trn_stub, "CreateDataset", @@ -215,7 +214,7 @@ def test_create_dataset_with_limit(self, client, mocker): client._dataset_service_stub = mock_trn_stub ds = client.create_dataset(fs, start_date, end_date, - limit=limit) + limit=limit) assert "dataset_name" == ds.name assert "project.dataset.table" == ds.table_id @@ -239,17 +238,18 @@ def test_create_dataset_with_name_prefix(self, client, mocker): name_prefix = "feast" ds_pb = DatasetInfo_pb(name="dataset_name", - tableUrl="project.dataset.table") + tableUrl="project.dataset.table") - mock_dssvc_stub = training.DatasetServiceStub(grpc.insecure_channel("")) + mock_dssvc_stub = training.DatasetServiceStub( + grpc.insecure_channel("")) mocker.patch.object(mock_dssvc_stub, "CreateDataset", return_value=DatasetServiceTypes .CreateDatasetResponse(datasetInfo=ds_pb)) client._dataset_service_stub = mock_dssvc_stub ds = client.create_dataset(fs, start_date, end_date, - limit=limit, - name_prefix=name_prefix) + limit=limit, + name_prefix=name_prefix) assert "dataset_name" == ds.name assert "project.dataset.table" == ds.table_id @@ -263,96 +263,60 @@ def test_create_dataset_with_name_prefix(self, client, mocker): ) ) - def test_build_serving_request_last(self, client): + def test_build_serving_request(self, client): feature_set = FeatureSet("entity", ["entity.none.feat1", "entity.none.feat2"]) - req = client._build_serving_request(feature_set, ["1", "2", "3"], - ServingRequestType.LAST, None, - None) - expected_request_detail = \ - [RequestDetail(featureId=feat_id, type=serving_pb.LAST) - for feat_id in ["entity.none.feat1", "entity.none.feat2"]] - expected = QueryFeatures.Request(entityName="entity", - entityId=["1", "2", "3"], - requestDetails=expected_request_detail) + req = client._build_serving_request(feature_set, ["1", "2", "3"], None) + expected = QueryFeaturesRequest(entityName="entity", + entityId=["1", "2", "3"], + featureId=feature_set.features, + timeRange=None) assert req == expected - def test_build_serving_request_list(self, client): - feature_set = FeatureSet("entity", - ["entity.none.feat1", "entity.none.feat2"]) - - req = client._build_serving_request( - feature_set, ["1", "2", "3"], ServingRequestType.LIST, - [Timestamp(seconds=10), Timestamp(seconds=11)], 2) - expected_request_detail = \ - [RequestDetail(featureId=feat_id, type=serving_pb.LIST, limit=2) - for feat_id in ["entity.none.feat1", "entity.none.feat2"]] - expected_ts_range = \ - TimestampRange(start=Timestamp(seconds=10), - end=Timestamp(seconds=11)) - expected = QueryFeatures.Request(entityName="entity", - entityId=["1", "2", "3"], - requestDetails=expected_request_detail, - timestampRange=expected_ts_range) + tsRange = ["2018-03-03T00:23:00Z", "2018-03-03T00:23:00Z"] + start = Timestamp() + start.FromJsonString(tsRange[0]) + end = Timestamp() + end.FromJsonString(tsRange[1]) + expTimeRange = TimestampRange(start=start, end=end) + expected = QueryFeaturesRequest(entityName="entity", + entityId=["1", "2", "3"], + featureId=feature_set.features, + timeRange=expTimeRange) + + req = client._build_serving_request(feature_set, ["1", "2", "3"], tsRange) assert req == expected + def test_serving_response_to_df(self, client): response = self._create_query_features_response( entity_name="entity", entities=[{"id": "1", - "feat1": [(3, Timestamp(seconds=10))], - "feat2": [(1, Timestamp(seconds=10))]}, + "feat1": (1, Timestamp(seconds=10)), + "feat2": (2, Timestamp(seconds=10))}, {"id": "2", - "feat1": [(3, Timestamp(seconds=10))], - "feat2": [(3, Timestamp(seconds=10))]}], + "feat1": (3, Timestamp(seconds=10)), + "feat2": (4, Timestamp(seconds=10))}], features=["feat1", "feat2"] ) expected_df = pd.DataFrame({'entity': ["1", "2"], - 'timestamp': [datetime.fromtimestamp(10), - datetime.fromtimestamp(10)], - 'feat1': [3, 3], - 'feat2': [1, 3]}) \ - .reset_index(drop=True) - df = client._response_to_df(FeatureSet("", []), response) \ - .sort_values(['entity', 'timestamp']) \ - .reset_index(drop=True)[expected_df.columns] - assert_frame_equal(df, expected_df) - - def test_serving_response_to_df_multiple(self, client): - response = self._create_query_features_response( - entity_name="entity", - entities=[{"id": "1", - "feat1": [(3, Timestamp(seconds=10)), - (4, Timestamp(seconds=11))], - "feat2": [(1, Timestamp(seconds=10)), - (3, Timestamp(seconds=11))]}, - {"id": "2", - "feat1": [(4, Timestamp(seconds=10)), - (6, Timestamp(seconds=11))], - "feat2": [(2, Timestamp(seconds=10)), - (5, Timestamp(seconds=11))]}], - features=["feat1", "feat2"] - ) - expected_df = pd.DataFrame({'entity': ["1", "1", "2", "2"], - 'timestamp': [datetime.fromtimestamp(10), - datetime.fromtimestamp(11), - datetime.fromtimestamp(10), - datetime.fromtimestamp(11)], - 'feat1': [3, 4, 4, 6], - 'feat2': [1, 3, 2, 5]}) \ + 'feat1': [1, 3], + 'feat2': [2, 4]}) \ .reset_index(drop=True) df = client._response_to_df(FeatureSet("", []), response) \ - .sort_values(['entity', 'timestamp']) \ + .sort_values(['entity']) \ .reset_index(drop=True)[expected_df.columns] assert_frame_equal(df, expected_df) def test_serving_response_to_df_no_data(self, client): - response = QueryFeatures.Response(entityName="entity") + response = QueryFeaturesResponse(entityName="entity") expected_df = pd.DataFrame(columns= - ['entity', 'timestamp', 'entity.day.feat1', 'entity.day.feat2']) + ['entity', 'entity.day.feat1', + 'entity.day.feat2']) df = client._response_to_df(FeatureSet("entity", - ["entity.day.feat1", "entity.day.feat2"]), response) + ["entity.day.feat1", + "entity.day.feat2"]), response) assert_frame_equal(df, expected_df) def test_download_dataset_as_file(self, client, mocker): @@ -378,21 +342,13 @@ def test_download_dataset_as_file(self, client, mocker): FileType.CSV) def _create_query_features_response(self, entity_name, entities, features): - # entities should be in the form: - # {"id": "1", - # "feature_1": [(val1, ts1), (val2, ts2)], - # "feature_2": ...} - response = QueryFeatures.Response(entityName=entity_name) + response = QueryFeaturesResponse(entityName=entity_name) for entity in entities: features_map = {} for feature in features: - features_map[feature] = FeatureValueList( - valueList=ValueList( - int32List=Int32List( - val=[v[0] for v in entity[feature]]) - ), - timestampList=TimestampList( - val=[v[1] for v in entity[feature]]) + features_map[feature] = FeatureValue( + value=Value(int32Val=entity[feature][0]), + timestamp=entity[feature][1] ) entity_pb = serving_pb.Entity(features=features_map) response.entities[entity["id"]].CopyFrom(entity_pb) From 2dbf86f78589b61fab2c519a081bbb68bf3d9ab2 Mon Sep 17 00:00:00 2001 From: Pradithya Aria Date: Mon, 21 Jan 2019 23:53:27 +0800 Subject: [PATCH 4/8] Refactor python SDK --- sdk/python/feast/sdk/client.py | 27 +++------ sdk/python/tests/sdk/test_client.py | 94 +++++++++++++++++++++-------- 2 files changed, 79 insertions(+), 42 deletions(-) diff --git a/sdk/python/feast/sdk/client.py b/sdk/python/feast/sdk/client.py index a2155893f0..285a864c2a 100644 --- a/sdk/python/feast/sdk/client.py +++ b/sdk/python/feast/sdk/client.py @@ -311,25 +311,16 @@ def _build_serving_request(self, feature_set, entity_keys, ts_range): return request def _response_to_df(self, feature_set, response): - entity_tables = [] - for entity_key in response.entities: - feature_tables = [] - features = response.entities[entity_key].features - for feature_name in features: - v = features[feature_name].value - v = getattr(v, v.WhichOneof("val")) - row = {response.entityName: entity_key, - feature_name: v} - feature_tables.append(pd.DataFrame(row, index=[0])) - entity_table = feature_tables[0] - for idx in range(1, len(feature_tables)): - entity_table = pd.merge(left=entity_table, - right=feature_tables[idx], how='outer') - entity_tables.append(entity_table) - if len(entity_tables) == 0: - return pd.DataFrame(columns=[feature_set.entity] + + df = pd.DataFrame(columns=[feature_set.entity] + feature_set.features) - df = pd.concat(entity_tables) + for entity_id in response.entities: + feature_map = response.entities[entity_id].features + row = {response.entityName: entity_id} + for feature_id in feature_map: + v = feature_map[feature_id].value + v = getattr(v, v.WhichOneof("val")) + row[feature_id] = v + df = df.append(row, ignore_index=True) return df.reset_index(drop=True) def _apply(self, obj): diff --git a/sdk/python/tests/sdk/test_client.py b/sdk/python/tests/sdk/test_client.py index 5dc8b18374..5f2f42e7ff 100644 --- a/sdk/python/tests/sdk/test_client.py +++ b/sdk/python/tests/sdk/test_client.py @@ -17,6 +17,7 @@ import grpc import pandas as pd import pytest +import numpy as np from google.protobuf.timestamp_pb2 import Timestamp from pandas.util.testing import assert_frame_equal @@ -285,29 +286,72 @@ def test_build_serving_request(self, client): featureId=feature_set.features, timeRange=expTimeRange) - req = client._build_serving_request(feature_set, ["1", "2", "3"], tsRange) + req = client._build_serving_request(feature_set, ["1", "2", "3"], + tsRange) assert req == expected - def test_serving_response_to_df(self, client): response = self._create_query_features_response( entity_name="entity", - entities=[{"id": "1", - "feat1": (1, Timestamp(seconds=10)), - "feat2": (2, Timestamp(seconds=10))}, - {"id": "2", - "feat1": (3, Timestamp(seconds=10)), - "feat2": (4, Timestamp(seconds=10))}], - features=["feat1", "feat2"] + entities={"1": {"entity.feat1": (1, Timestamp(seconds=1)), + "entity.feat2": (2, Timestamp(seconds=2))}, + "2": {"entity.feat1": (3, Timestamp(seconds=3)), + "entity.feat2": (4, Timestamp(seconds=4))}} + ) + expected_df = pd.DataFrame({'entity': ["1", "2"], + 'entity.feat1': [1, 3], + 'entity.feat2': [2, 4]}) \ + .reset_index(drop=True) + df = client._response_to_df(FeatureSet("entity", ["entity.feat1", + "entity.feat2"]), + response) \ + .sort_values(['entity']) \ + .reset_index(drop=True)[expected_df.columns] + assert_frame_equal(df, expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) + + def test_serving_response_to_df_with_missing_value(self, client): + response = self._create_query_features_response( + entity_name="entity", + entities={"1": {"entity.feat1": (1, Timestamp(seconds=1))}, + "2": {"entity.feat1": (3, Timestamp(seconds=3)), + "entity.feat2": (4, Timestamp(seconds=4))}} + ) + expected_df = pd.DataFrame({'entity': ["1", "2"], + 'entity.feat1': [1, 3], + 'entity.feat2': [np.NaN, 4]}) \ + .reset_index(drop=True) + df = client._response_to_df(FeatureSet("entity", ["entity.feat1", + "entity.feat2"]), + response) \ + .sort_values(['entity']) \ + .reset_index(drop=True)[expected_df.columns] + assert_frame_equal(df, expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) + + def test_serving_response_to_df_with_missing_feature(self, client): + response = self._create_query_features_response( + entity_name="entity", + entities={"1": {"entity.feat1": (1, Timestamp(seconds=1))}, + "2": {"entity.feat1": (3, Timestamp(seconds=3))}} ) expected_df = pd.DataFrame({'entity': ["1", "2"], - 'feat1': [1, 3], - 'feat2': [2, 4]}) \ + 'entity.feat1': [1, 3], + 'entity.feat2': [np.NaN, np.NaN]}) \ .reset_index(drop=True) - df = client._response_to_df(FeatureSet("", []), response) \ + df = client._response_to_df(FeatureSet("entity", ["entity.feat1", + "entity.feat2"]), + response) \ .sort_values(['entity']) \ .reset_index(drop=True)[expected_df.columns] - assert_frame_equal(df, expected_df) + assert_frame_equal(df, expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) def test_serving_response_to_df_no_data(self, client): response = QueryFeaturesResponse(entityName="entity") @@ -317,7 +361,10 @@ def test_serving_response_to_df_no_data(self, client): df = client._response_to_df(FeatureSet("entity", ["entity.day.feat1", "entity.day.feat2"]), response) - assert_frame_equal(df, expected_df) + assert_frame_equal(df, expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) def test_download_dataset_as_file(self, client, mocker): destination = "/tmp/dest_file" @@ -341,17 +388,16 @@ def test_download_dataset_as_file(self, client, mocker): staging_location, FileType.CSV) - def _create_query_features_response(self, entity_name, entities, features): + def _create_query_features_response(self, entity_name, entities): response = QueryFeaturesResponse(entityName=entity_name) - for entity in entities: - features_map = {} - for feature in features: - features_map[feature] = FeatureValue( - value=Value(int32Val=entity[feature][0]), - timestamp=entity[feature][1] - ) - entity_pb = serving_pb.Entity(features=features_map) - response.entities[entity["id"]].CopyFrom(entity_pb) + for entity_id, feature_map in entities.items(): + feature = {} + for feature_id, feature_value in feature_map.items(): + feature[feature_id] = FeatureValue( + value=Value(int32Val=feature_value[0]), + timestamp=feature_value[1]) + entity_pb = serving_pb.Entity(features=feature) + response.entities[entity_id].CopyFrom(entity_pb) return response From 2e9d522e14658c6ea42983d56c3bd7a4e939da63 Mon Sep 17 00:00:00 2001 From: Pradithya Aria Date: Tue, 19 Feb 2019 08:03:43 +0800 Subject: [PATCH 5/8] Remove server-side timestamp filter --- protos/feast/serving/Serving.proto | 11 ---- .../service/BigTableFeatureStorage.java | 32 ++-------- .../feast/serving/service/FeastServing.java | 3 +- .../service/FeatureRetrievalDispatcher.java | 22 ++----- .../feast/serving/service/FeatureStorage.java | 6 +- .../serving/service/RedisFeatureStorage.java | 53 +-------------- .../feast/serving/util/RequestHelper.java | 25 -------- .../java/feast/serving/util/TimeUtil.java | 7 -- .../feast/serving/grpc/FeastServingTest.java | 25 +------- .../serving/grpc/ServingGrpcServiceTest.java | 64 ------------------- .../BigTableFeatureStorageTestITCase.java | 48 ++------------ .../FeatureRetrievalDispatcherTest.java | 41 ++++-------- .../service/RedisFeatureStorageTest.java | 40 ++---------- .../testutil/FeatureStoragePopulator.java | 17 +---- 14 files changed, 41 insertions(+), 353 deletions(-) diff --git a/protos/feast/serving/Serving.proto b/protos/feast/serving/Serving.proto index 5ded4becf2..cbb1d38657 100644 --- a/protos/feast/serving/Serving.proto +++ b/protos/feast/serving/Serving.proto @@ -40,8 +40,6 @@ message QueryFeaturesRequest { // e.g: "driver.day.total_accepted_booking" // all requested feature ID shall have same entity name. repeated string featureId = 3; - // [optional] time range filter. If not specified all feature with any timestamp will be returned. - TimestampRange timeRange = 4; } message QueryFeaturesResponse { @@ -51,15 +49,6 @@ message QueryFeaturesResponse { map entities = 2; } -// range of timestamp for querying -// valid timestamp range is having start <= end -message TimestampRange { - // start time of the range query. - google.protobuf.Timestamp start = 1; - // end time of the range query. - google.protobuf.Timestamp end = 2; -} - message Entity { // map of feature ID and its feature value. map features = 1; diff --git a/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java b/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java index d279249cde..a990985839 100644 --- a/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java +++ b/serving/src/main/java/feast/serving/service/BigTableFeatureStorage.java @@ -20,13 +20,10 @@ import com.google.common.base.Strings; import com.google.protobuf.Timestamp; import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.exception.FeatureRetrievalException; import feast.serving.model.FeatureValue; -import feast.serving.util.TimeUtil; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.storage.BigTableProto.BigTableRowKey; -import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.Value; import java.io.IOException; import java.util.ArrayList; @@ -60,13 +57,10 @@ public BigTableFeatureStorage(Connection connection) { /** {@inheritDoc} */ @Override public List getFeature( - String entityName, - Collection entityIds, - Collection featureSpecs, - TimestampRange tsRange) { + String entityName, Collection entityIds, Collection featureSpecs) { List featureValues = new ArrayList<>(entityIds.size() * featureSpecs.size()); for (FeatureSpec featureSpec : featureSpecs) { - featureValues.addAll(getCurrentFeatureInternal(entityName, entityIds, featureSpec, tsRange)); + featureValues.addAll(getCurrentFeatureInternal(entityName, entityIds, featureSpec)); } return featureValues; } @@ -77,18 +71,14 @@ public List getFeature( * @param entityName entity name. * @param entityIds list of entity id. * @param featureSpec spec of the feature. - * @param tsRange time range filter * @return list of feature value. */ private List getCurrentFeatureInternal( - String entityName, - Collection entityIds, - FeatureSpec featureSpec, - TimestampRange tsRange) { + String entityName, Collection entityIds, FeatureSpec featureSpec) { List features = new ArrayList<>(entityIds.size()); String featureId = featureSpec.getId(); byte[] featureIdBytes = featureSpec.getId().getBytes(); - List gets = createGets(entityIds, featureSpec, tsRange); + List gets = createGets(entityIds, featureSpec); try (Table table = connection.getTable(TableName.valueOf(entityName))) { Result[] results = table.get(gets); for (Result result : results) { @@ -128,20 +118,11 @@ private List getCurrentFeatureInternal( * * @param entityIds list of entity ID. * @param featureSpec feature spec - * @param tsRange time filter * @return list of Get operation. */ - private List createGets( - Collection entityIds, FeatureSpec featureSpec, TimestampRange tsRange) { + private List createGets(Collection entityIds, FeatureSpec featureSpec) { byte[] featureIdBytes = featureSpec.getId().getBytes(); byte[] columnFamily = getColumnFamily(featureSpec); - long start = 0; - long end = 0; - if (TimeUtil.isTimeFilterRequired(tsRange)) { - start = Timestamps.toMillis(tsRange.getStart()); - end = Timestamps.toMillis(tsRange.getEnd()); - } - List gets = new ArrayList<>(); for (String entityId : entityIds) { String entityIdPrefix = DigestUtils.sha1Hex(entityId.getBytes()).substring(0, 7); @@ -149,9 +130,6 @@ private List createGets( Get get = new Get(btKey.toByteArray()); get.addColumn(columnFamily, featureIdBytes); try { - if (!featureSpec.getGranularity().equals(Enum.NONE) && tsRange != null) { - get.setTimeRange(start, end); - } get.readVersions(1); } catch (IOException e) { log.error("should not happen"); diff --git a/serving/src/main/java/feast/serving/service/FeastServing.java b/serving/src/main/java/feast/serving/service/FeastServing.java index ad89adf813..0e1ee830ab 100644 --- a/serving/src/main/java/feast/serving/service/FeastServing.java +++ b/serving/src/main/java/feast/serving/service/FeastServing.java @@ -81,8 +81,7 @@ public QueryFeaturesResponse queryFeatures(QueryFeaturesRequest request) { featureRetrievalDispatcher.dispatchFeatureRetrieval( request.getEntityName(), request.getEntityIdList(), - featureSpecs, - request.getTimeRange()); + featureSpecs); scope.span().log("finished retrieving all feature"); diff --git a/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java b/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java index 7673476128..97ed05957a 100644 --- a/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java +++ b/serving/src/main/java/feast/serving/service/FeatureRetrievalDispatcher.java @@ -23,7 +23,6 @@ import com.google.common.util.concurrent.ListenableFuture; import com.google.common.util.concurrent.ListeningExecutorService; import feast.serving.ServingAPIProto.Entity; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.config.AppConfig; import feast.serving.exception.FeatureRetrievalException; import feast.serving.model.FeatureValue; @@ -73,21 +72,17 @@ public FeatureRetrievalDispatcher( * @param entityName entity name of the feature. * @param entityIds list of entity ids. * @param featureSpecs list of request. - * @param timestampRange timestamp range of feature to be retrieved. * @return map of entityID and Entity instance. */ public Map dispatchFeatureRetrieval( - String entityName, - Collection entityIds, - Collection featureSpecs, - TimestampRange timestampRange) { + String entityName, Collection entityIds, Collection featureSpecs) { Map> groupedFeatureSpecs = groupByStorage(featureSpecs); if (groupedFeatureSpecs.size() <= 1) { - return runInCurrentThread(entityName, entityIds, timestampRange, groupedFeatureSpecs); + return runInCurrentThread(entityName, entityIds, groupedFeatureSpecs); } else { - return runWithExecutorService(entityName, entityIds, timestampRange, groupedFeatureSpecs); + return runWithExecutorService(entityName, entityIds, groupedFeatureSpecs); } } @@ -96,14 +91,12 @@ public Map dispatchFeatureRetrieval( * * @param entityName entity name of of the feature. * @param entityIds list of entity ID of the feature to be retrieved. - * @param tsRange timestamp range of the feature to be retrieved. * @param groupedFeatureSpecs feature spec grouped by storage ID. * @return entity map containing the result of feature retrieval. */ private Map runInCurrentThread( String entityName, Collection entityIds, - TimestampRange tsRange, Map> groupedFeatureSpecs) { try (Scope scope = tracer.buildSpan("FeatureRetrievalDispatcher-runInCurrentThread").startActive(true)) { @@ -113,7 +106,7 @@ private Map runInCurrentThread( FeatureStorage featureStorage = featureStorageRegistry.get(storageId); List featureValues; - featureValues = featureStorage.getFeature(entityName, entityIds, featureSpecs, tsRange); + featureValues = featureStorage.getFeature(entityName, entityIds, featureSpecs); EntityMapBuilder builder = new EntityMapBuilder(); builder.addFeatureValueList(featureValues); @@ -126,18 +119,15 @@ private Map runInCurrentThread( * * @param entityName entity name of the feature. * @param entityIds list of entity ID. - * @param tsRange timestamp range of the feature. * @param groupedFeatureSpec feature specs grouped by serving storage ID. * @return entity map containing result of feature retrieval. */ private Map runWithExecutorService( String entityName, Collection entityIds, - TimestampRange tsRange, Map> groupedFeatureSpec) { try (Scope scope = - tracer.buildSpan("FeatureRetrievalDispatcher-runWithExecutorService") - .startActive(true)) { + tracer.buildSpan("FeatureRetrievalDispatcher-runWithExecutorService").startActive(true)) { Span span = scope.span(); List> futures = new ArrayList<>(); EntityMapBuilder entityMapBuilder = new EntityMapBuilder(); @@ -148,7 +138,7 @@ private Map runWithExecutorService( executorService.submit( () -> { List featureValues = - featureStorage.getFeature(entityName, entityIds, featureSpecs, tsRange); + featureStorage.getFeature(entityName, entityIds, featureSpecs); entityMapBuilder.addFeatureValueList(featureValues); return null; })); diff --git a/serving/src/main/java/feast/serving/service/FeatureStorage.java b/serving/src/main/java/feast/serving/service/FeatureStorage.java index ebcd2675e3..e1ffb0306e 100644 --- a/serving/src/main/java/feast/serving/service/FeatureStorage.java +++ b/serving/src/main/java/feast/serving/service/FeatureStorage.java @@ -17,7 +17,6 @@ package feast.serving.service; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.exception.FeatureRetrievalException; import feast.serving.model.FeatureValue; import feast.specs.FeatureSpecProto.FeatureSpec; @@ -32,14 +31,11 @@ public interface FeatureStorage { * @param entityName entity name, e.g. 'driver', 'customer', 'area' * @param entityIds list of entity id. * @param featureSpecs list of feature spec for which the feature should be retrieved. * @param - * @param tsRange allowed event timestamp of the feature to be returned. Pass null to not filter * - * by time. * @return list of feature value. * @throws FeatureRetrievalException if anything goes wrong during feature retrieval. */ List getFeature( String entityName, Collection entityIds, - Collection featureSpecs, - TimestampRange tsRange); + Collection featureSpecs); } diff --git a/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java b/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java index 3bff50549d..d481c9554e 100644 --- a/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java +++ b/serving/src/main/java/feast/serving/service/RedisFeatureStorage.java @@ -18,16 +18,11 @@ package feast.serving.service; import com.google.protobuf.InvalidProtocolBufferException; -import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.exception.FeatureRetrievalException; import feast.serving.model.FeatureValue; -import feast.serving.util.TimeUtil; import feast.specs.FeatureSpecProto.FeatureSpec; import feast.storage.RedisProto.RedisBucketKey; import feast.storage.RedisProto.RedisBucketValue; -import feast.types.GranularityProto.Granularity.Enum; import io.opentracing.Scope; import io.opentracing.Span; import io.opentracing.Tracer; @@ -71,41 +66,19 @@ public RedisFeatureStorage(JedisPool jedisPool, Tracer tracer) { /** {@inheritDoc} */ @Override public List getFeature( - String entityName, - Collection entityIds, - Collection featureSpecs, - TimestampRange tsRange) { + String entityName, Collection entityIds, Collection featureSpecs) { try (Scope scope = tracer.buildSpan("Redis-getFeature").startActive(true)) { List getRequests = new ArrayList<>(entityIds.size() * featureSpecs.size()); - // Granularity NONE is currently treated differently since the event timestamp is always EPOCH - List getRequestsGranularityNone = new ArrayList<>(); for (FeatureSpec featureSpec : featureSpecs) { String featureId = featureSpec.getId(); String featureIdSha1Prefix = makeFeatureIdSha1Prefix(featureId); for (String entityId : entityIds) { RedisBucketKey key = makeBucketKey(entityId, featureIdSha1Prefix, BUCKET_ID_ZERO); - if (featureSpec.getGranularity().equals(Enum.NONE)) { - getRequestsGranularityNone.add(new GetRequest(entityId, featureId, key)); - } else { getRequests.add(new GetRequest(entityId, featureId, key)); - } } } scope.span().log("completed request creation"); - List unfilteredResult = sendAndProcessMultiGet(getRequests); - List granularityNoneResult = sendAndProcessMultiGet(getRequestsGranularityNone); - List combinedResult = - new ArrayList<>(unfilteredResult.size() + granularityNoneResult.size()); - if (!TimeUtil.isTimeFilterRequired(tsRange)) { - combinedResult.addAll(unfilteredResult); - combinedResult.addAll(granularityNoneResult); - return combinedResult; - } - - List filteredResult = filterByTimeRange(unfilteredResult, tsRange); - combinedResult.addAll(granularityNoneResult); - combinedResult.addAll(filteredResult); - return combinedResult; + return sendAndProcessMultiGet(getRequests); } } @@ -210,28 +183,6 @@ private String makeFeatureIdSha1Prefix(String featureId) { return DigestUtils.sha1Hex(featureId.getBytes()).substring(0, 7); } - /** - * Filter list of feature value by its timestamp - * - * @param unfilteredResult unflitered list of feature value - * @param tsRange time range filter - * @return filtered result - */ - private List filterByTimeRange( - List unfilteredResult, TimestampRange tsRange) { - return unfilteredResult - .stream() - .filter( - v -> { - Timestamp start = tsRange.getStart(); - Timestamp end = tsRange.getEnd(); - - return Timestamps.compare(start, v.getTimestamp()) <= 0 - && Timestamps.compare(v.getTimestamp(), end) <= 0; - }) - .collect(Collectors.toList()); - } - @AllArgsConstructor @Getter private static class GetRequest { diff --git a/serving/src/main/java/feast/serving/util/RequestHelper.java b/serving/src/main/java/feast/serving/util/RequestHelper.java index ed7d7194a0..c5e915dc27 100644 --- a/serving/src/main/java/feast/serving/util/RequestHelper.java +++ b/serving/src/main/java/feast/serving/util/RequestHelper.java @@ -18,8 +18,6 @@ package feast.serving.util; import com.google.common.base.Strings; -import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; import feast.serving.ServingAPIProto.QueryFeaturesRequest; public class RequestHelper { @@ -49,28 +47,5 @@ public static void validateRequest(QueryFeaturesRequest request) { "entity name of all feature ID in request details must be: " + entityName); } } - - checkTimestampRange(request); - } - - private static void checkTimestampRange(QueryFeaturesRequest request) { - if (!request.hasTimeRange()) { - return; - } - - Timestamp start = request.getTimeRange().getStart(); - Timestamp end = request.getTimeRange().getEnd(); - if (Timestamps.EPOCH.equals(start)) { - throw new IllegalArgumentException("start time must not be empty"); - } - - if (Timestamps.EPOCH.equals(end)) { - throw new IllegalArgumentException("end time must not be empty"); - } - - if (Timestamps.compare(start, end) > 0) { - throw new IllegalArgumentException( - "'end' of timestampRange must be before or same time as 'start'"); - } } } diff --git a/serving/src/main/java/feast/serving/util/TimeUtil.java b/serving/src/main/java/feast/serving/util/TimeUtil.java index 643c544e0f..24c4aa8950 100644 --- a/serving/src/main/java/feast/serving/util/TimeUtil.java +++ b/serving/src/main/java/feast/serving/util/TimeUtil.java @@ -17,19 +17,12 @@ package feast.serving.util; -import feast.serving.ServingAPIProto.TimestampRange; - /** Utility class for time-related function. */ public class TimeUtil { - public static final int NANO_IN_MILLI = 1000000; public static final int NANO_IN_MICRO = 1000; - public static final int MILLI_IN_SECOND = 1000; private TimeUtil() {} - public static boolean isTimeFilterRequired(TimestampRange timeRange) { - return timeRange != null && !TimestampRange.getDefaultInstance().equals(timeRange); - } /** * Returns the current value of the running Java Virtual Machine's high-resolution time source, in diff --git a/serving/src/test/java/feast/serving/grpc/FeastServingTest.java b/serving/src/test/java/feast/serving/grpc/FeastServingTest.java index ca0a10fa7e..3ba9c2b735 100644 --- a/serving/src/test/java/feast/serving/grpc/FeastServingTest.java +++ b/serving/src/test/java/feast/serving/grpc/FeastServingTest.java @@ -23,11 +23,8 @@ import static org.junit.Assert.assertThat; import static org.mockito.Mockito.verify; -import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; import feast.serving.ServingAPIProto.QueryFeaturesRequest; import feast.serving.ServingAPIProto.QueryFeaturesResponse; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.service.FeastServing; import feast.serving.service.FeatureRetrievalDispatcher; import feast.serving.service.FeatureStorageRegistry; @@ -66,16 +63,10 @@ public void setUp() throws Exception { @Test public void shouldReturnSameEntityNameAsRequest() { String entityName = "driver"; - TimestampRange tsRange = - TimestampRange.newBuilder() - .setStart(Timestamps.EPOCH) - .setEnd(Timestamp.newBuilder().setSeconds(1234).build()) - .build(); QueryFeaturesRequest request = QueryFeaturesRequest.newBuilder() .setEntityName(entityName) .addFeatureId("driver.day.total_completed_booking") - .setTimeRange(tsRange) .build(); QueryFeaturesResponse response = feast.queryFeatures(request); @@ -89,36 +80,26 @@ public void shouldPassValidRequestToFeatureRetrievalDispatcher() { String entityName = "driver"; Collection entityIds = Arrays.asList("entity1", "entity2", "entity3"); Collection featureIds = Arrays.asList("driver.day.total_completed_booking"); - TimestampRange tsRange = - TimestampRange.newBuilder() - .setStart(Timestamps.EPOCH) - .setEnd(Timestamp.newBuilder().setSeconds(1234).build()) - .build(); QueryFeaturesRequest request = QueryFeaturesRequest.newBuilder() .setEntityName(entityName) .addAllEntityId(entityIds) .addAllFeatureId(featureIds) - .setTimeRange(tsRange) .build(); ArgumentCaptor entityNameArg = ArgumentCaptor.forClass(String.class); ArgumentCaptor> entityIdsArg = ArgumentCaptor.forClass(List.class); - ArgumentCaptor> featureSpecArg = ArgumentCaptor.forClass(Collection.class); - ArgumentCaptor tsRangeArg = ArgumentCaptor.forClass(TimestampRange.class); + ArgumentCaptor> featureSpecArg = + ArgumentCaptor.forClass(Collection.class); QueryFeaturesResponse response = feast.queryFeatures(request); verify(featureRetrievalDispatcher) .dispatchFeatureRetrieval( - entityNameArg.capture(), - entityIdsArg.capture(), - featureSpecArg.capture(), - tsRangeArg.capture()); + entityNameArg.capture(), entityIdsArg.capture(), featureSpecArg.capture()); assertNotNull(response); assertThat(response.getEntityName(), equalTo(entityName)); assertThat(entityNameArg.getValue(), equalTo(entityName)); assertThat(entityIdsArg.getValue(), containsInAnyOrder(entityIds.toArray())); - assertThat(tsRangeArg.getValue(), equalTo(tsRange)); } } diff --git a/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java b/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java index 9e4e6cea3a..75c80f8aa2 100644 --- a/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java +++ b/serving/src/test/java/feast/serving/grpc/ServingGrpcServiceTest.java @@ -20,12 +20,9 @@ import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.verify; -import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; import com.timgroup.statsd.StatsDClient; import feast.serving.ServingAPIProto.QueryFeaturesRequest; import feast.serving.ServingAPIProto.QueryFeaturesResponse; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.service.FeastServing; import io.grpc.StatusRuntimeException; import io.grpc.stub.StreamObserver; @@ -53,19 +50,12 @@ public class ServingGrpcServiceTest { public void setUp() throws Exception { MockitoAnnotations.initMocks(this); - TimestampRange tsRange = - TimestampRange.newBuilder() - .setStart(Timestamp.newBuilder().setSeconds(10).build()) - .setEnd(Timestamp.newBuilder().setSeconds(11).build()) - .build(); - validRequest = QueryFeaturesRequest.newBuilder() .setEntityName("driver") .addAllEntityId(Arrays.asList("driver1", "driver2", "driver3")) .addAllFeatureId( Arrays.asList("driver.day.completed_booking", "driver.none.last_opportunity")) - .setTimeRange(tsRange) .build(); Tracer tracer = Configuration.fromEnv("dummy").getTracer(); @@ -119,58 +109,4 @@ public void shouldCallOnErrorIfFeatureIdsContainsDifferentEntity() { verify(mockStreamObserver).onError(any(StatusRuntimeException.class)); } - - @Test - public void shouldCallOnErrorIfTimestampRangeStartIsAfterEnd() { - TimestampRange invalidTsRange = - TimestampRange.newBuilder() - .setStart(Timestamps.fromSeconds(1001)) - .setEnd(Timestamps.fromSeconds(1000)) - .build(); - - QueryFeaturesRequest invalidTsRangeReq = - QueryFeaturesRequest.newBuilder(validRequest).setTimeRange(invalidTsRange).build(); - - service.queryFeatures(invalidTsRangeReq, mockStreamObserver); - - verify(mockStreamObserver).onError(any(StatusRuntimeException.class)); - } - - @Test - public void shouldCallOnErrorIfTimestampRangeDoesntHaveStartAndEnd() { - TimestampRange invalidTsRange = TimestampRange.newBuilder().build(); - - QueryFeaturesRequest invalidTsRangeReq = - QueryFeaturesRequest.newBuilder(validRequest).setTimeRange(invalidTsRange).build(); - - service.queryFeatures(invalidTsRangeReq, mockStreamObserver); - - verify(mockStreamObserver).onError(any(StatusRuntimeException.class)); - } - - @Test - public void shouldCallOnErrorIfTimestampRangeDoesntHaveStart() { - TimestampRange invalidTsRange = - TimestampRange.newBuilder().setEnd(Timestamps.fromSeconds(1001)).build(); - - QueryFeaturesRequest invalidTsRangeReq = - QueryFeaturesRequest.newBuilder(validRequest).setTimeRange(invalidTsRange).build(); - - service.queryFeatures(invalidTsRangeReq, mockStreamObserver); - - verify(mockStreamObserver).onError(any(StatusRuntimeException.class)); - } - - @Test - public void shouldCallOnErrorIfTimestampRangeDoesntHaveEnd() { - TimestampRange invalidTsRange = - TimestampRange.newBuilder().setStart(Timestamps.fromSeconds(1001)).build(); - - QueryFeaturesRequest invalidTsRangeReq = - QueryFeaturesRequest.newBuilder(validRequest).setTimeRange(invalidTsRange).build(); - - service.queryFeatures(invalidTsRangeReq, mockStreamObserver); - - verify(mockStreamObserver).onError(any(StatusRuntimeException.class)); - } } diff --git a/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java b/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java index 82f4edf94e..f8905e6342 100644 --- a/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java +++ b/serving/src/test/java/feast/serving/service/BigTableFeatureStorageTestITCase.java @@ -20,9 +20,6 @@ import com.google.cloud.bigtable.hbase.BigtableConfiguration; import com.google.cloud.bigtable.hbase.BigtableOptionsFactory; import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Durations; -import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.model.FeatureValue; import feast.serving.testutil.BigTablePopulator; import feast.specs.FeatureSpecProto.FeatureSpec; @@ -31,7 +28,6 @@ import feast.types.ValueProto.ValueType; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.client.Connection; @@ -91,13 +87,9 @@ public void getFeatures_shouldWorkForGranularityNone() { List featureSpecs = Arrays.asList(featureSpec1, featureSpec2); bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, now); - List results = - featureStorage.getFeature(ENTITY_NAME, entityIds, featureSpecs, null); - List results2 = - featureStorage.getFeature(ENTITY_NAME, entityIds, featureSpecs, TimestampRange.getDefaultInstance()); + List results = featureStorage.getFeature(ENTITY_NAME, entityIds, featureSpecs); - bigTablePopulator.validate(results, entityIds, featureSpecs, null); - bigTablePopulator.validate(results2, entityIds, featureSpecs, null); + bigTablePopulator.validate(results, entityIds, featureSpecs); } @Test @@ -125,8 +117,8 @@ public void getFeatures_shouldGracefullyHandleMissingEntity() { entityIdsWithMissingEntity.add("100"); bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, now); List results = - featureStorage.getFeature(ENTITY_NAME, entityIdsWithMissingEntity, featureSpecs, null); - bigTablePopulator.validate(results, entityIds, featureSpecs, null); + featureStorage.getFeature(ENTITY_NAME, entityIdsWithMissingEntity, featureSpecs); + bigTablePopulator.validate(results, entityIds, featureSpecs); } @Test @@ -141,40 +133,12 @@ public void getFeatures_shouldWorkForOtherGranularity() { List featureSpecs = Arrays.asList(spec1, spec2); bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, now); - List result = - featureStorage.getFeature(ENTITY_NAME, entityIds, featureSpecs, null); + List result = featureStorage.getFeature(ENTITY_NAME, entityIds, featureSpecs); - bigTablePopulator.validate(result, entityIds, featureSpecs, null); + bigTablePopulator.validate(result, entityIds, featureSpecs); } } - @Test - public void getFeatures_shouldFilterOutOldData() { - String entityIdWithOldData = "entity_old_data"; - Granularity.Enum granularity = Enum.MINUTE; - Timestamp fiveMinutesAgo = Timestamps.subtract(now, Durations.fromSeconds(300)); - - Timestamp start = Timestamps.subtract(now, Durations.fromSeconds(60)); - Timestamp end = now; - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - - FeatureSpec spec1 = createFeatureSpec("feature_1", granularity, ValueType.Enum.STRING); - FeatureSpec spec2 = createFeatureSpec("feature_2", granularity, ValueType.Enum.STRING); - - List featureSpecs = Arrays.asList(spec1, spec2); - bigTablePopulator.populate(ENTITY_NAME, entityIds, featureSpecs, now); - bigTablePopulator.populate( - ENTITY_NAME, Collections.singletonList(entityIdWithOldData), featureSpecs, fiveMinutesAgo); - - List allEntityIds = new ArrayList<>(entityIds); - allEntityIds.add(entityIdWithOldData); - List result = - featureStorage.getFeature(ENTITY_NAME, allEntityIds, featureSpecs, tsRange); - System.out.println(result); - - bigTablePopulator.validate(result, entityIds, featureSpecs, tsRange); - } - private FeatureSpec createFeatureSpec( String featureName, Enum granularity, ValueType.Enum valType) { String entityName = ENTITY_NAME; diff --git a/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java b/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java index b02671ee6f..11b3f8fa87 100644 --- a/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java +++ b/serving/src/test/java/feast/serving/service/FeatureRetrievalDispatcherTest.java @@ -27,8 +27,6 @@ import com.google.common.util.concurrent.ListeningExecutorService; import com.google.common.util.concurrent.MoreExecutors; -import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.config.AppConfig; import feast.specs.FeatureSpecProto.DataStore; import feast.specs.FeatureSpecProto.DataStores; @@ -67,19 +65,18 @@ public void setUp() throws Exception { public void shouldUseCurrentThreadIfRequestIsSmallEnough() { String entityName = "entity"; FeatureStorage featureStorage = mock(FeatureStorage.class); - when(featureStorage.getFeature( - any(String.class), any(List.class), any(List.class), any(TimestampRange.class))) + when(featureStorage.getFeature(any(String.class), any(List.class), any(List.class))) .thenReturn(Collections.emptyList()); when(featureStorageRegistry.get(any(String.class))).thenReturn(featureStorage); String featureId = "entity.none.feature_1"; FeatureSpec featureSpec = FeatureSpec.newBuilder().setId(featureId).build(); dispatcher.dispatchFeatureRetrieval( - entityName, entityIds, Collections.singletonList(featureSpec), null); + entityName, entityIds, Collections.singletonList(featureSpec)); verifyZeroInteractions(executorService); verify(featureStorage) - .getFeature(entityName, entityIds, Collections.singletonList(featureSpec), null); + .getFeature(entityName, entityIds, Collections.singletonList(featureSpec)); } @Test @@ -89,8 +86,7 @@ public void shouldUseCurrentThreadIfRequestFromSameStorage() { FeatureStorage redis1 = mock(FeatureStorage.class); when(featureStorageRegistry.get(storageId1)).thenReturn(redis1); - when(redis1.getFeature( - any(String.class), any(List.class), any(List.class), any(TimestampRange.class))) + when(redis1.getFeature(any(String.class), any(List.class), any(List.class))) .thenReturn(Collections.emptyList()); String entityName = "entity"; @@ -110,23 +106,12 @@ public void shouldUseCurrentThreadIfRequestFromSameStorage() { DataStores.newBuilder().setServing(DataStore.newBuilder().setId(storageId1))) .build(); - TimestampRange tsRange = - TimestampRange.newBuilder() - .setStart(Timestamps.fromSeconds(0)) - .setEnd(Timestamps.fromSeconds(10)) - .build(); dispatcher.dispatchFeatureRetrieval( entityName, entityIds, - Arrays.asList(featureSpec1, featureSpec2), - tsRange); - - verify(redis1) - .getFeature( - entityName, - entityIds, - Arrays.asList(featureSpec1, featureSpec2), - tsRange); + Arrays.asList(featureSpec1, featureSpec2)); + + verify(redis1).getFeature(entityName, entityIds, Arrays.asList(featureSpec1, featureSpec2)); verifyZeroInteractions(executorService); } @@ -140,11 +125,9 @@ public void shouldUseExecutorServiceIfRequestFromMoreThanOneStorage() { when(featureStorageRegistry.get(storageId1)).thenReturn(redis1); when(featureStorageRegistry.get(storageId2)).thenReturn(redis2); - when(redis1.getFeature( - any(String.class), any(List.class), any(List.class), any(TimestampRange.class))) + when(redis1.getFeature(any(String.class), any(List.class), any(List.class))) .thenReturn(Collections.emptyList()); - when(redis2.getFeature( - any(String.class), any(List.class), any(List.class), any(TimestampRange.class))) + when(redis2.getFeature(any(String.class), any(List.class), any(List.class))) .thenReturn(Collections.emptyList()); String entityName = "entity"; @@ -165,10 +148,10 @@ public void shouldUseExecutorServiceIfRequestFromMoreThanOneStorage() { .build(); dispatcher.dispatchFeatureRetrieval( - entityName, entityIds, Arrays.asList(featureSpec1, featureSpec2), null); + entityName, entityIds, Arrays.asList(featureSpec1, featureSpec2)); - verify(redis1).getFeature(entityName, entityIds, Collections.singletonList(featureSpec1), null); - verify(redis2).getFeature(entityName, entityIds, Collections.singletonList(featureSpec2), null); + verify(redis1).getFeature(entityName, entityIds, Collections.singletonList(featureSpec1)); + verify(redis2).getFeature(entityName, entityIds, Collections.singletonList(featureSpec2)); verify(executorService, atLeast(2)).submit(any(Callable.class)); } diff --git a/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java b/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java index 7c6dd43e48..be70fee48d 100644 --- a/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java +++ b/serving/src/test/java/feast/serving/service/RedisFeatureStorageTest.java @@ -20,9 +20,6 @@ import static junit.framework.TestCase.fail; import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Durations; -import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.model.FeatureValue; import feast.serving.testutil.RedisPopulator; import feast.specs.FeatureSpecProto.DataStore; @@ -34,7 +31,6 @@ import io.opentracing.util.GlobalTracer; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.List; import org.junit.After; import org.junit.Before; @@ -102,9 +98,8 @@ public void getFeatures_shouldNotReturnMissingValue() { // add entity without feature List requestEntityIds = new ArrayList<>(entityIds); requestEntityIds.add("100"); - List result = - redisFs.getFeature(entityName, requestEntityIds, featureSpecs, null); - redisPopulator.validate(result, entityIds, featureSpecs, null); + List result = redisFs.getFeature(entityName, requestEntityIds, featureSpecs); + redisPopulator.validate(result, entityIds, featureSpecs); } @Test @@ -119,39 +114,12 @@ public void getFeatures_shouldReturnLastValue() { redisPopulator.populate(entityName, entityIds, featureSpecs, now); - List result = redisFs.getFeature(entityName, entityIds, featureSpecs, null); - List result2 = redisFs.getFeature(entityName, entityIds, featureSpecs, TimestampRange.getDefaultInstance()); + List result = redisFs.getFeature(entityName, entityIds, featureSpecs); - redisPopulator.validate(result, entityIds, featureSpecs, null); - redisPopulator.validate(result2, entityIds, featureSpecs, null); + redisPopulator.validate(result, entityIds, featureSpecs); } } - @Test - public void getFeatures_shouldReturnLastValueFilteredByTimestampIfSpecified() { - String entityIdWithOldData = "entity_old"; - Timestamp fiveMinutesAgo = Timestamps.subtract(now, Durations.fromSeconds(300)); - - Timestamp start = Timestamps.subtract(now, Durations.fromSeconds(60)); - Timestamp end = now; - TimestampRange tsRange = TimestampRange.newBuilder().setStart(start).setEnd(end).build(); - - Granularity.Enum granularity = Enum.SECOND; - FeatureSpec spec1 = createFeatureSpec("feature_1", granularity); - FeatureSpec spec2 = createFeatureSpec("feature_2", granularity); - List featureSpecs = Arrays.asList(spec1, spec2); - - redisPopulator.populate(entityName, entityIds, featureSpecs, now); - redisPopulator.populate( - entityName, Collections.singletonList(entityIdWithOldData), featureSpecs, fiveMinutesAgo); - - List allEntityIds = new ArrayList<>(entityIds); - allEntityIds.add(entityIdWithOldData); - - List result = redisFs.getFeature(entityName, allEntityIds, featureSpecs, tsRange); - redisPopulator.validate(result, entityIds, featureSpecs, tsRange); - } - private FeatureSpec createFeatureSpec(String featureName, Enum granularity) { DataStore servingDatastoreSpec = DataStore.newBuilder().setId("REDIS").build(); return createFeatureSpec(featureName, granularity, ValueType.Enum.STRING, servingDatastoreSpec); diff --git a/serving/src/test/java/feast/serving/testutil/FeatureStoragePopulator.java b/serving/src/test/java/feast/serving/testutil/FeatureStoragePopulator.java index 4cd851b9e4..823be09d21 100644 --- a/serving/src/test/java/feast/serving/testutil/FeatureStoragePopulator.java +++ b/serving/src/test/java/feast/serving/testutil/FeatureStoragePopulator.java @@ -20,15 +20,11 @@ import static java.util.stream.Collectors.groupingBy; import static org.apache.hadoop.hbase.shaded.org.junit.Assert.assertNotNull; import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.lessThanOrEqualTo; import static org.junit.Assert.assertThat; import com.google.protobuf.Timestamp; -import com.google.protobuf.util.Timestamps; -import feast.serving.ServingAPIProto.TimestampRange; import feast.serving.model.FeatureValue; import feast.specs.FeatureSpecProto.FeatureSpec; -import feast.types.GranularityProto.Granularity.Enum; import feast.types.ValueProto.Value; import feast.types.ValueProto.ValueType; import java.util.Collection; @@ -75,10 +71,7 @@ protected Value createValue( } public void validate( - List result, - List entityIds, - List featureSpecs, - TimestampRange tsRange) { + List result, List entityIds, List featureSpecs) { Map>> entityMap = toEntityMap(result); assertNotNull(entityMap); @@ -96,14 +89,6 @@ public void validate( FeatureValue featureValue = featureValueList.get(0); Timestamp timestamp = featureValue.getTimestamp(); validateValue(featureValue, entityId, featureSpec, timestamp); - - if (tsRange != null && !featureSpec.getGranularity().equals(Enum.NONE)) { - Timestamp start = tsRange.getStart(); - Timestamp end = tsRange.getEnd(); - - assertThat(Timestamps.compare(start, timestamp), lessThanOrEqualTo(0)); - assertThat(Timestamps.compare(timestamp, end), lessThanOrEqualTo(0)); - } } } } From bdacb855aa4d96cce1f74b52ed5978ba3b65cbcb Mon Sep 17 00:00:00 2001 From: Pradithya Aria Date: Tue, 19 Feb 2019 08:04:11 +0800 Subject: [PATCH 6/8] Include timestamp filter in Python sdk --- sdk/python/feast/sdk/client.py | 61 ++++++++++-------- sdk/python/feast/serving/Serving_pb2.py | 86 +++++-------------------- sdk/python/tests/sdk/test_client.py | 75 +++++++++++++++------ 3 files changed, 104 insertions(+), 118 deletions(-) diff --git a/sdk/python/feast/sdk/client.py b/sdk/python/feast/sdk/client.py index 285a864c2a..38256b1705 100644 --- a/sdk/python/feast/sdk/client.py +++ b/sdk/python/feast/sdk/client.py @@ -18,6 +18,7 @@ import os from datetime import datetime +import dateutil.parser import grpc import pandas as pd @@ -36,7 +37,7 @@ from feast.sdk.resources.storage import Storage from feast.sdk.utils.bq_util import TableDownloader from feast.sdk.utils.print_utils import spec_to_yaml -from feast.serving.Serving_pb2 import QueryFeaturesRequest, TimestampRange +from feast.serving.Serving_pb2 import QueryFeaturesRequest from feast.serving.Serving_pb2_grpc import ServingAPIStub @@ -110,7 +111,6 @@ def verbose(self, val): raise TypeError("verbose should be a boolean value") self._verbose = val - def apply(self, obj): """Create or update one or many feast's resource (feature, entity, importer, storage). @@ -167,7 +167,7 @@ def run(self, importer, name_override=None, return response.jobId def create_dataset(self, feature_set, start_date, end_date, - limit=None, name_prefix=None): + limit=None, name_prefix=None): """ Create training dataset for a feature set. The training dataset will be bounded by event timestamp between start_date and end_date. @@ -189,7 +189,7 @@ def create_dataset(self, feature_set, start_date, end_date, the information of training dataset """ self._check_create_dataset_args(feature_set, start_date, - end_date, limit) + end_date, limit) req = DatasetServiceTypes.CreateDatasetRequest( featureSet=feature_set.proto, @@ -227,11 +227,10 @@ def get_serving_data(self, feature_set, entity_keys, ts_range=None): Returns: pandas.DataFrame: DataFrame of results """ - request = self._build_serving_request(feature_set, entity_keys, - ts_range) + request = self._build_serving_request(feature_set, entity_keys) self._connect_serving() return self._response_to_df(feature_set, self._serving_service_stub - .QueryFeatures(request)) + .QueryFeatures(request), ts_range) def download_dataset(self, dataset_info, dest, staging_location, file_type=FileType.CSV): @@ -285,7 +284,8 @@ def _connect_core(self): self.__core_channel = grpc.insecure_channel(self.core_url) self._core_service_stub = CoreServiceStub(self.__core_channel) self._job_service_stub = JobServiceStub(self.__core_channel) - self._dataset_service_stub = DatasetServiceStub(self.__core_channel) + self._dataset_service_stub = DatasetServiceStub( + self.__core_channel) def _connect_serving(self): """Connect to serving api""" @@ -293,31 +293,32 @@ def _connect_serving(self): self.__serving_channel = grpc.insecure_channel(self.serving_url) self._serving_service_stub = ServingAPIStub(self.__serving_channel) - def _build_serving_request(self, feature_set, entity_keys, ts_range): + def _build_serving_request(self, feature_set, entity_keys): """Helper function to build serving service request.""" + return QueryFeaturesRequest(entityName=feature_set.entity, + entityId=entity_keys, + featureId=feature_set.features) + + def _response_to_df(self, feature_set, response, ts_range = None): + start = None + end = None if ts_range is not None: if len(ts_range) != 2: raise ValueError("ts_range must have len 2") + start = dateutil.parser.parse(ts_range[0]) + end = dateutil.parser.parse(ts_range[1]) - start = Timestamp() - end = Timestamp() - start.FromJsonString(ts_range[0]) - end.FromJsonString(ts_range[1]) - ts_range = TimestampRange(start=start, end=end) - request = QueryFeaturesRequest(entityName=feature_set.entity, - entityId=entity_keys, - featureId=feature_set.features, - timeRange=ts_range) - return request - - def _response_to_df(self, feature_set, response): - df = pd.DataFrame(columns=[feature_set.entity] + - feature_set.features) + df = pd.DataFrame(columns=[feature_set.entity] + feature_set.features) for entity_id in response.entities: feature_map = response.entities[entity_id].features row = {response.entityName: entity_id} for feature_id in feature_map: v = feature_map[feature_id].value + if ts_range is not None and not _is_granularity_none( + feature_id): + ts = feature_map[feature_id].timestamp.ToDatetime() + if ts < start or ts > end: + continue v = getattr(v, v.WhichOneof("val")) row[feature_id] = v df = df.append(row, ignore_index=True) @@ -350,8 +351,9 @@ def _apply_feature(self, feature): """ self._connect_core() response = self._core_service_stub.ApplyFeature(feature.spec) - if self.verbose: print("Successfully applied feature with id: {}\n---\n{}" - .format(response.featureId, feature)) + if self.verbose: print( + "Successfully applied feature with id: {}\n---\n{}" + .format(response.featureId, feature)) return response.featureId def _apply_entity(self, entity): @@ -375,7 +377,8 @@ def _apply_feature_group(self, feature_group): feature group to apply """ self._connect_core() - response = self._core_service_stub.ApplyFeatureGroup(feature_group.spec) + response = self._core_service_stub.ApplyFeatureGroup( + feature_group.spec) if self.verbose: print("Successfully applied feature group with id: " + "{}\n---\n{}".format(response.featureGroupId, feature_group)) @@ -394,7 +397,7 @@ def _apply_storage(self, storage): return response.storageId def _check_create_dataset_args(self, feature_set, start_date, - end_date, limit): + end_date, limit): if len(feature_set.features) < 1: raise ValueError("feature set is empty") @@ -426,3 +429,7 @@ def _timestamp_from_datetime(dt): ts = Timestamp() ts.FromDatetime(dt) return ts + + +def _is_granularity_none(feature_id): + return feature_id.split(".")[1] == "none" diff --git a/sdk/python/feast/serving/Serving_pb2.py b/sdk/python/feast/serving/Serving_pb2.py index 9aab187c2b..4f7900e2c3 100644 --- a/sdk/python/feast/serving/Serving_pb2.py +++ b/sdk/python/feast/serving/Serving_pb2.py @@ -21,7 +21,7 @@ package='feast.serving', syntax='proto3', serialized_options=_b('\n\rfeast.servingB\017ServingAPIProtoZ8github.com/gojek/feast/protos/generated/go/feast/serving'), - serialized_pb=_b('\n\x1b\x66\x65\x61st/serving/Serving.proto\x12\rfeast.serving\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17\x66\x65\x61st/types/Value.proto\"\x81\x01\n\x14QueryFeaturesRequest\x12\x12\n\nentityName\x18\x01 \x01(\t\x12\x10\n\x08\x65ntityId\x18\x02 \x03(\t\x12\x11\n\tfeatureId\x18\x03 \x03(\t\x12\x30\n\ttimeRange\x18\x04 \x01(\x0b\x32\x1d.feast.serving.TimestampRange\"\xb9\x01\n\x15QueryFeaturesResponse\x12\x12\n\nentityName\x18\x01 \x01(\t\x12\x44\n\x08\x65ntities\x18\x02 \x03(\x0b\x32\x32.feast.serving.QueryFeaturesResponse.EntitiesEntry\x1a\x46\n\rEntitiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.feast.serving.Entity:\x02\x38\x01\"d\n\x0eTimestampRange\x12)\n\x05start\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\'\n\x03\x65nd\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\x8d\x01\n\x06\x45ntity\x12\x35\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32#.feast.serving.Entity.FeaturesEntry\x1aL\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.feast.serving.FeatureValue:\x02\x38\x01\"`\n\x0c\x46\x65\x61tureValue\x12!\n\x05value\x18\x01 \x01(\x0b\x32\x12.feast.types.Value\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp2j\n\nServingAPI\x12\\\n\rQueryFeatures\x12#.feast.serving.QueryFeaturesRequest\x1a$.feast.serving.QueryFeaturesResponse\"\x00\x42Z\n\rfeast.servingB\x0fServingAPIProtoZ8github.com/gojek/feast/protos/generated/go/feast/servingb\x06proto3') + serialized_pb=_b('\n\x1b\x66\x65\x61st/serving/Serving.proto\x12\rfeast.serving\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17\x66\x65\x61st/types/Value.proto\"O\n\x14QueryFeaturesRequest\x12\x12\n\nentityName\x18\x01 \x01(\t\x12\x10\n\x08\x65ntityId\x18\x02 \x03(\t\x12\x11\n\tfeatureId\x18\x03 \x03(\t\"\xb9\x01\n\x15QueryFeaturesResponse\x12\x12\n\nentityName\x18\x01 \x01(\t\x12\x44\n\x08\x65ntities\x18\x02 \x03(\x0b\x32\x32.feast.serving.QueryFeaturesResponse.EntitiesEntry\x1a\x46\n\rEntitiesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12$\n\x05value\x18\x02 \x01(\x0b\x32\x15.feast.serving.Entity:\x02\x38\x01\"\x8d\x01\n\x06\x45ntity\x12\x35\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32#.feast.serving.Entity.FeaturesEntry\x1aL\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x1b.feast.serving.FeatureValue:\x02\x38\x01\"`\n\x0c\x46\x65\x61tureValue\x12!\n\x05value\x18\x01 \x01(\x0b\x32\x12.feast.types.Value\x12-\n\ttimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp2j\n\nServingAPI\x12\\\n\rQueryFeatures\x12#.feast.serving.QueryFeaturesRequest\x1a$.feast.serving.QueryFeaturesResponse\"\x00\x42Z\n\rfeast.servingB\x0fServingAPIProtoZ8github.com/gojek/feast/protos/generated/go/feast/servingb\x06proto3') , dependencies=[google_dot_protobuf_dot_timestamp__pb2.DESCRIPTOR,feast_dot_types_dot_Value__pb2.DESCRIPTOR,]) @@ -56,13 +56,6 @@ message_type=None, enum_type=None, containing_type=None, is_extension=False, extension_scope=None, serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='timeRange', full_name='feast.serving.QueryFeaturesRequest.timeRange', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), ], extensions=[ ], @@ -75,8 +68,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=105, - serialized_end=234, + serialized_start=104, + serialized_end=183, ) @@ -113,8 +106,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=352, - serialized_end=422, + serialized_start=301, + serialized_end=371, ) _QUERYFEATURESRESPONSE = _descriptor.Descriptor( @@ -150,46 +143,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=237, - serialized_end=422, -) - - -_TIMESTAMPRANGE = _descriptor.Descriptor( - name='TimestampRange', - full_name='feast.serving.TimestampRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='start', full_name='feast.serving.TimestampRange.start', index=0, - number=1, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - _descriptor.FieldDescriptor( - name='end', full_name='feast.serving.TimestampRange.end', index=1, - number=2, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - serialized_options=None, file=DESCRIPTOR), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - serialized_options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=424, - serialized_end=524, + serialized_start=186, + serialized_end=371, ) @@ -226,8 +181,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=592, - serialized_end=668, + serialized_start=439, + serialized_end=515, ) _ENTITY = _descriptor.Descriptor( @@ -256,8 +211,8 @@ extension_ranges=[], oneofs=[ ], - serialized_start=527, - serialized_end=668, + serialized_start=374, + serialized_end=515, ) @@ -294,16 +249,13 @@ extension_ranges=[], oneofs=[ ], - serialized_start=670, - serialized_end=766, + serialized_start=517, + serialized_end=613, ) -_QUERYFEATURESREQUEST.fields_by_name['timeRange'].message_type = _TIMESTAMPRANGE _QUERYFEATURESRESPONSE_ENTITIESENTRY.fields_by_name['value'].message_type = _ENTITY _QUERYFEATURESRESPONSE_ENTITIESENTRY.containing_type = _QUERYFEATURESRESPONSE _QUERYFEATURESRESPONSE.fields_by_name['entities'].message_type = _QUERYFEATURESRESPONSE_ENTITIESENTRY -_TIMESTAMPRANGE.fields_by_name['start'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP -_TIMESTAMPRANGE.fields_by_name['end'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP _ENTITY_FEATURESENTRY.fields_by_name['value'].message_type = _FEATUREVALUE _ENTITY_FEATURESENTRY.containing_type = _ENTITY _ENTITY.fields_by_name['features'].message_type = _ENTITY_FEATURESENTRY @@ -311,7 +263,6 @@ _FEATUREVALUE.fields_by_name['timestamp'].message_type = google_dot_protobuf_dot_timestamp__pb2._TIMESTAMP DESCRIPTOR.message_types_by_name['QueryFeaturesRequest'] = _QUERYFEATURESREQUEST DESCRIPTOR.message_types_by_name['QueryFeaturesResponse'] = _QUERYFEATURESRESPONSE -DESCRIPTOR.message_types_by_name['TimestampRange'] = _TIMESTAMPRANGE DESCRIPTOR.message_types_by_name['Entity'] = _ENTITY DESCRIPTOR.message_types_by_name['FeatureValue'] = _FEATUREVALUE _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -338,13 +289,6 @@ _sym_db.RegisterMessage(QueryFeaturesResponse) _sym_db.RegisterMessage(QueryFeaturesResponse.EntitiesEntry) -TimestampRange = _reflection.GeneratedProtocolMessageType('TimestampRange', (_message.Message,), dict( - DESCRIPTOR = _TIMESTAMPRANGE, - __module__ = 'feast.serving.Serving_pb2' - # @@protoc_insertion_point(class_scope:feast.serving.TimestampRange) - )) -_sym_db.RegisterMessage(TimestampRange) - Entity = _reflection.GeneratedProtocolMessageType('Entity', (_message.Message,), dict( FeaturesEntry = _reflection.GeneratedProtocolMessageType('FeaturesEntry', (_message.Message,), dict( @@ -378,8 +322,8 @@ file=DESCRIPTOR, index=0, serialized_options=None, - serialized_start=768, - serialized_end=874, + serialized_start=615, + serialized_end=721, methods=[ _descriptor.MethodDescriptor( name='QueryFeatures', diff --git a/sdk/python/tests/sdk/test_client.py b/sdk/python/tests/sdk/test_client.py index 5f2f42e7ff..0caf889521 100644 --- a/sdk/python/tests/sdk/test_client.py +++ b/sdk/python/tests/sdk/test_client.py @@ -37,8 +37,8 @@ from feast.sdk.resources.feature_set import FeatureSet, DatasetInfo, FileType from feast.sdk.resources.storage import Storage from feast.sdk.utils.bq_util import TableDownloader -from feast.serving.Serving_pb2 import TimestampRange, \ - QueryFeaturesRequest, QueryFeaturesResponse, FeatureValue +from feast.serving.Serving_pb2 import QueryFeaturesRequest, \ + QueryFeaturesResponse, FeatureValue from feast.specs.FeatureSpec_pb2 import FeatureSpec, DataStores, DataStore from feast.specs.ImportSpec_pb2 import ImportSpec from feast.specs.StorageSpec_pb2 import StorageSpec @@ -268,26 +268,10 @@ def test_build_serving_request(self, client): feature_set = FeatureSet("entity", ["entity.none.feat1", "entity.none.feat2"]) - req = client._build_serving_request(feature_set, ["1", "2", "3"], None) + req = client._build_serving_request(feature_set, ["1", "2", "3"]) expected = QueryFeaturesRequest(entityName="entity", entityId=["1", "2", "3"], - featureId=feature_set.features, - timeRange=None) - assert req == expected - - tsRange = ["2018-03-03T00:23:00Z", "2018-03-03T00:23:00Z"] - start = Timestamp() - start.FromJsonString(tsRange[0]) - end = Timestamp() - end.FromJsonString(tsRange[1]) - expTimeRange = TimestampRange(start=start, end=end) - expected = QueryFeaturesRequest(entityName="entity", - entityId=["1", "2", "3"], - featureId=feature_set.features, - timeRange=expTimeRange) - - req = client._build_serving_request(feature_set, ["1", "2", "3"], - tsRange) + featureId=feature_set.features) assert req == expected def test_serving_response_to_df(self, client): @@ -366,6 +350,57 @@ def test_serving_response_to_df_no_data(self, client): check_column_type=False, check_index_type=False) + def test_serving_response_to_df_with_time_filter(self, client): + response = self._create_query_features_response( + entity_name="entity", + entities={"1": {"entity.feat1": (1, Timestamp(seconds=1))}, + "2": {"entity.feat1": (3, Timestamp(seconds=3))}} + ) + expected_df = pd.DataFrame({'entity': ["1", "2"], + 'entity.feat1': [np.NaN, 3], + 'entity.feat2': [np.NaN, np.NaN]}) \ + .reset_index(drop=True) + start = datetime.utcfromtimestamp(2).isoformat() + end = datetime.utcfromtimestamp(5).isoformat() + + ts_range = [start, end] + df = client._response_to_df(FeatureSet("entity", ["entity.feat1", + "entity.feat2"]), + response, ts_range) \ + .sort_values(['entity']) \ + .reset_index(drop=True)[expected_df.columns] + print(df) + assert_frame_equal(df, expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) + + def test_serving_response_to_df_with_time_filter_granularity_none(self, + client): + response = self._create_query_features_response( + entity_name="entity", + entities={"1": {"entity.none.feat1": (1, Timestamp(seconds=1))}, + "2": {"entity.none.feat1": (3, Timestamp(seconds=3))}} + ) + expected_df = pd.DataFrame({'entity': ["1", "2"], + 'entity.none.feat1': [1, 3], + 'entity.none.feat2': [np.NaN, np.NaN]}) \ + .reset_index(drop=True) + start = datetime.utcfromtimestamp(2).isoformat() + end = datetime.utcfromtimestamp(5).isoformat() + + ts_range = [start, end] + df = client._response_to_df(FeatureSet("entity", ["entity.none.feat1", + "entity.none.feat2"]), + response, ts_range) \ + .sort_values(['entity']) \ + .reset_index(drop=True)[expected_df.columns] + print(df) + assert_frame_equal(df, expected_df, + check_dtype=False, + check_column_type=False, + check_index_type=False) + def test_download_dataset_as_file(self, client, mocker): destination = "/tmp/dest_file" From 22b3d44f4f2a652830bdb62158e738f7959ab47a Mon Sep 17 00:00:00 2001 From: Pradithya Aria Date: Tue, 19 Feb 2019 08:05:06 +0800 Subject: [PATCH 7/8] Update generated go file --- .../go/feast/core/DatasetService.pb.go | 36 +++-- .../generated/go/feast/serving/Serving.pb.go | 132 +++++------------- .../generated/go/feast/specs/ImportSpec.pb.go | 24 ++-- .../generated/go/feast/storage/BigTable.pb.go | 12 +- protos/generated/go/feast/types/Feature.pb.go | 10 +- .../generated/go/feast/types/FeatureRow.pb.go | 18 ++- 6 files changed, 80 insertions(+), 152 deletions(-) diff --git a/protos/generated/go/feast/core/DatasetService.pb.go b/protos/generated/go/feast/core/DatasetService.pb.go index d2d3b01c93..d83ec1a1dc 100644 --- a/protos/generated/go/feast/core/DatasetService.pb.go +++ b/protos/generated/go/feast/core/DatasetService.pb.go @@ -34,7 +34,7 @@ func (m *DatasetServiceTypes) Reset() { *m = DatasetServiceTypes{} } func (m *DatasetServiceTypes) String() string { return proto.CompactTextString(m) } func (*DatasetServiceTypes) ProtoMessage() {} func (*DatasetServiceTypes) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{0} + return fileDescriptor_3edc37a8b0d37b39, []int{0} } func (m *DatasetServiceTypes) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetServiceTypes.Unmarshal(m, b) @@ -42,8 +42,8 @@ func (m *DatasetServiceTypes) XXX_Unmarshal(b []byte) error { func (m *DatasetServiceTypes) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetServiceTypes.Marshal(b, m, deterministic) } -func (dst *DatasetServiceTypes) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetServiceTypes.Merge(dst, src) +func (m *DatasetServiceTypes) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetServiceTypes.Merge(m, src) } func (m *DatasetServiceTypes) XXX_Size() int { return xxx_messageInfo_DatasetServiceTypes.Size(m) @@ -77,7 +77,7 @@ func (m *DatasetServiceTypes_CreateDatasetRequest) Reset() { func (m *DatasetServiceTypes_CreateDatasetRequest) String() string { return proto.CompactTextString(m) } func (*DatasetServiceTypes_CreateDatasetRequest) ProtoMessage() {} func (*DatasetServiceTypes_CreateDatasetRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{0, 0} + return fileDescriptor_3edc37a8b0d37b39, []int{0, 0} } func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Unmarshal(m, b) @@ -85,8 +85,8 @@ func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Unmarshal(b []byte) error func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Marshal(b, m, deterministic) } -func (dst *DatasetServiceTypes_CreateDatasetRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Merge(dst, src) +func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Merge(m, src) } func (m *DatasetServiceTypes_CreateDatasetRequest) XXX_Size() int { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetRequest.Size(m) @@ -146,7 +146,7 @@ func (m *DatasetServiceTypes_CreateDatasetResponse) Reset() { func (m *DatasetServiceTypes_CreateDatasetResponse) String() string { return proto.CompactTextString(m) } func (*DatasetServiceTypes_CreateDatasetResponse) ProtoMessage() {} func (*DatasetServiceTypes_CreateDatasetResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{0, 1} + return fileDescriptor_3edc37a8b0d37b39, []int{0, 1} } func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Unmarshal(m, b) @@ -154,8 +154,8 @@ func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Unmarshal(b []byte) erro func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Marshal(b, m, deterministic) } -func (dst *DatasetServiceTypes_CreateDatasetResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Merge(dst, src) +func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Merge(m, src) } func (m *DatasetServiceTypes_CreateDatasetResponse) XXX_Size() int { return xxx_messageInfo_DatasetServiceTypes_CreateDatasetResponse.Size(m) @@ -188,7 +188,7 @@ func (m *FeatureSet) Reset() { *m = FeatureSet{} } func (m *FeatureSet) String() string { return proto.CompactTextString(m) } func (*FeatureSet) ProtoMessage() {} func (*FeatureSet) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{1} + return fileDescriptor_3edc37a8b0d37b39, []int{1} } func (m *FeatureSet) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureSet.Unmarshal(m, b) @@ -196,8 +196,8 @@ func (m *FeatureSet) XXX_Unmarshal(b []byte) error { func (m *FeatureSet) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureSet.Marshal(b, m, deterministic) } -func (dst *FeatureSet) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureSet.Merge(dst, src) +func (m *FeatureSet) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureSet.Merge(m, src) } func (m *FeatureSet) XXX_Size() int { return xxx_messageInfo_FeatureSet.Size(m) @@ -237,7 +237,7 @@ func (m *DatasetInfo) Reset() { *m = DatasetInfo{} } func (m *DatasetInfo) String() string { return proto.CompactTextString(m) } func (*DatasetInfo) ProtoMessage() {} func (*DatasetInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_DatasetService_37ae639a8c7b5dd5, []int{2} + return fileDescriptor_3edc37a8b0d37b39, []int{2} } func (m *DatasetInfo) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_DatasetInfo.Unmarshal(m, b) @@ -245,8 +245,8 @@ func (m *DatasetInfo) XXX_Unmarshal(b []byte) error { func (m *DatasetInfo) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_DatasetInfo.Marshal(b, m, deterministic) } -func (dst *DatasetInfo) XXX_Merge(src proto.Message) { - xxx_messageInfo_DatasetInfo.Merge(dst, src) +func (m *DatasetInfo) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatasetInfo.Merge(m, src) } func (m *DatasetInfo) XXX_Size() int { return xxx_messageInfo_DatasetInfo.Size(m) @@ -353,11 +353,9 @@ var _DatasetService_serviceDesc = grpc.ServiceDesc{ Metadata: "feast/core/DatasetService.proto", } -func init() { - proto.RegisterFile("feast/core/DatasetService.proto", fileDescriptor_DatasetService_37ae639a8c7b5dd5) -} +func init() { proto.RegisterFile("feast/core/DatasetService.proto", fileDescriptor_3edc37a8b0d37b39) } -var fileDescriptor_DatasetService_37ae639a8c7b5dd5 = []byte{ +var fileDescriptor_3edc37a8b0d37b39 = []byte{ // 414 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0xc1, 0x6e, 0xd4, 0x30, 0x10, 0x25, 0xbb, 0x2d, 0x90, 0x59, 0xc1, 0xc1, 0x14, 0x88, 0x72, 0xa0, 0x51, 0x4e, 0x7b, 0xb2, diff --git a/protos/generated/go/feast/serving/Serving.pb.go b/protos/generated/go/feast/serving/Serving.pb.go index 60ee7d3663..64979c154a 100644 --- a/protos/generated/go/feast/serving/Serving.pb.go +++ b/protos/generated/go/feast/serving/Serving.pb.go @@ -34,12 +34,10 @@ type QueryFeaturesRequest struct { // feature ID is in the form of [entity_name].[granularity].[feature_name] // e.g: "driver.day.total_accepted_booking" // all requested feature ID shall have same entity name. - FeatureId []string `protobuf:"bytes,3,rep,name=featureId,proto3" json:"featureId,omitempty"` - // [optional] time range filter. If not specified all feature with any timestamp will be returned. - TimeRange *TimestampRange `protobuf:"bytes,4,opt,name=timeRange,proto3" json:"timeRange,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + FeatureId []string `protobuf:"bytes,3,rep,name=featureId,proto3" json:"featureId,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *QueryFeaturesRequest) Reset() { *m = QueryFeaturesRequest{} } @@ -87,13 +85,6 @@ func (m *QueryFeaturesRequest) GetFeatureId() []string { return nil } -func (m *QueryFeaturesRequest) GetTimeRange() *TimestampRange { - if m != nil { - return m.TimeRange - } - return nil -} - type QueryFeaturesResponse struct { // Entity name of the response EntityName string `protobuf:"bytes,1,opt,name=entityName,proto3" json:"entityName,omitempty"` @@ -142,56 +133,6 @@ func (m *QueryFeaturesResponse) GetEntities() map[string]*Entity { return nil } -// range of timestamp for querying -// valid timestamp range is having start <= end -type TimestampRange struct { - // start time of the range query. - Start *timestamp.Timestamp `protobuf:"bytes,1,opt,name=start,proto3" json:"start,omitempty"` - // end time of the range query. - End *timestamp.Timestamp `protobuf:"bytes,2,opt,name=end,proto3" json:"end,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *TimestampRange) Reset() { *m = TimestampRange{} } -func (m *TimestampRange) String() string { return proto.CompactTextString(m) } -func (*TimestampRange) ProtoMessage() {} -func (*TimestampRange) Descriptor() ([]byte, []int) { - return fileDescriptor_7609de6de542e6f0, []int{2} -} -func (m *TimestampRange) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_TimestampRange.Unmarshal(m, b) -} -func (m *TimestampRange) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_TimestampRange.Marshal(b, m, deterministic) -} -func (m *TimestampRange) XXX_Merge(src proto.Message) { - xxx_messageInfo_TimestampRange.Merge(m, src) -} -func (m *TimestampRange) XXX_Size() int { - return xxx_messageInfo_TimestampRange.Size(m) -} -func (m *TimestampRange) XXX_DiscardUnknown() { - xxx_messageInfo_TimestampRange.DiscardUnknown(m) -} - -var xxx_messageInfo_TimestampRange proto.InternalMessageInfo - -func (m *TimestampRange) GetStart() *timestamp.Timestamp { - if m != nil { - return m.Start - } - return nil -} - -func (m *TimestampRange) GetEnd() *timestamp.Timestamp { - if m != nil { - return m.End - } - return nil -} - type Entity struct { // map of feature ID and its feature value. Features map[string]*FeatureValue `protobuf:"bytes,1,rep,name=features,proto3" json:"features,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` @@ -204,7 +145,7 @@ func (m *Entity) Reset() { *m = Entity{} } func (m *Entity) String() string { return proto.CompactTextString(m) } func (*Entity) ProtoMessage() {} func (*Entity) Descriptor() ([]byte, []int) { - return fileDescriptor_7609de6de542e6f0, []int{3} + return fileDescriptor_7609de6de542e6f0, []int{2} } func (m *Entity) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Entity.Unmarshal(m, b) @@ -245,7 +186,7 @@ func (m *FeatureValue) Reset() { *m = FeatureValue{} } func (m *FeatureValue) String() string { return proto.CompactTextString(m) } func (*FeatureValue) ProtoMessage() {} func (*FeatureValue) Descriptor() ([]byte, []int) { - return fileDescriptor_7609de6de542e6f0, []int{4} + return fileDescriptor_7609de6de542e6f0, []int{3} } func (m *FeatureValue) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureValue.Unmarshal(m, b) @@ -283,7 +224,6 @@ func init() { proto.RegisterType((*QueryFeaturesRequest)(nil), "feast.serving.QueryFeaturesRequest") proto.RegisterType((*QueryFeaturesResponse)(nil), "feast.serving.QueryFeaturesResponse") proto.RegisterMapType((map[string]*Entity)(nil), "feast.serving.QueryFeaturesResponse.EntitiesEntry") - proto.RegisterType((*TimestampRange)(nil), "feast.serving.TimestampRange") proto.RegisterType((*Entity)(nil), "feast.serving.Entity") proto.RegisterMapType((map[string]*FeatureValue)(nil), "feast.serving.Entity.FeaturesEntry") proto.RegisterType((*FeatureValue)(nil), "feast.serving.FeatureValue") @@ -366,36 +306,32 @@ var _ServingAPI_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("feast/serving/Serving.proto", fileDescriptor_7609de6de542e6f0) } var fileDescriptor_7609de6de542e6f0 = []byte{ - // 486 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4d, 0x6f, 0xd3, 0x40, - 0x10, 0xc5, 0x09, 0xad, 0x9a, 0x09, 0x01, 0xb4, 0xa2, 0xc2, 0x72, 0xf9, 0x88, 0x5c, 0x0e, 0x91, - 0x40, 0xbb, 0x60, 0x2e, 0x11, 0x1c, 0x10, 0x95, 0x8a, 0x94, 0x4b, 0x55, 0x0c, 0x42, 0xa8, 0xe2, - 0xe2, 0x90, 0x89, 0x71, 0xdb, 0x78, 0xcd, 0xee, 0xba, 0x92, 0x7f, 0x0f, 0x67, 0x7e, 0x11, 0x7f, - 0x06, 0x79, 0x77, 0xdd, 0x78, 0xad, 0xa8, 0xcd, 0x29, 0xd9, 0x7d, 0x6f, 0x66, 0xde, 0xbc, 0x9d, - 0x31, 0x1c, 0x2c, 0x31, 0x91, 0x8a, 0x49, 0x14, 0x57, 0x59, 0x9e, 0xb2, 0x2f, 0xe6, 0x97, 0x16, - 0x82, 0x2b, 0x4e, 0x46, 0x1a, 0xa4, 0x16, 0x0c, 0x9e, 0xa7, 0x9c, 0xa7, 0x97, 0xc8, 0x34, 0x38, - 0x2f, 0x97, 0x4c, 0x65, 0x2b, 0x94, 0x2a, 0x59, 0x15, 0x86, 0x1f, 0x3c, 0x36, 0xc9, 0x54, 0x55, - 0xa0, 0x64, 0xdf, 0x92, 0xcb, 0x12, 0x0d, 0x10, 0xfe, 0xf5, 0xe0, 0xd1, 0xe7, 0x12, 0x45, 0xf5, - 0x09, 0x13, 0x55, 0x0a, 0x94, 0x31, 0xfe, 0x2e, 0x51, 0x2a, 0xf2, 0x0c, 0x00, 0x73, 0x95, 0xa9, - 0xea, 0x24, 0x59, 0xa1, 0xef, 0x8d, 0xbd, 0xc9, 0x20, 0x6e, 0xdd, 0x90, 0x00, 0xf6, 0xcc, 0x69, - 0xb6, 0xf0, 0x7b, 0xe3, 0xfe, 0x64, 0x10, 0x5f, 0x9f, 0xc9, 0x13, 0x18, 0x2c, 0x4d, 0xba, 0xd9, - 0xc2, 0xef, 0x6b, 0x70, 0x7d, 0x41, 0xde, 0xc3, 0xa0, 0x96, 0x17, 0x27, 0x79, 0x8a, 0xfe, 0xdd, - 0xb1, 0x37, 0x19, 0x46, 0x4f, 0xa9, 0xd3, 0x0f, 0xfd, 0xda, 0xc8, 0xd7, 0xa4, 0x78, 0xcd, 0x0f, - 0xff, 0x79, 0xb0, 0xdf, 0xd1, 0x2b, 0x0b, 0x9e, 0x4b, 0xbc, 0x55, 0xf0, 0x89, 0x15, 0x9c, 0xa1, - 0xd4, 0x82, 0x87, 0x51, 0xd4, 0xa9, 0xba, 0x31, 0x2f, 0x3d, 0xb6, 0x41, 0xc7, 0xb9, 0x12, 0x55, - 0x7c, 0x9d, 0x23, 0x88, 0x61, 0xe4, 0x40, 0xe4, 0x21, 0xf4, 0x2f, 0xb0, 0xb2, 0x95, 0xeb, 0xbf, - 0xe4, 0x25, 0xec, 0x5c, 0xd5, 0x5e, 0xfb, 0x3d, 0xdd, 0xe5, 0x7e, 0xa7, 0x9e, 0x0e, 0xaf, 0x62, - 0xc3, 0x79, 0xd7, 0x9b, 0x7a, 0x61, 0x01, 0xf7, 0xdd, 0xd6, 0xc9, 0x6b, 0xd8, 0x91, 0x2a, 0x11, - 0x4a, 0xa7, 0x1d, 0x46, 0x01, 0x35, 0x2f, 0x4d, 0x9b, 0x97, 0x6e, 0x59, 0x65, 0x88, 0xe4, 0x15, - 0xf4, 0x31, 0x5f, 0xd8, 0x92, 0x37, 0xf1, 0x6b, 0x5a, 0xf8, 0xc7, 0x83, 0x5d, 0xa3, 0x83, 0x7c, - 0x80, 0x3d, 0xfb, 0x48, 0xd2, 0xf7, 0xb4, 0x41, 0x87, 0x1b, 0x05, 0xd3, 0xc6, 0x22, 0xeb, 0x48, - 0x13, 0x14, 0x7c, 0x87, 0x91, 0x03, 0x6d, 0x70, 0xe4, 0x8d, 0xeb, 0xc8, 0x41, 0xa7, 0x80, 0x0d, - 0xd7, 0x03, 0xda, 0xf6, 0x45, 0xc0, 0xbd, 0x36, 0x44, 0x26, 0x4d, 0x1a, 0xe3, 0x0a, 0xb1, 0x69, - 0xf4, 0x78, 0xd3, 0x76, 0x34, 0x99, 0x9a, 0x61, 0xd3, 0x1d, 0x6f, 0xe1, 0xc9, 0x9a, 0x1c, 0x9d, - 0x03, 0xd8, 0x9d, 0xfb, 0x78, 0x3a, 0x23, 0x3f, 0x60, 0xe4, 0x8c, 0x07, 0x39, 0xbc, 0x79, 0x78, - 0xf4, 0x12, 0x05, 0x2f, 0xb6, 0x99, 0xb0, 0xf0, 0xce, 0xd1, 0x19, 0xb8, 0x0b, 0x7d, 0xf4, 0x60, - 0x5d, 0xfa, 0xb4, 0x56, 0x79, 0x36, 0x4d, 0x33, 0xf5, 0xab, 0x9c, 0xd3, 0x9f, 0x7c, 0xc5, 0x52, - 0x7e, 0x8e, 0x17, 0xcc, 0x6c, 0xb4, 0xee, 0x41, 0xb2, 0x14, 0x73, 0x14, 0x89, 0xc2, 0x05, 0x4b, - 0x39, 0x73, 0x3e, 0x1c, 0xf3, 0x5d, 0x4d, 0x79, 0xfb, 0x3f, 0x00, 0x00, 0xff, 0xff, 0x14, 0x04, - 0x54, 0x19, 0x50, 0x04, 0x00, 0x00, + // 429 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x53, 0xd1, 0x8a, 0xd3, 0x40, + 0x14, 0x75, 0x5a, 0x5c, 0x36, 0x77, 0x0d, 0xca, 0xe0, 0x62, 0xc8, 0x8a, 0x96, 0xac, 0x0f, 0x01, + 0x61, 0x06, 0xe3, 0x4b, 0xf1, 0x45, 0x5c, 0x58, 0xa1, 0x2f, 0x4b, 0x8d, 0x22, 0x52, 0x7c, 0x49, + 0xed, 0x6d, 0x4c, 0xdb, 0x64, 0x62, 0x66, 0x52, 0xc8, 0xf7, 0xf8, 0x59, 0xfe, 0x8c, 0x74, 0x66, + 0xd2, 0x26, 0xa5, 0xe8, 0x3e, 0x25, 0x33, 0xe7, 0xde, 0x73, 0xce, 0xcc, 0x3d, 0x03, 0x57, 0x4b, + 0x4c, 0xa4, 0xe2, 0x12, 0xab, 0x6d, 0x56, 0xa4, 0xfc, 0xb3, 0xf9, 0xb2, 0xb2, 0x12, 0x4a, 0x50, + 0x57, 0x83, 0xcc, 0x82, 0xfe, 0xcb, 0x54, 0x88, 0x74, 0x83, 0x5c, 0x83, 0xf3, 0x7a, 0xc9, 0x55, + 0x96, 0xa3, 0x54, 0x49, 0x5e, 0x9a, 0x7a, 0xff, 0x99, 0x21, 0x53, 0x4d, 0x89, 0x92, 0x7f, 0x4d, + 0x36, 0x35, 0x1a, 0x20, 0x28, 0xe1, 0xe9, 0xa7, 0x1a, 0xab, 0xe6, 0x23, 0x26, 0xaa, 0xae, 0x50, + 0xc6, 0xf8, 0xab, 0x46, 0xa9, 0xe8, 0x0b, 0x00, 0x2c, 0x54, 0xa6, 0x9a, 0xbb, 0x24, 0x47, 0x8f, + 0x8c, 0x48, 0xe8, 0xc4, 0x9d, 0x1d, 0xea, 0xc3, 0xb9, 0x59, 0x4d, 0x16, 0xde, 0x60, 0x34, 0x0c, + 0x9d, 0x78, 0xbf, 0xa6, 0xcf, 0xc1, 0x59, 0x1a, 0xba, 0xc9, 0xc2, 0x1b, 0x6a, 0xf0, 0xb0, 0x11, + 0xfc, 0x21, 0x70, 0x79, 0x24, 0x29, 0x4b, 0x51, 0x48, 0xfc, 0xaf, 0xe6, 0x9d, 0xd5, 0xcc, 0x50, + 0x6a, 0xcd, 0x8b, 0x28, 0x62, 0xbd, 0x7b, 0x60, 0x27, 0x79, 0xd9, 0xad, 0x6d, 0xba, 0x2d, 0x54, + 0xd5, 0xc4, 0x7b, 0x0e, 0x3f, 0x06, 0xb7, 0x07, 0xd1, 0x27, 0x30, 0x5c, 0x63, 0x63, 0x95, 0x77, + 0xbf, 0xf4, 0x35, 0x3c, 0xdc, 0xee, 0x6e, 0xcb, 0x1b, 0x8c, 0x48, 0x78, 0x11, 0x5d, 0x1e, 0xe9, + 0xe9, 0xf6, 0x26, 0x36, 0x35, 0xef, 0x06, 0x63, 0x12, 0xfc, 0x26, 0x70, 0x66, 0x76, 0xe9, 0x7b, + 0x38, 0xb7, 0xa7, 0x96, 0x1e, 0xd1, 0x76, 0xaf, 0x4f, 0xb6, 0xb3, 0xd6, 0xb0, 0xf5, 0xd7, 0x36, + 0xf9, 0xdf, 0xc0, 0xed, 0x41, 0x27, 0xfc, 0xbd, 0xe9, 0xfb, 0xbb, 0x3a, 0x12, 0xb0, 0xed, 0x7a, + 0xe0, 0x5d, 0x97, 0x15, 0x3c, 0xea, 0x42, 0x34, 0x6c, 0x69, 0x88, 0xa6, 0xa1, 0x96, 0x46, 0xc7, + 0x85, 0x75, 0xbb, 0xe9, 0x18, 0x9c, 0x7d, 0xb6, 0xac, 0xa8, 0xcf, 0x4c, 0xfa, 0x58, 0x9b, 0x3e, + 0xf6, 0xa5, 0xad, 0x88, 0x0f, 0xc5, 0xd1, 0x0a, 0xc0, 0x66, 0xf8, 0xc3, 0x74, 0x42, 0xbf, 0x83, + 0xdb, 0x1b, 0x16, 0xbd, 0xfe, 0xf7, 0x28, 0x75, 0x2a, 0xfd, 0x57, 0xf7, 0x99, 0x77, 0xf0, 0xe0, + 0x66, 0x06, 0xfd, 0x07, 0x72, 0xf3, 0xf8, 0x20, 0x3d, 0xdd, 0xb9, 0x9c, 0x8d, 0xd3, 0x4c, 0xfd, + 0xac, 0xe7, 0xec, 0x87, 0xc8, 0x79, 0x2a, 0x56, 0xb8, 0xe6, 0xe6, 0x85, 0xe8, 0x33, 0x48, 0x9e, + 0x62, 0x81, 0x55, 0xa2, 0x70, 0xc1, 0x53, 0xc1, 0x7b, 0x0f, 0x71, 0x7e, 0xa6, 0x4b, 0xde, 0xfe, + 0x0d, 0x00, 0x00, 0xff, 0xff, 0x77, 0xb5, 0x0e, 0xd1, 0xa0, 0x03, 0x00, 0x00, } diff --git a/protos/generated/go/feast/specs/ImportSpec.pb.go b/protos/generated/go/feast/specs/ImportSpec.pb.go index d16f9fa07b..3a4c8d0801 100644 --- a/protos/generated/go/feast/specs/ImportSpec.pb.go +++ b/protos/generated/go/feast/specs/ImportSpec.pb.go @@ -34,7 +34,7 @@ func (m *ImportSpec) Reset() { *m = ImportSpec{} } func (m *ImportSpec) String() string { return proto.CompactTextString(m) } func (*ImportSpec) ProtoMessage() {} func (*ImportSpec) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportSpec_673bc4f248a91137, []int{0} + return fileDescriptor_bf21bfd2215b3e0c, []int{0} } func (m *ImportSpec) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_ImportSpec.Unmarshal(m, b) @@ -42,8 +42,8 @@ func (m *ImportSpec) XXX_Unmarshal(b []byte) error { func (m *ImportSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_ImportSpec.Marshal(b, m, deterministic) } -func (dst *ImportSpec) XXX_Merge(src proto.Message) { - xxx_messageInfo_ImportSpec.Merge(dst, src) +func (m *ImportSpec) XXX_Merge(src proto.Message) { + xxx_messageInfo_ImportSpec.Merge(m, src) } func (m *ImportSpec) XXX_Size() int { return xxx_messageInfo_ImportSpec.Size(m) @@ -107,7 +107,7 @@ func (m *Schema) Reset() { *m = Schema{} } func (m *Schema) String() string { return proto.CompactTextString(m) } func (*Schema) ProtoMessage() {} func (*Schema) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportSpec_673bc4f248a91137, []int{1} + return fileDescriptor_bf21bfd2215b3e0c, []int{1} } func (m *Schema) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Schema.Unmarshal(m, b) @@ -115,8 +115,8 @@ func (m *Schema) XXX_Unmarshal(b []byte) error { func (m *Schema) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Schema.Marshal(b, m, deterministic) } -func (dst *Schema) XXX_Merge(src proto.Message) { - xxx_messageInfo_Schema.Merge(dst, src) +func (m *Schema) XXX_Merge(src proto.Message) { + xxx_messageInfo_Schema.Merge(m, src) } func (m *Schema) XXX_Size() int { return xxx_messageInfo_Schema.Size(m) @@ -260,7 +260,7 @@ func (m *Field) Reset() { *m = Field{} } func (m *Field) String() string { return proto.CompactTextString(m) } func (*Field) ProtoMessage() {} func (*Field) Descriptor() ([]byte, []int) { - return fileDescriptor_ImportSpec_673bc4f248a91137, []int{2} + return fileDescriptor_bf21bfd2215b3e0c, []int{2} } func (m *Field) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Field.Unmarshal(m, b) @@ -268,8 +268,8 @@ func (m *Field) XXX_Unmarshal(b []byte) error { func (m *Field) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Field.Marshal(b, m, deterministic) } -func (dst *Field) XXX_Merge(src proto.Message) { - xxx_messageInfo_Field.Merge(dst, src) +func (m *Field) XXX_Merge(src proto.Message) { + xxx_messageInfo_Field.Merge(m, src) } func (m *Field) XXX_Size() int { return xxx_messageInfo_Field.Size(m) @@ -302,11 +302,9 @@ func init() { proto.RegisterType((*Field)(nil), "feast.specs.Field") } -func init() { - proto.RegisterFile("feast/specs/ImportSpec.proto", fileDescriptor_ImportSpec_673bc4f248a91137) -} +func init() { proto.RegisterFile("feast/specs/ImportSpec.proto", fileDescriptor_bf21bfd2215b3e0c) } -var fileDescriptor_ImportSpec_673bc4f248a91137 = []byte{ +var fileDescriptor_bf21bfd2215b3e0c = []byte{ // 440 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x53, 0x5d, 0x8f, 0x93, 0x40, 0x14, 0x5d, 0xca, 0x16, 0xed, 0x6d, 0xdc, 0x9a, 0xab, 0x0f, 0x84, 0x6c, 0x62, 0xd3, 0x07, 0x6d, diff --git a/protos/generated/go/feast/storage/BigTable.pb.go b/protos/generated/go/feast/storage/BigTable.pb.go index e65b5e35fc..9943d0a884 100644 --- a/protos/generated/go/feast/storage/BigTable.pb.go +++ b/protos/generated/go/feast/storage/BigTable.pb.go @@ -33,7 +33,7 @@ func (m *BigTableRowKey) Reset() { *m = BigTableRowKey{} } func (m *BigTableRowKey) String() string { return proto.CompactTextString(m) } func (*BigTableRowKey) ProtoMessage() {} func (*BigTableRowKey) Descriptor() ([]byte, []int) { - return fileDescriptor_BigTable_e25bdeffe0669ae7, []int{0} + return fileDescriptor_1d33ec3bd45c712c, []int{0} } func (m *BigTableRowKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_BigTableRowKey.Unmarshal(m, b) @@ -41,8 +41,8 @@ func (m *BigTableRowKey) XXX_Unmarshal(b []byte) error { func (m *BigTableRowKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_BigTableRowKey.Marshal(b, m, deterministic) } -func (dst *BigTableRowKey) XXX_Merge(src proto.Message) { - xxx_messageInfo_BigTableRowKey.Merge(dst, src) +func (m *BigTableRowKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_BigTableRowKey.Merge(m, src) } func (m *BigTableRowKey) XXX_Size() int { return xxx_messageInfo_BigTableRowKey.Size(m) @@ -78,11 +78,9 @@ func init() { proto.RegisterType((*BigTableRowKey)(nil), "feast.storage.BigTableRowKey") } -func init() { - proto.RegisterFile("feast/storage/BigTable.proto", fileDescriptor_BigTable_e25bdeffe0669ae7) -} +func init() { proto.RegisterFile("feast/storage/BigTable.proto", fileDescriptor_1d33ec3bd45c712c) } -var fileDescriptor_BigTable_e25bdeffe0669ae7 = []byte{ +var fileDescriptor_1d33ec3bd45c712c = []byte{ // 193 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x5c, 0x8f, 0xb1, 0x0b, 0x82, 0x40, 0x18, 0x47, 0xb1, 0x20, 0xf0, 0x40, 0x87, 0x9b, 0x1c, 0x24, 0xa2, 0x21, 0x9a, 0x3c, 0xa2, 0xa5, diff --git a/protos/generated/go/feast/types/Feature.pb.go b/protos/generated/go/feast/types/Feature.pb.go index 51ed41fb54..44534da5c7 100644 --- a/protos/generated/go/feast/types/Feature.pb.go +++ b/protos/generated/go/feast/types/Feature.pb.go @@ -30,7 +30,7 @@ func (m *Feature) Reset() { *m = Feature{} } func (m *Feature) String() string { return proto.CompactTextString(m) } func (*Feature) ProtoMessage() {} func (*Feature) Descriptor() ([]byte, []int) { - return fileDescriptor_Feature_c2a5d99d9bf3ca9c, []int{0} + return fileDescriptor_4e19474999533fc9, []int{0} } func (m *Feature) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_Feature.Unmarshal(m, b) @@ -38,8 +38,8 @@ func (m *Feature) XXX_Unmarshal(b []byte) error { func (m *Feature) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_Feature.Marshal(b, m, deterministic) } -func (dst *Feature) XXX_Merge(src proto.Message) { - xxx_messageInfo_Feature.Merge(dst, src) +func (m *Feature) XXX_Merge(src proto.Message) { + xxx_messageInfo_Feature.Merge(m, src) } func (m *Feature) XXX_Size() int { return xxx_messageInfo_Feature.Size(m) @@ -68,9 +68,9 @@ func init() { proto.RegisterType((*Feature)(nil), "feast.types.Feature") } -func init() { proto.RegisterFile("feast/types/Feature.proto", fileDescriptor_Feature_c2a5d99d9bf3ca9c) } +func init() { proto.RegisterFile("feast/types/Feature.proto", fileDescriptor_4e19474999533fc9) } -var fileDescriptor_Feature_c2a5d99d9bf3ca9c = []byte{ +var fileDescriptor_4e19474999533fc9 = []byte{ // 173 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4c, 0x4b, 0x4d, 0x2c, 0x2e, 0xd1, 0x2f, 0xa9, 0x2c, 0x48, 0x2d, 0xd6, 0x77, 0x4b, 0x4d, 0x2c, 0x29, 0x2d, 0x4a, 0xd5, diff --git a/protos/generated/go/feast/types/FeatureRow.pb.go b/protos/generated/go/feast/types/FeatureRow.pb.go index 3fb6bc72f9..ba8712ce34 100644 --- a/protos/generated/go/feast/types/FeatureRow.pb.go +++ b/protos/generated/go/feast/types/FeatureRow.pb.go @@ -33,7 +33,7 @@ func (m *FeatureRowKey) Reset() { *m = FeatureRowKey{} } func (m *FeatureRowKey) String() string { return proto.CompactTextString(m) } func (*FeatureRowKey) ProtoMessage() {} func (*FeatureRowKey) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRow_435f7325616f2983, []int{0} + return fileDescriptor_fbbea9c89787d1c7, []int{0} } func (m *FeatureRowKey) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRowKey.Unmarshal(m, b) @@ -41,8 +41,8 @@ func (m *FeatureRowKey) XXX_Unmarshal(b []byte) error { func (m *FeatureRowKey) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureRowKey.Marshal(b, m, deterministic) } -func (dst *FeatureRowKey) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureRowKey.Merge(dst, src) +func (m *FeatureRowKey) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureRowKey.Merge(m, src) } func (m *FeatureRowKey) XXX_Size() int { return xxx_messageInfo_FeatureRowKey.Size(m) @@ -96,7 +96,7 @@ func (m *FeatureRow) Reset() { *m = FeatureRow{} } func (m *FeatureRow) String() string { return proto.CompactTextString(m) } func (*FeatureRow) ProtoMessage() {} func (*FeatureRow) Descriptor() ([]byte, []int) { - return fileDescriptor_FeatureRow_435f7325616f2983, []int{1} + return fileDescriptor_fbbea9c89787d1c7, []int{1} } func (m *FeatureRow) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_FeatureRow.Unmarshal(m, b) @@ -104,8 +104,8 @@ func (m *FeatureRow) XXX_Unmarshal(b []byte) error { func (m *FeatureRow) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return xxx_messageInfo_FeatureRow.Marshal(b, m, deterministic) } -func (dst *FeatureRow) XXX_Merge(src proto.Message) { - xxx_messageInfo_FeatureRow.Merge(dst, src) +func (m *FeatureRow) XXX_Merge(src proto.Message) { + xxx_messageInfo_FeatureRow.Merge(m, src) } func (m *FeatureRow) XXX_Size() int { return xxx_messageInfo_FeatureRow.Size(m) @@ -156,11 +156,9 @@ func init() { proto.RegisterType((*FeatureRow)(nil), "feast.types.FeatureRow") } -func init() { - proto.RegisterFile("feast/types/FeatureRow.proto", fileDescriptor_FeatureRow_435f7325616f2983) -} +func init() { proto.RegisterFile("feast/types/FeatureRow.proto", fileDescriptor_fbbea9c89787d1c7) } -var fileDescriptor_FeatureRow_435f7325616f2983 = []byte{ +var fileDescriptor_fbbea9c89787d1c7 = []byte{ // 305 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x92, 0x41, 0x4b, 0xfb, 0x40, 0x10, 0xc5, 0x49, 0xfb, 0xff, 0x8b, 0x9d, 0x60, 0x85, 0xe0, 0x21, 0x96, 0x56, 0x43, 0x4f, 0x39, From 317fdd34b8a0e4865bcce589044362b6bbb7a8de Mon Sep 17 00:00:00 2001 From: Pradithya Aria Date: Sat, 23 Feb 2019 23:14:28 +0800 Subject: [PATCH 8/8] Use datetime type for time range filter --- sdk/python/feast/sdk/client.py | 28 +++++++++++++++------------- sdk/python/tests/sdk/test_client.py | 27 +++++++++++++++++---------- 2 files changed, 32 insertions(+), 23 deletions(-) diff --git a/sdk/python/feast/sdk/client.py b/sdk/python/feast/sdk/client.py index 38256b1705..a8d146be89 100644 --- a/sdk/python/feast/sdk/client.py +++ b/sdk/python/feast/sdk/client.py @@ -18,7 +18,6 @@ import os from datetime import datetime -import dateutil.parser import grpc import pandas as pd @@ -220,17 +219,27 @@ def get_serving_data(self, feature_set, entity_keys, ts_range=None): representing the data wanted entity_keys (:obj: `list` of :obj: `str): list of entity keys ts_range (:obj: `list` of str, optional): size 2 list of start - timestamp and end timestamp, in ISO 8601 format. It will + and end time, in datetime type. It will filter out any feature value having event timestamp outside of the ts_range. Returns: pandas.DataFrame: DataFrame of results """ + start = None + end = None + if ts_range is not None: + if len(ts_range) != 2: + raise ValueError("ts_range must have len 2") + start = ts_range[0] + end = ts_range[1] + if type(start) is not datetime or type(end) is not datetime: + raise TypeError("start and end must be datetime type") + request = self._build_serving_request(feature_set, entity_keys) self._connect_serving() return self._response_to_df(feature_set, self._serving_service_stub - .QueryFeatures(request), ts_range) + .QueryFeatures(request), start, end) def download_dataset(self, dataset_info, dest, staging_location, file_type=FileType.CSV): @@ -299,22 +308,15 @@ def _build_serving_request(self, feature_set, entity_keys): entityId=entity_keys, featureId=feature_set.features) - def _response_to_df(self, feature_set, response, ts_range = None): - start = None - end = None - if ts_range is not None: - if len(ts_range) != 2: - raise ValueError("ts_range must have len 2") - start = dateutil.parser.parse(ts_range[0]) - end = dateutil.parser.parse(ts_range[1]) - + def _response_to_df(self, feature_set, response, start=None, end=None): + is_filter_time = start is not None and end is not None df = pd.DataFrame(columns=[feature_set.entity] + feature_set.features) for entity_id in response.entities: feature_map = response.entities[entity_id].features row = {response.entityName: entity_id} for feature_id in feature_map: v = feature_map[feature_id].value - if ts_range is not None and not _is_granularity_none( + if is_filter_time and not _is_granularity_none( feature_id): ts = feature_map[feature_id].timestamp.ToDatetime() if ts < start or ts > end: diff --git a/sdk/python/tests/sdk/test_client.py b/sdk/python/tests/sdk/test_client.py index 0caf889521..c13c350ed4 100644 --- a/sdk/python/tests/sdk/test_client.py +++ b/sdk/python/tests/sdk/test_client.py @@ -360,13 +360,11 @@ def test_serving_response_to_df_with_time_filter(self, client): 'entity.feat1': [np.NaN, 3], 'entity.feat2': [np.NaN, np.NaN]}) \ .reset_index(drop=True) - start = datetime.utcfromtimestamp(2).isoformat() - end = datetime.utcfromtimestamp(5).isoformat() - - ts_range = [start, end] + start = datetime.utcfromtimestamp(2) + end = datetime.utcfromtimestamp(5) df = client._response_to_df(FeatureSet("entity", ["entity.feat1", "entity.feat2"]), - response, ts_range) \ + response, start, end) \ .sort_values(['entity']) \ .reset_index(drop=True)[expected_df.columns] print(df) @@ -386,13 +384,11 @@ def test_serving_response_to_df_with_time_filter_granularity_none(self, 'entity.none.feat1': [1, 3], 'entity.none.feat2': [np.NaN, np.NaN]}) \ .reset_index(drop=True) - start = datetime.utcfromtimestamp(2).isoformat() - end = datetime.utcfromtimestamp(5).isoformat() - - ts_range = [start, end] + start = datetime.utcfromtimestamp(2) + end = datetime.utcfromtimestamp(5) df = client._response_to_df(FeatureSet("entity", ["entity.none.feat1", "entity.none.feat2"]), - response, ts_range) \ + response, start, end) \ .sort_values(['entity']) \ .reset_index(drop=True)[expected_df.columns] print(df) @@ -401,6 +397,17 @@ def test_serving_response_to_df_with_time_filter_granularity_none(self, check_column_type=False, check_index_type=False) + def test_serving_invalid_type(self, client): + start = "2018-01-01T01:01:01" + end = "2018-01-01T01:01:01" + ts_range = [start, end] + with pytest.raises(TypeError, match="start and end must be datetime " + "type"): + client.get_serving_data(FeatureSet("entity", ["entity.none.feat1", + "entity.none.feat2"]), + ["1234", "5678"], + ts_range) + def test_download_dataset_as_file(self, client, mocker): destination = "/tmp/dest_file"