Skip to content

Commit

Permalink
move read transforms into "source" package as FeatureSources (#74)
Browse files Browse the repository at this point in the history
* move read transforms into "source" package as FeatureSources

* fix core import spec validator to check for file.csv and file.json

* update import spec validator tests
  • Loading branch information
tims authored and feast-ci-bot committed Jan 19, 2019
1 parent 9e2de1b commit b5e19bd
Show file tree
Hide file tree
Showing 32 changed files with 704 additions and 673 deletions.
6 changes: 2 additions & 4 deletions core/src/main/java/feast/core/validators/SpecValidator.java
Original file line number Diff line number Diff line change
Expand Up @@ -224,7 +224,8 @@ public void validateImportSpec(ImportSpec spec) throws IllegalArgumentException
case "pubsub":
checkPubSubImportSpecOption(spec);
break;
case "file":
case "file.csv":
case "file.json":
checkFileImportSpecOption(spec);
checkArgument(
!spec.getSchema().getEntityIdColumn().equals(""),
Expand Down Expand Up @@ -263,9 +264,6 @@ public void validateImportSpec(ImportSpec spec) throws IllegalArgumentException

private void checkFileImportSpecOption(ImportSpec spec) throws IllegalArgumentException {
try {
checkArgument(
Lists.newArrayList("json", "csv").contains(spec.getOptionsOrThrow("format")),
"File format must be of type 'json' or 'csv'");
checkArgument(!spec.getOptionsOrDefault("path", "").equals(""), "File path cannot be empty");
} catch (NullPointerException | IllegalArgumentException e) {
throw new IllegalArgumentException(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -670,10 +670,10 @@ public void fileImportSpecWithoutSupportedFileFormatShouldThrowIllegalArgumentEx
featureGroupInfoRepository,
featureInfoRepository);
ImportSpec input =
ImportSpec.newBuilder().setType("file").putOptions("format", "notSupported").build();
ImportSpec.newBuilder().setType("file.wat?").build();
exception.expect(IllegalArgumentException.class);
exception.expectMessage(
"Validation for import spec failed: Invalid options: File format must be of type 'json' or 'csv'");
"Validation for import spec failed: Type file.wat? not supported");
validator.validateImportSpec(input);
}

Expand All @@ -685,7 +685,7 @@ public void fileImportSpecWithoutValidPathShouldThrowIllegalArgumentException()
entityInfoRepository,
featureGroupInfoRepository,
featureInfoRepository);
ImportSpec input = ImportSpec.newBuilder().setType("file").putOptions("format", "csv").build();
ImportSpec input = ImportSpec.newBuilder().setType("file.csv").build();
exception.expect(IllegalArgumentException.class);
exception.expectMessage(
"Validation for import spec failed: Invalid options: File path cannot be empty");
Expand All @@ -702,8 +702,7 @@ public void fileImportSpecWithoutEntityIdColumnInSchemaShouldThrowIllegalArgumen
featureInfoRepository);
ImportSpec input =
ImportSpec.newBuilder()
.setType("file")
.putOptions("format", "csv")
.setType("file.csv")
.putOptions("path", "gs://asdasd")
.build();
exception.expect(IllegalArgumentException.class);
Expand Down
135 changes: 0 additions & 135 deletions ingestion/src/main/java/feast/ingestion/transform/FeatureRowCsvIO.java

This file was deleted.

This file was deleted.

This file was deleted.

Loading

0 comments on commit b5e19bd

Please sign in to comment.