From b0299f9dad01dde8a2f305cc9c30588512e51548 Mon Sep 17 00:00:00 2001 From: gapra Date: Tue, 4 May 2021 11:53:27 -0700 Subject: [PATCH 1/4] Hid QQ Parquet --- .../blob/implementation/util/BlobQueryReader.java | 9 ++++----- .../blob/models/BlobQueryParquetSerialization.java | 2 +- .../storage/blob/models/BlobQuerySerialization.java | 7 +++---- .../storage/blob/specialized/BlobBaseAPITest.groovy | 2 ++ .../com/azure/storage/file/datalake/Transforms.java | 12 +++++------- .../models/FileQueryParquetSerialization.java | 2 +- .../azure/storage/file/datalake/FileAPITest.groovy | 1 + 7 files changed, 17 insertions(+), 18 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobQueryReader.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobQueryReader.java index b4e4e04c04276..26cec2f3347b1 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobQueryReader.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobQueryReader.java @@ -16,7 +16,6 @@ import com.azure.storage.blob.models.BlobQueryDelimitedSerialization; import com.azure.storage.blob.models.BlobQueryError; import com.azure.storage.blob.models.BlobQueryJsonSerialization; -import com.azure.storage.blob.models.BlobQueryParquetSerialization; import com.azure.storage.blob.models.BlobQueryProgress; import com.azure.storage.blob.models.BlobQuerySerialization; import com.azure.storage.internal.avro.implementation.AvroConstants; @@ -231,12 +230,12 @@ public static QuerySerialization transformInputSerialization(BlobQuerySerializat generatedFormat.setJsonTextConfiguration(transformJson( (BlobQueryJsonSerialization) userSerialization)); - } else if (userSerialization instanceof BlobQueryParquetSerialization) { + /*} else if (userSerialization instanceof BlobQueryParquetSerialization) { generatedFormat.setType(QueryFormatType.PARQUET); generatedFormat.setParquetTextConfiguration(transformParquet( (BlobQueryParquetSerialization) userSerialization)); - + */ } else { throw logger.logExceptionAsError(new IllegalArgumentException( "Please see values of valid input serialization in the documentation " @@ -323,12 +322,12 @@ private static JsonTextConfiguration transformJson(BlobQueryJsonSerialization js * @param parquetSerialization {@link BlobQueryParquetSerialization} * @return {@link JsonTextConfiguration} */ - private static Object transformParquet(BlobQueryParquetSerialization parquetSerialization) { + /*private static Object transformParquet(BlobQueryParquetSerialization parquetSerialization) { if (parquetSerialization == null) { return null; } return new Object(); - } + }*/ /** * Transforms a BlobQueryArrowSerialization into a ArrowConfiguration. diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQueryParquetSerialization.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQueryParquetSerialization.java index 2b0220b5f1497..342780f384fa9 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQueryParquetSerialization.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQueryParquetSerialization.java @@ -6,4 +6,4 @@ /** * Defines the input parquet serialization for a blob quick query request. */ -public class BlobQueryParquetSerialization implements BlobQuerySerialization { } +class BlobQueryParquetSerialization implements BlobQuerySerialization { } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQuerySerialization.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQuerySerialization.java index e604559c42a6b..e320ba4e60e66 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQuerySerialization.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQuerySerialization.java @@ -5,11 +5,10 @@ /** * Defines the input and output serialization for a blob quick query request. - * either {@link BlobQueryJsonSerialization}, {@link BlobQueryDelimitedSerialization}, - * {@link BlobQueryArrowSerialization}, or {@link BlobQueryParquetSerialization}. + * either {@link BlobQueryJsonSerialization}, {@link BlobQueryDelimitedSerialization}, or + * {@link BlobQueryArrowSerialization}. *

- * Note: {@link BlobQueryParquetSerialization} can only be used as an input and - * {@link BlobQueryArrowSerialization} can only be used as an output. + * Note: {@link BlobQueryArrowSerialization} can only be used as an output. *

*/ public interface BlobQuerySerialization { diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobBaseAPITest.groovy b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobBaseAPITest.groovy index 6a3d91e1063ba..16f96b8899c29 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobBaseAPITest.groovy +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobBaseAPITest.groovy @@ -9,6 +9,7 @@ import reactor.core.Exceptions import spock.lang.Requires import spock.lang.Retry import spock.lang.Unroll +import spock.lang.Ignore import java.util.function.Consumer @@ -292,6 +293,7 @@ class BlobBaseAPITest extends APISpec { } @Unroll + @Ignore /* TODO: Unignore when parquet is officially supported. */ def "Query Input parquet"() { setup: String fileName = "parquet.parquet" diff --git a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/Transforms.java b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/Transforms.java index a6fa899f69401..9ed7f4b88f08e 100644 --- a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/Transforms.java +++ b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/Transforms.java @@ -25,7 +25,6 @@ import com.azure.storage.blob.models.BlobQueryError; import com.azure.storage.blob.models.BlobQueryHeaders; import com.azure.storage.blob.models.BlobQueryJsonSerialization; -import com.azure.storage.blob.models.BlobQueryParquetSerialization; import com.azure.storage.blob.models.BlobQueryProgress; import com.azure.storage.blob.models.BlobQueryResponse; import com.azure.storage.blob.models.BlobQuerySerialization; @@ -61,7 +60,6 @@ import com.azure.storage.file.datalake.models.FileQueryError; import com.azure.storage.file.datalake.models.FileQueryHeaders; import com.azure.storage.file.datalake.models.FileQueryJsonSerialization; -import com.azure.storage.file.datalake.models.FileQueryParquetSerialization; import com.azure.storage.file.datalake.models.FileQueryProgress; import com.azure.storage.file.datalake.models.FileQueryResponse; import com.azure.storage.file.datalake.models.FileQuerySerialization; @@ -446,15 +444,15 @@ static BlobQuerySerialization toBlobQuerySerialization(FileQuerySerialization se } else if (ser instanceof FileQueryArrowSerialization) { FileQueryArrowSerialization arrSer = (FileQueryArrowSerialization) ser; return new BlobQueryArrowSerialization().setSchema(toBlobQueryArrowSchema(arrSer.getSchema())); - } else if (ser instanceof FileQueryParquetSerialization) { - return new BlobQueryParquetSerialization(); + /*} else if (ser instanceof FileQueryParquetSerialization) { + return new BlobQueryParquetSerialization(); */ } else { throw new IllegalArgumentException( - String.format("'serialization' must be one of %s, %s, %s or %s", + String.format("'serialization' must be one of %s, %s, or %s", FileQueryJsonSerialization.class.getSimpleName(), FileQueryDelimitedSerialization.class.getSimpleName(), - FileQueryArrowSerialization.class.getSimpleName(), - FileQueryParquetSerialization.class.getSimpleName())); + FileQueryArrowSerialization.class.getSimpleName() + /*FileQueryParquetSerialization.class.getSimpleName()*/)); } } diff --git a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/models/FileQueryParquetSerialization.java b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/models/FileQueryParquetSerialization.java index e6ce7049c21a3..9a1b6a66c657c 100644 --- a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/models/FileQueryParquetSerialization.java +++ b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/models/FileQueryParquetSerialization.java @@ -6,4 +6,4 @@ /** * Defines the input parquet serialization for a file quick query request. */ -public class FileQueryParquetSerialization implements FileQuerySerialization { } +class FileQueryParquetSerialization implements FileQuerySerialization { } diff --git a/sdk/storage/azure-storage-file-datalake/src/test/java/com/azure/storage/file/datalake/FileAPITest.groovy b/sdk/storage/azure-storage-file-datalake/src/test/java/com/azure/storage/file/datalake/FileAPITest.groovy index d3d57384318f9..d35190974526b 100644 --- a/sdk/storage/azure-storage-file-datalake/src/test/java/com/azure/storage/file/datalake/FileAPITest.groovy +++ b/sdk/storage/azure-storage-file-datalake/src/test/java/com/azure/storage/file/datalake/FileAPITest.groovy @@ -3157,6 +3157,7 @@ class FileAPITest extends APISpec { } @Unroll + @Ignore /* TODO: Unignore when parquet is officially supported. */ def "Query Input parquet"() { setup: String fileName = "parquet.parquet" From d9598a5568ee2e01f684df1ea8f616e603b5ba5e Mon Sep 17 00:00:00 2001 From: gapra Date: Tue, 4 May 2021 11:53:50 -0700 Subject: [PATCH 2/4] Changelog --- sdk/storage/azure-storage-blob/CHANGELOG.md | 1 - sdk/storage/azure-storage-file-datalake/CHANGELOG.md | 1 - 2 files changed, 2 deletions(-) diff --git a/sdk/storage/azure-storage-blob/CHANGELOG.md b/sdk/storage/azure-storage-blob/CHANGELOG.md index 67f99ea372bab..b93ecc99a5c45 100644 --- a/sdk/storage/azure-storage-blob/CHANGELOG.md +++ b/sdk/storage/azure-storage-blob/CHANGELOG.md @@ -2,7 +2,6 @@ ## 12.12.0-beta.1 (Unreleased) - Added support for the 2020-08-04 service version. -- Added support to specify Parquet Input Serialization when querying a blob. ## 12.11.0 (2021-04-29) - Fixed a bug where large files would hang when the upload method was called. diff --git a/sdk/storage/azure-storage-file-datalake/CHANGELOG.md b/sdk/storage/azure-storage-file-datalake/CHANGELOG.md index 7068f995a7e9e..d1c405181b284 100644 --- a/sdk/storage/azure-storage-file-datalake/CHANGELOG.md +++ b/sdk/storage/azure-storage-file-datalake/CHANGELOG.md @@ -2,7 +2,6 @@ ## 12.6.0-beta.1 (Unreleased) - Added support for the 2020-08-04 service version. -- Added support to specify Parquet Input Serialization when querying a file. - Added support to undelete a file or directory - Added support to list deletedPaths - Added support to get/set service properties From b7dac5928ded0cf47b4e681880927fc5c1904e40 Mon Sep 17 00:00:00 2001 From: gapra Date: Tue, 4 May 2021 12:21:11 -0700 Subject: [PATCH 3/4] Ignored another parquet test --- .../com/azure/storage/blob/specialized/BlobBaseAPITest.groovy | 1 + .../test/java/com/azure/storage/file/datalake/FileAPITest.groovy | 1 + 2 files changed, 2 insertions(+) diff --git a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobBaseAPITest.groovy b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobBaseAPITest.groovy index 16f96b8899c29..f31d7470a91d1 100644 --- a/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobBaseAPITest.groovy +++ b/sdk/storage/azure-storage-blob/src/test/java/com/azure/storage/blob/specialized/BlobBaseAPITest.groovy @@ -695,6 +695,7 @@ class BlobBaseAPITest extends APISpec { thrown(IllegalArgumentException) } + @Ignore /* TODO: Unignore when parquet is officially supported. */ def "Query parquet output IA"() { setup: def outSer = new BlobQueryParquetSerialization() diff --git a/sdk/storage/azure-storage-file-datalake/src/test/java/com/azure/storage/file/datalake/FileAPITest.groovy b/sdk/storage/azure-storage-file-datalake/src/test/java/com/azure/storage/file/datalake/FileAPITest.groovy index d35190974526b..9ab2b0e866479 100644 --- a/sdk/storage/azure-storage-file-datalake/src/test/java/com/azure/storage/file/datalake/FileAPITest.groovy +++ b/sdk/storage/azure-storage-file-datalake/src/test/java/com/azure/storage/file/datalake/FileAPITest.groovy @@ -3528,6 +3528,7 @@ class FileAPITest extends APISpec { thrown(IllegalArgumentException) } + @Ignore /* TODO: Unignore when parquet is officially supported. */ def "Query parquet output IA"() { setup: def outSer = new FileQueryParquetSerialization() From b89de1fad0d0afd8c71cc4bdd05c6d722d2ed8f9 Mon Sep 17 00:00:00 2001 From: gapra Date: Tue, 4 May 2021 16:09:52 -0700 Subject: [PATCH 4/4] Added TODOs --- .../storage/blob/implementation/util/BlobQueryReader.java | 2 ++ .../storage/blob/models/BlobQueryParquetSerialization.java | 1 + .../azure/storage/blob/models/BlobQuerySerialization.java | 2 +- .../java/com/azure/storage/file/datalake/Transforms.java | 1 + .../file/datalake/models/FileQueryParquetSerialization.java | 1 + .../file/datalake/models/FileQuerySerialization.java | 6 +++--- 6 files changed, 9 insertions(+), 4 deletions(-) diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobQueryReader.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobQueryReader.java index 26cec2f3347b1..8f4e6bf16d8e5 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobQueryReader.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/implementation/util/BlobQueryReader.java @@ -230,6 +230,7 @@ public static QuerySerialization transformInputSerialization(BlobQuerySerializat generatedFormat.setJsonTextConfiguration(transformJson( (BlobQueryJsonSerialization) userSerialization)); + // TODO (gapra): uncomment when parquet is released /*} else if (userSerialization instanceof BlobQueryParquetSerialization) { generatedFormat.setType(QueryFormatType.PARQUET); @@ -322,6 +323,7 @@ private static JsonTextConfiguration transformJson(BlobQueryJsonSerialization js * @param parquetSerialization {@link BlobQueryParquetSerialization} * @return {@link JsonTextConfiguration} */ + // TODO (gapra): uncomment when parquet is released /*private static Object transformParquet(BlobQueryParquetSerialization parquetSerialization) { if (parquetSerialization == null) { return null; diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQueryParquetSerialization.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQueryParquetSerialization.java index 342780f384fa9..003d6fbb74a4a 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQueryParquetSerialization.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQueryParquetSerialization.java @@ -6,4 +6,5 @@ /** * Defines the input parquet serialization for a blob quick query request. */ +// TODO (gapra): public when parquet is released class BlobQueryParquetSerialization implements BlobQuerySerialization { } diff --git a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQuerySerialization.java b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQuerySerialization.java index e320ba4e60e66..1e6cb00319686 100644 --- a/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQuerySerialization.java +++ b/sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/models/BlobQuerySerialization.java @@ -2,7 +2,7 @@ // Licensed under the MIT License. package com.azure.storage.blob.models; - +// TODO (gapra): add parquet to inputs when parquet is released /** * Defines the input and output serialization for a blob quick query request. * either {@link BlobQueryJsonSerialization}, {@link BlobQueryDelimitedSerialization}, or diff --git a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/Transforms.java b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/Transforms.java index 9ed7f4b88f08e..df60063c4a830 100644 --- a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/Transforms.java +++ b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/Transforms.java @@ -444,6 +444,7 @@ static BlobQuerySerialization toBlobQuerySerialization(FileQuerySerialization se } else if (ser instanceof FileQueryArrowSerialization) { FileQueryArrowSerialization arrSer = (FileQueryArrowSerialization) ser; return new BlobQueryArrowSerialization().setSchema(toBlobQueryArrowSchema(arrSer.getSchema())); + // TODO (gapra): uncomment when parquet is released /*} else if (ser instanceof FileQueryParquetSerialization) { return new BlobQueryParquetSerialization(); */ } else { diff --git a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/models/FileQueryParquetSerialization.java b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/models/FileQueryParquetSerialization.java index 9a1b6a66c657c..5d8cc5597210f 100644 --- a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/models/FileQueryParquetSerialization.java +++ b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/models/FileQueryParquetSerialization.java @@ -6,4 +6,5 @@ /** * Defines the input parquet serialization for a file quick query request. */ +// TODO (gapra): public when parquet is released class FileQueryParquetSerialization implements FileQuerySerialization { } diff --git a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/models/FileQuerySerialization.java b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/models/FileQuerySerialization.java index 63ed8c2c3d28a..0b5f12eeedc70 100644 --- a/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/models/FileQuerySerialization.java +++ b/sdk/storage/azure-storage-file-datalake/src/main/java/com/azure/storage/file/datalake/models/FileQuerySerialization.java @@ -3,13 +3,13 @@ package com.azure.storage.file.datalake.models; +// TODO (gapra): add parquet to inputs when parquet is released /** * Defines the input and output serialization for a file quick query request. * either {@link FileQueryJsonSerialization}, {@link FileQueryDelimitedSerialization}, - * {@link FileQueryArrowSerialization}, or {@link FileQueryParquetSerialization}. + * {@link FileQueryArrowSerialization}. *

- * Note: {@link FileQueryParquetSerialization} can only be used as an input and - * {@link FileQueryArrowSerialization} can only be used as an output. + * Note: {@link FileQueryArrowSerialization} can only be used as an output. *

*/ public interface FileQuerySerialization {