Implemented Storage 71 features (#7589)

eng/code-quality-reports/src/main/resources/checkstyle/checkstyle-suppressions.xml

@@ -234,7 +234,7 @@
   <!-- Suppress the check on any implementation code and codegen classes -->
   <suppress checks="LineLength" files=".*[/\\]storage[/\\].*implementation[/\\].*"/>
   <suppress checks="LineLength" files="com.azure.storage.blob.models.BlobErrorCode.java"/>
-  <suppress checks="LineLength" files="com.azure.storage.file.share.models.FileErrorCode.java"/>
+  <suppress checks="LineLength" files="com.azure.storage.file.share.models.ShareErrorCode.java"/>
   <suppress checks="LineLength" files="com.azure.storage.queue.models.QueueErrorCode.java"/>
 
   <!-- Suppress datalake for now as it is new development -->
@@ -249,6 +249,8 @@
   <suppress checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" files="com.azure.data.appconfiguration.ConfigurationAsyncClient.java"/>
   <suppress checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" files="com.azure.storage.file.datalake.specialized.DataLakeLeaseClient.java"/>
   <suppress checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" files="com.azure.storage.file.datalake.specialized.DataLakeLeaseAsyncClient.java"/>
+  <suppress checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" files="com.azure.storage.file.share.specialized.ShareLeaseClient.java"/>
+  <suppress checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" files="com.azure.storage.file.share.specialized.ShareLeaseAsyncClient.java"/>
 
   <!-- Suppress public/private constructor check since BlobClients need protected constructors to create EncryptedClients -->
   <suppress checks="com.azure.tools.checkstyle.checks.ServiceClientCheck" files="com.azure.storage.blob.BlobAsyncClient.java"/>

eng/code-quality-reports/src/main/resources/spotbugs/spotbugs-exclude.xml

@@ -564,6 +564,7 @@
     <Or>
       <Class name="~.*JavaDoc(CodeSnippets|CodeSamples|Samples)"/>
       <Class name="~.*ReadmeSamples"/>
+      <Class name="com.azure.storage.blob.batch.ReadmeCodeSamples"/>
       <Class name="com.azure.storage.file.datalake.GetSetAccessControlExample"/>
     </Or>
     <Bug pattern="DLS_DEAD_LOCAL_STORE,

sdk/storage/azure-storage-blob-cryptography/src/main/java/com/azure/storage/blob/specialized/cryptography/EncryptedBlobAsyncClient.java

@@ -98,7 +98,8 @@ public class EncryptedBlobAsyncClient extends BlobAsyncClient {
      */
     EncryptedBlobAsyncClient(HttpPipeline pipeline, String url, BlobServiceVersion serviceVersion, String accountName,
         String containerName, String blobName, String snapshot, AsyncKeyEncryptionKey key, String keyWrapAlgorithm) {
-        super(pipeline, url, serviceVersion, accountName, containerName, blobName, snapshot, null);
+        super(pipeline, url, serviceVersion, accountName, containerName, blobName, snapshot, null /*cpk*/,
+            null /*encryption scope*/);
 
         this.keyWrapper = key;
         this.keyWrapAlgorithm = keyWrapAlgorithm;

sdk/storage/azure-storage-blob-cryptography/src/test/java/com/azure/storage/blob/specialized/cryptography/EncyptedBlockBlobAPITest.groovy

@@ -537,23 +537,25 @@ class EncyptedBlockBlobAPITest extends APISpec {
         BlobEncryptionPolicy uploadPolicy = new BlobEncryptionPolicy(fakeKey, null)
         BlobRequestOptions uploadOptions = new BlobRequestOptions()
         uploadOptions.setEncryptionPolicy(uploadPolicy)
-        ByteArrayInputStream inputStream = new ByteArrayInputStream(defaultData.array())
 
         EncryptedBlobClient decryptClient =
             getEncryptedClientBuilder(fakeKey as AsyncKeyEncryptionKey, null, primaryCredential, cc.getBlobContainerUrl())
                 .blobName(blobName)
                 .buildEncryptedBlobClient()
 
+        def streamSize = Constants.KB + 1
+        def data = getRandomByteArray(streamSize)
+        def inputStream = new ByteArrayInputStream(data)
         when:
         // Upload with v8
-        v8EncryptBlob.upload(inputStream, defaultDataSize, null, uploadOptions, null)
+        v8EncryptBlob.upload(inputStream, streamSize, null, uploadOptions, null)
 
         // Download with current version
-        ByteArrayOutputStream stream = new ByteArrayOutputStream()
-        decryptClient.download(stream)
+        ByteArrayOutputStream outputStream = new ByteArrayOutputStream()
+        decryptClient.download(outputStream)
 
         then:
-        stream.toByteArray() == defaultData.array()
+        outputStream.toByteArray() == data
     }
 
     // Upload with new SDK download with old SDK.
@@ -572,6 +574,7 @@ class EncyptedBlockBlobAPITest extends APISpec {
         CloudStorageAccount v8Account = CloudStorageAccount.parse(connectionString)
         CloudBlobClient blobClient = v8Account.createCloudBlobClient()
         CloudBlobContainer container = blobClient.getContainerReference(containerName)
+        container.createIfNotExists()
         CloudBlockBlob v8DecryptBlob = container.getBlockBlobReference(blobName)
         BlobEncryptionPolicy policy = new BlobEncryptionPolicy(fakeKey, null)
         BlobRequestOptions downloadOptions = new BlobRequestOptions()

sdk/storage/azure-storage-blob/CHANGELOG.md

@@ -4,6 +4,10 @@
 - Added ability to access BlobProperties from BlobInputStream.
 - Modified downloadToFile to populate BlobProperties.blobSize to be the actual blob size instead of the content length of the first range. 
 
+- Added support for the 2019-07-07 service version.
+- Added support for encryption scopes service, container and blob builders now accept an encryption scope parameter and service and container builders accept a BlobContainerEncryptionScope parameter.
+- Added support for managed disk page range diff for managed disk accounts.
+
 ## 12.3.0 (2020-01-16)
 This package's
 [documentation](https://github.com/Azure/azure-sdk-for-java/blob/azure-storage-blob_12.3.0/sdk/storage/azure-storage-blob/README.md)

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobAsyncClient.java

@@ -7,6 +7,7 @@
 import com.azure.core.http.rest.Response;
 import com.azure.core.util.FluxUtil;
 import com.azure.core.util.logging.ClientLogger;
+import com.azure.storage.blob.implementation.models.EncryptionScope;
 import com.azure.storage.blob.implementation.util.ModelHelper;
 import com.azure.storage.blob.models.AccessTier;
 import com.azure.storage.blob.models.BlobHttpHeaders;
@@ -109,6 +110,28 @@ protected BlobAsyncClient(HttpPipeline pipeline, String url, BlobServiceVersion
         super(pipeline, url, serviceVersion, accountName, containerName, blobName, snapshot, customerProvidedKey);
     }
 
+    /**
+     * Protected constructor for use by {@link BlobClientBuilder}.
+     *
+     * @param pipeline The pipeline used to send and receive service requests.
+     * @param url The endpoint where to send service requests.
+     * @param serviceVersion The version of the service to receive requests.
+     * @param accountName The storage account name.
+     * @param containerName The container name.
+     * @param blobName The blob name.
+     * @param snapshot The snapshot identifier for the blob, pass {@code null} to interact with the blob directly.
+     * @param customerProvidedKey Customer provided key used during encryption of the blob's data on the server, pass
+     * {@code null} to allow the service to use its own encryption.
+     * @param encryptionScope Encryption scope used during encryption of the blob's data on the server, pass
+     * {@code null} to allow the service to use its own encryption.
+     */
+    protected BlobAsyncClient(HttpPipeline pipeline, String url, BlobServiceVersion serviceVersion, String accountName,
+        String containerName, String blobName, String snapshot, CpkInfo customerProvidedKey,
+        EncryptionScope encryptionScope) {
+        super(pipeline, url, serviceVersion, accountName, containerName, blobName, snapshot, customerProvidedKey,
+            encryptionScope);
+    }
+
     /**
      * Creates a new {@link BlobAsyncClient} linked to the {@code snapshot} of this blob resource.
      *
@@ -118,7 +141,7 @@ protected BlobAsyncClient(HttpPipeline pipeline, String url, BlobServiceVersion
     @Override
     public BlobAsyncClient getSnapshotClient(String snapshot) {
         return new BlobAsyncClient(getHttpPipeline(), getBlobUrl(), getServiceVersion(), getAccountName(),
-            getContainerName(), getBlobName(), snapshot, getCustomerProvidedKey());
+            getContainerName(), getBlobName(), snapshot, getCustomerProvidedKey(), encryptionScope);
     }
 
     /**
@@ -160,6 +183,10 @@ private SpecializedBlobClientBuilder prepareBuilder() {
             builder.customerProvidedKey(new CustomerProvidedKey(cpk.getEncryptionKey()));
         }
 
+        if (encryptionScope != null) {
+            builder.encryptionScope(encryptionScope.getEncryptionScope());
+        }
+
         return builder;
     }
 

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobClientBuilder.java

@@ -12,6 +12,7 @@
 import com.azure.core.util.Configuration;
 import com.azure.core.util.CoreUtils;
 import com.azure.core.util.logging.ClientLogger;
+import com.azure.storage.blob.implementation.models.EncryptionScope;
 import com.azure.storage.blob.implementation.util.BuilderHelper;
 import com.azure.storage.blob.models.CpkInfo;
 import com.azure.storage.blob.models.CustomerProvidedKey;
@@ -55,6 +56,7 @@ public final class BlobClientBuilder {
     private String snapshot;
 
     private CpkInfo customerProvidedKey;
+    private EncryptionScope encryptionScope;
     private StorageSharedKeyCredential storageSharedKeyCredential;
     private TokenCredential tokenCredential;
     private SasTokenCredential sasTokenCredential;
@@ -110,6 +112,11 @@ public BlobAsyncClient buildAsyncClient() {
 
         BuilderHelper.httpsValidation(customerProvidedKey, "customer provided key", endpoint, logger);
 
+        if (Objects.nonNull(customerProvidedKey) && Objects.nonNull(encryptionScope)) {
+            throw logger.logExceptionAsError(new IllegalArgumentException("Customer provided key and encryption "
+                + "scope cannot both be set"));
+        }
+
         /*
         Implicit and explicit root container access are functionally equivalent, but explicit references are easier
         to read and debug.
@@ -124,7 +131,7 @@ public BlobAsyncClient buildAsyncClient() {
             httpClient, additionalPolicies, configuration, logger);
 
         return new BlobAsyncClient(pipeline, String.format("%s/%s/%s", endpoint, blobContainerName, blobName),
-            serviceVersion, accountName, blobContainerName, blobName, snapshot, customerProvidedKey);
+            serviceVersion, accountName, blobContainerName, blobName, snapshot, customerProvidedKey, encryptionScope);
     }
 
     /**
@@ -146,6 +153,22 @@ public BlobClientBuilder customerProvidedKey(CustomerProvidedKey customerProvide
         return this;
     }
 
+    /**
+     * Sets the {@code encryption scope} that is used to encrypt blob contents on the server.
+     *
+     * @param encryptionScope Encryption scope containing the encryption key information.
+     * @return the updated BlobClientBuilder object
+     */
+    public BlobClientBuilder encryptionScope(String encryptionScope) {
+        if (encryptionScope == null) {
+            this.encryptionScope = null;
+        } else {
+            this.encryptionScope = new EncryptionScope().setEncryptionScope(encryptionScope);
+        }
+
+        return this;
+    }
+
     /**
      * Sets the {@link StorageSharedKeyCredential} used to authorize requests sent to the service.
      *

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobContainerAsyncClient.java

@@ -20,8 +20,10 @@
 import com.azure.storage.blob.implementation.models.ContainerGetPropertiesHeaders;
 import com.azure.storage.blob.implementation.models.ContainersListBlobFlatSegmentResponse;
 import com.azure.storage.blob.implementation.models.ContainersListBlobHierarchySegmentResponse;
+import com.azure.storage.blob.implementation.models.EncryptionScope;
 import com.azure.storage.blob.implementation.util.BlobSasImplUtil;
 import com.azure.storage.blob.models.BlobContainerAccessPolicies;
+import com.azure.storage.blob.models.BlobContainerEncryptionScope;
 import com.azure.storage.blob.models.BlobContainerProperties;
 import com.azure.storage.blob.models.BlobItem;
 import com.azure.storage.blob.models.BlobRequestConditions;
@@ -88,6 +90,8 @@ public final class BlobContainerAsyncClient {
     private final String containerName;
     private final BlobServiceVersion serviceVersion;
     private final CpkInfo customerProvidedKey; // only used to pass down to blob clients
+    private final EncryptionScope encryptionScope; // only used to pass down to blob clients
+    private final BlobContainerEncryptionScope blobContainerEncryptionScope;
 
     /**
      * Package-private constructor for use by {@link BlobContainerClientBuilder}.
@@ -99,9 +103,12 @@ public final class BlobContainerAsyncClient {
      * @param containerName The container name.
      * @param customerProvidedKey Customer provided key used during encryption of the blob's data on the server, pass
      * {@code null} to allow the service to use its own encryption.
+     * @param encryptionScope Encryption scope used during encryption of the blob's data on the server, pass
+     * {@code null} to allow the service to use its own encryption.
      */
     BlobContainerAsyncClient(HttpPipeline pipeline, String url, BlobServiceVersion serviceVersion,
-        String accountName, String containerName, CpkInfo customerProvidedKey) {
+        String accountName, String containerName, CpkInfo customerProvidedKey, EncryptionScope encryptionScope,
+        BlobContainerEncryptionScope blobContainerEncryptionScope) {
         this.azureBlobStorage = new AzureBlobStorageBuilder()
             .pipeline(pipeline)
             .url(url)
@@ -112,6 +119,8 @@ public final class BlobContainerAsyncClient {
         this.accountName = accountName;
         this.containerName = containerName;
         this.customerProvidedKey = customerProvidedKey;
+        this.encryptionScope = encryptionScope;
+        this.blobContainerEncryptionScope = blobContainerEncryptionScope;
     }
 
     /**
@@ -148,7 +157,7 @@ public BlobAsyncClient getBlobAsyncClient(String blobName) {
     public BlobAsyncClient getBlobAsyncClient(String blobName, String snapshot) {
         return new BlobAsyncClient(getHttpPipeline(), StorageImplUtils.appendToUrlPath(getBlobContainerUrl(),
             Utility.urlEncode(Utility.urlDecode(blobName))).toString(), getServiceVersion(), getAccountName(),
-            getBlobContainerName(), blobName, snapshot, getCustomerProvidedKey());
+            getBlobContainerName(), blobName, snapshot, getCustomerProvidedKey(), encryptionScope);
     }
 
     /**
@@ -210,6 +219,18 @@ public CpkInfo getCustomerProvidedKey() {
         return customerProvidedKey;
     }
 
+    /**
+     * Gets the {@link EncryptionScope} used to encrypt this blob's content on the server.
+     *
+     * @return the encryption scope used for encryption.
+     */
+    public String getEncryptionScope() {
+        if (encryptionScope == null) {
+            return null;
+        }
+        return encryptionScope.getEncryptionScope();
+    }
+
     /**
      * Gets if the container this client represents exists in the cloud.
      *
@@ -304,7 +325,7 @@ public Mono<Response<Void>> createWithResponse(Map<String, String> metadata, Pub
     Mono<Response<Void>> createWithResponse(Map<String, String> metadata, PublicAccessType accessType,
         Context context) {
         return this.azureBlobStorage.containers().createWithRestResponseAsync(
-            null, null, metadata, accessType, null, context)
+            null, null, metadata, accessType, null, blobContainerEncryptionScope, context)
             .map(response -> new SimpleResponse<>(response, null));
     }
 
@@ -410,7 +431,8 @@ Mono<Response<BlobContainerProperties>> getPropertiesWithResponse(String leaseId
                 ContainerGetPropertiesHeaders hd = rb.getDeserializedHeaders();
                 BlobContainerProperties properties = new BlobContainerProperties(hd.getMetadata(), hd.getETag(),
                     hd.getLastModified(), hd.getLeaseDuration(), hd.getLeaseState(), hd.getLeaseStatus(),
-                    hd.getBlobPublicAccess(), hd.isHasImmutabilityPolicy(), hd.isHasLegalHold());
+                    hd.getBlobPublicAccess(), hd.isHasImmutabilityPolicy(), hd.isHasLegalHold(),
+                    hd.getDefaultEncryptionScope(), hd.isDenyEncryptionScopeOverride());
                 return new SimpleResponse<>(rb, properties);
             });
     }

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobContainerClient.java

@@ -157,6 +157,15 @@ public CpkInfo getCustomerProvidedKey() {
         return client.getCustomerProvidedKey();
     }
 
+    /**
+     * Gets the {@code encryption scope} used to encrypt this blob's content on the server.
+     *
+     * @return the encryption scope used for encryption.
+     */
+    public String getEncryptionScope() {
+        return client.getEncryptionScope();
+    }
+
     /**
      * Gets if the container this client represents exists in the cloud.
      *

sdk/storage/azure-storage-blob/src/main/java/com/azure/storage/blob/BlobContainerClientBuilder.java

@@ -12,7 +12,9 @@
 import com.azure.core.util.Configuration;
 import com.azure.core.util.CoreUtils;
 import com.azure.core.util.logging.ClientLogger;
+import com.azure.storage.blob.implementation.models.EncryptionScope;
 import com.azure.storage.blob.implementation.util.BuilderHelper;
+import com.azure.storage.blob.models.BlobContainerEncryptionScope;
 import com.azure.storage.blob.models.CpkInfo;
 import com.azure.storage.blob.models.CustomerProvidedKey;
 import com.azure.storage.common.StorageSharedKeyCredential;
@@ -52,6 +54,8 @@ public final class BlobContainerClientBuilder {
     private String containerName;
 
     private CpkInfo customerProvidedKey;
+    private EncryptionScope encryptionScope;
+    private BlobContainerEncryptionScope blobContainerEncryptionScope;
     private StorageSharedKeyCredential storageSharedKeyCredential;
     private TokenCredential tokenCredential;
     private SasTokenCredential sasTokenCredential;
@@ -98,6 +102,11 @@ public BlobContainerClient buildClient() {
     public BlobContainerAsyncClient buildAsyncClient() {
         BuilderHelper.httpsValidation(customerProvidedKey, "customer provided key", endpoint, logger);
 
+        if (Objects.nonNull(customerProvidedKey) && Objects.nonNull(encryptionScope)) {
+            throw logger.logExceptionAsError(new IllegalArgumentException("Customer provided key and encryption "
+                + "scope cannot both be set"));
+        }
+
         /*
         Implicit and explicit root container access are functionally equivalent, but explicit references are easier
         to read and debug.
@@ -113,7 +122,8 @@ public BlobContainerAsyncClient buildAsyncClient() {
             httpClient, additionalPolicies, configuration, logger);
 
         return new BlobContainerAsyncClient(pipeline, String.format("%s/%s", endpoint, blobContainerName),
-            serviceVersion, accountName, blobContainerName, customerProvidedKey);
+            serviceVersion, accountName, blobContainerName, customerProvidedKey, encryptionScope,
+            blobContainerEncryptionScope);
     }
 
     /**
@@ -163,6 +173,35 @@ public BlobContainerClientBuilder customerProvidedKey(CustomerProvidedKey custom
         return this;
     }
 
+    /**
+     * Sets the {@code encryption scope} that is used to encrypt blob contents on the server.
+     *
+     * @param encryptionScope Encryption scope containing the encryption key information.
+     * @return the updated BlobContainerClientBuilder object
+     */
+    public BlobContainerClientBuilder encryptionScope(String encryptionScope) {
+        if (encryptionScope == null) {
+            this.encryptionScope = null;
+        } else {
+            this.encryptionScope = new EncryptionScope().setEncryptionScope(encryptionScope);
+        }
+
+        return this;
+    }
+
+    /**
+     * Sets the {@link BlobContainerEncryptionScope encryption scope} that is used to determine how blob contents are
+     * encrypted on the server.
+     *
+     * @param blobContainerEncryptionScope Encryption scope containing the encryption key information.
+     * @return the updated BlobContainerClientBuilder object
+     */
+    public BlobContainerClientBuilder blobContainerEncryptionScope(
+        BlobContainerEncryptionScope blobContainerEncryptionScope) {
+        this.blobContainerEncryptionScope = blobContainerEncryptionScope;
+        return this;
+    }
+
     /**
      * Sets the {@link StorageSharedKeyCredential} used to authorize requests sent to the service.
      *