Datalake CPK (#22159)

* corrected release date in changelog

* datalake cpk

* Update CHANGELOG.md

add newline changelog

* download file & stream tests

* recordings

* recordings

* change to unrecorded test

* add negative tests for CPK

sdk/storage/azdatalake/CHANGELOG.md

@@ -3,6 +3,8 @@
 ## 1.0.1 (Unreleased)
 
 ### Features Added
+* Encryption Scope For SAS
+* CPK for Datalake
 
 ### Breaking Changes
 

sdk/storage/azdatalake/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "go",
   "TagPrefix": "go/storage/azdatalake",
-  "Tag": "go/storage/azdatalake_1ee92f87a9"
+  "Tag": "go/storage/azdatalake_36b1978c0d"
 }

sdk/storage/azdatalake/directory/client.go

@@ -295,7 +295,7 @@ func (d *Client) Rename(ctx context.Context, destinationPath string, options *Re
 		newPathURL = strings.Split(newPathURL, "?")[0] + "?" + newDestQuery
 	}
 	newBlobURL, _ := shared.GetURLs(newPathURL)
-	lac, mac, smac, createOpts := path.FormatRenameOptions(options, newSrcPath)
+	lac, mac, smac, createOpts, cpkOpts := path.FormatRenameOptions(options, newSrcPath)
 
 	if d.identityCredential() != nil {
 		newBlobClient, err = blockblob.NewClient(newBlobURL, *d.identityCredential(), nil)
@@ -310,7 +310,7 @@ func (d *Client) Rename(ctx context.Context, destinationPath string, options *Re
 		return RenameResponse{}, exported.ConvertToDFSError(err)
 	}
 	newDirClient := (*Client)(base.NewPathClient(newPathURL, newBlobURL, newBlobClient, d.generatedDirClientWithDFS().InternalClient().WithClientName(shared.DirectoryClient), d.sharedKey(), d.identityCredential(), d.getClientOptions()))
-	resp, err := newDirClient.generatedDirClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, nil)
+	resp, err := newDirClient.generatedDirClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, cpkOpts)
 	//return RenameResponse{
 	//	Response:           resp,
 	//	NewDirectoryClient: newDirClient,

sdk/storage/azdatalake/directory/client_test.go

@@ -147,6 +147,31 @@ func (s *RecordedTestSuite) TestCreateDirAndDelete() {
 	_require.NotNil(resp)
 }
 
+func (s *RecordedTestSuite) TestCreateDirUsingCPK() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	dirOpts := &directory.CreateOptions{CPKInfo: &testcommon.TestCPKByValue}
+	resp, err := dirClient.Create(context.Background(), dirOpts)
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	_require.Equal(true, *(resp.IsServerEncrypted))
+	_require.Equal(testcommon.TestCPKByValue.EncryptionKeySHA256, resp.EncryptionKeySHA256)
+}
+
 func (s *RecordedTestSuite) TestGetAndCreateFileClient() {
 	_require := require.New(s.T())
 	testName := s.T().Name()
@@ -1912,6 +1937,65 @@ func (s *RecordedTestSuite) TestDirSetMetadataWithAccessConditions() {
 	_require.NoError(err)
 }
 
+func (s *RecordedTestSuite) TestDirSetMetadataWithCPK() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	defer testcommon.DeleteDir(context.Background(), _require, dirClient)
+
+	resp, err := dirClient.Create(context.Background(), &directory.CreateOptions{CPKInfo: &testcommon.TestCPKByValue})
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	opts := &directory.SetMetadataOptions{
+		CPKInfo: &testcommon.TestCPKByValue,
+	}
+	_, err = dirClient.SetMetadata(context.Background(), testcommon.BasicMetadata, opts)
+	_require.NoError(err)
+}
+
+func (s *RecordedTestSuite) TestDirSetMetadataWithCPKNegative() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	defer testcommon.DeleteDir(context.Background(), _require, dirClient)
+
+	resp, err := dirClient.Create(context.Background(), nil)
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	opts := &directory.SetMetadataOptions{
+		CPKInfo: &testcommon.TestCPKByValue,
+	}
+	_, err = dirClient.SetMetadata(context.Background(), testcommon.BasicMetadata, opts)
+	_require.Error(err)
+	_require.ErrorContains(err, "PathDoesNotUseCustomerSpecifiedEncryption")
+}
+
 func validatePropertiesSet(_require *require.Assertions, dirClient *directory.Client, disposition string) {
 	resp, err := dirClient.GetProperties(context.Background(), nil)
 	_require.NoError(err)
@@ -2243,6 +2327,38 @@ func (s *RecordedTestSuite) TestDirRenameNoOptions() {
 	//_require.Contains(resp1.NewDirectoryClient.DFSURL(), "newName")
 }
 
+func (s *RecordedTestSuite) TestDirRenameRequestWithCPK() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	createOpts := &directory.CreateOptions{
+		CPKInfo: &testcommon.TestCPKByValue,
+	}
+
+	resp, err := dirClient.Create(context.Background(), createOpts)
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	renameFileOpts := &directory.RenameOptions{
+		CPKInfo: &testcommon.TestCPKByValue,
+	}
+
+	_, err = dirClient.Rename(context.Background(), "newName", renameFileOpts)
+	_require.NoError(err)
+}
+
 func (s *RecordedTestSuite) TestRenameDirWithNilAccessConditions() {
 	_require := require.New(s.T())
 	testName := s.T().Name()
@@ -2548,6 +2664,35 @@ func (s *RecordedTestSuite) TestDirGetPropertiesResponseCapture() {
 	_require.Equal("directory", respFromCtxService.Header.Get("x-ms-resource-type"))
 }
 
+func (s *RecordedTestSuite) TestDirGetPropertiesWithCPK() {
+	_require := require.New(s.T())
+	testName := s.T().Name()
+
+	filesystemName := testcommon.GenerateFileSystemName(testName)
+	fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+	defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient)
+
+	_, err = fsClient.Create(context.Background(), nil)
+	_require.NoError(err)
+
+	dirName := testcommon.GenerateDirName(testName)
+	dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil)
+	_require.NoError(err)
+
+	dirOpts := &directory.CreateOptions{CPKInfo: &testcommon.TestCPKByValue}
+	resp, err := dirClient.Create(context.Background(), dirOpts)
+	_require.NoError(err)
+	_require.NotNil(resp)
+
+	getPropertiesOpts := &directory.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue}
+	response, err := dirClient.GetProperties(context.Background(), getPropertiesOpts)
+	_require.NoError(err)
+	_require.NotNil(response)
+	_require.Equal(*(resp.IsServerEncrypted), true)
+	_require.Equal(resp.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256)
+}
+
 func (s *UnrecordedTestSuite) TestDirCreateDeleteUsingOAuth() {
 	_require := require.New(s.T())
 	testName := s.T().Name()

sdk/storage/azdatalake/file/client.go

@@ -278,7 +278,7 @@ func (f *Client) Rename(ctx context.Context, destinationPath string, options *Re
 		newPathURL = strings.Split(newPathURL, "?")[0] + "?" + newDestQuery
 	}
 	newBlobURL, _ := shared.GetURLs(newPathURL)
-	lac, mac, smac, createOpts := path.FormatRenameOptions(options, newSrcPath)
+	lac, mac, smac, createOpts, cpkOpts := path.FormatRenameOptions(options, newSrcPath)
 
 	if f.identityCredential() != nil {
 		newBlobClient, err = blockblob.NewClient(newBlobURL, *f.identityCredential(), nil)
@@ -293,7 +293,7 @@ func (f *Client) Rename(ctx context.Context, destinationPath string, options *Re
 		return RenameResponse{}, exported.ConvertToDFSError(err)
 	}
 	newFileClient := (*Client)(base.NewPathClient(newPathURL, newBlobURL, newBlobClient, f.generatedFileClientWithDFS().InternalClient().WithClientName(shared.FileClient), f.sharedKey(), f.identityCredential(), f.getClientOptions()))
-	resp, err := newFileClient.generatedFileClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, nil)
+	resp, err := newFileClient.generatedFileClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, cpkOpts)
 
 	//return RenameResponse{
 	//	Response:      resp,