From a0ec87456dd450d203bcd93d91ddbcb6270df30a Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Wed, 15 Nov 2023 21:14:54 +0530 Subject: [PATCH 1/8] corrected release date in changelog --- sdk/storage/azfile/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/azfile/CHANGELOG.md b/sdk/storage/azfile/CHANGELOG.md index 25b07c61f211..bee741d21d06 100644 --- a/sdk/storage/azfile/CHANGELOG.md +++ b/sdk/storage/azfile/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.1.1 (2023-12-05) +## 1.1.1 (2023-11-15) ### Bugs Fixed From f0a41d42c1a1a6daf20f59eb42deafb794fa1560 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Tue, 19 Dec 2023 11:53:26 +0530 Subject: [PATCH 2/8] datalake cpk --- sdk/storage/azdatalake/CHANGELOG.md | 2 + sdk/storage/azdatalake/assets.json | 2 +- sdk/storage/azdatalake/directory/client.go | 4 +- .../azdatalake/directory/client_test.go | 115 +++++++++ sdk/storage/azdatalake/file/client.go | 4 +- sdk/storage/azdatalake/file/client_test.go | 227 +++++++++++++++++- .../azdatalake/internal/path/models.go | 18 +- .../internal/testcommon/clients_auth.go | 11 + 8 files changed, 371 insertions(+), 12 deletions(-) diff --git a/sdk/storage/azdatalake/CHANGELOG.md b/sdk/storage/azdatalake/CHANGELOG.md index 62598c61d6e2..bc7f4bc6c481 100644 --- a/sdk/storage/azdatalake/CHANGELOG.md +++ b/sdk/storage/azdatalake/CHANGELOG.md @@ -3,6 +3,8 @@ ## 1.0.1 (Unreleased) ### Features Added +* Encryption Scope For SAS +* CPK for Datalake ### Breaking Changes diff --git a/sdk/storage/azdatalake/assets.json b/sdk/storage/azdatalake/assets.json index 2a8f02268497..ed7099ce4794 100644 --- a/sdk/storage/azdatalake/assets.json +++ b/sdk/storage/azdatalake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "go", "TagPrefix": "go/storage/azdatalake", - "Tag": "go/storage/azdatalake_7c0fb050cb" + "Tag": "go/storage/azdatalake_d78e1df2e9" } \ No newline at end of file diff --git a/sdk/storage/azdatalake/directory/client.go b/sdk/storage/azdatalake/directory/client.go index 45f4f0b31c94..97e907e11f7c 100644 --- a/sdk/storage/azdatalake/directory/client.go +++ b/sdk/storage/azdatalake/directory/client.go @@ -295,7 +295,7 @@ func (d *Client) Rename(ctx context.Context, destinationPath string, options *Re newPathURL = strings.Split(newPathURL, "?")[0] + "?" + newDestQuery } newBlobURL, _ := shared.GetURLs(newPathURL) - lac, mac, smac, createOpts := path.FormatRenameOptions(options, newSrcPath) + lac, mac, smac, createOpts, cpkOpts := path.FormatRenameOptions(options, newSrcPath) if d.identityCredential() != nil { newBlobClient, err = blockblob.NewClient(newBlobURL, *d.identityCredential(), nil) @@ -310,7 +310,7 @@ func (d *Client) Rename(ctx context.Context, destinationPath string, options *Re return RenameResponse{}, exported.ConvertToDFSError(err) } newDirClient := (*Client)(base.NewPathClient(newPathURL, newBlobURL, newBlobClient, d.generatedDirClientWithDFS().InternalClient().WithClientName(shared.DirectoryClient), d.sharedKey(), d.identityCredential(), d.getClientOptions())) - resp, err := newDirClient.generatedDirClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, nil) + resp, err := newDirClient.generatedDirClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, cpkOpts) //return RenameResponse{ // Response: resp, // NewDirectoryClient: newDirClient, diff --git a/sdk/storage/azdatalake/directory/client_test.go b/sdk/storage/azdatalake/directory/client_test.go index cc2f7bab8ee0..53ef88fd0349 100644 --- a/sdk/storage/azdatalake/directory/client_test.go +++ b/sdk/storage/azdatalake/directory/client_test.go @@ -147,6 +147,31 @@ func (s *RecordedTestSuite) TestCreateDirAndDelete() { _require.NotNil(resp) } +func (s *RecordedTestSuite) TestCreateDirUsingCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + dirName := testcommon.GenerateDirName(testName) + dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + dirOpts := &directory.CreateOptions{CPKInfo: &testcommon.TestCPKByValue} + resp, err := dirClient.Create(context.Background(), dirOpts) + _require.NoError(err) + _require.NotNil(resp) + + _require.Equal(true, *(resp.IsServerEncrypted)) + _require.Equal(testcommon.TestCPKByValue.EncryptionKeySHA256, resp.EncryptionKeySHA256) +} + func (s *RecordedTestSuite) TestGetAndCreateFileClient() { _require := require.New(s.T()) testName := s.T().Name() @@ -1912,6 +1937,35 @@ func (s *RecordedTestSuite) TestDirSetMetadataWithAccessConditions() { _require.NoError(err) } +func (s *RecordedTestSuite) TestDirSetMetadataWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + dirName := testcommon.GenerateDirName(testName) + dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + defer testcommon.DeleteDir(context.Background(), _require, dirClient) + + resp, err := dirClient.Create(context.Background(), &directory.CreateOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.NotNil(resp) + + opts := &directory.SetMetadataOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + _, err = dirClient.SetMetadata(context.Background(), testcommon.BasicMetadata, opts) + _require.NoError(err) +} + func validatePropertiesSet(_require *require.Assertions, dirClient *directory.Client, disposition string) { resp, err := dirClient.GetProperties(context.Background(), nil) _require.NoError(err) @@ -2243,6 +2297,38 @@ func (s *RecordedTestSuite) TestDirRenameNoOptions() { //_require.Contains(resp1.NewDirectoryClient.DFSURL(), "newName") } +func (s *RecordedTestSuite) TestDirRenameRequestWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + dirName := testcommon.GenerateDirName(testName) + dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + createOpts := &directory.CreateOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + + resp, err := dirClient.Create(context.Background(), createOpts) + _require.NoError(err) + _require.NotNil(resp) + + renameFileOpts := &directory.RenameOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + + _, err = dirClient.Rename(context.Background(), "newName", renameFileOpts) + _require.NoError(err) +} + func (s *RecordedTestSuite) TestRenameDirWithNilAccessConditions() { _require := require.New(s.T()) testName := s.T().Name() @@ -2548,6 +2634,35 @@ func (s *RecordedTestSuite) TestDirGetPropertiesResponseCapture() { _require.Equal("directory", respFromCtxService.Header.Get("x-ms-resource-type")) } +func (s *RecordedTestSuite) TestDirGetPropertiesWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + dirName := testcommon.GenerateDirName(testName) + dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + dirOpts := &directory.CreateOptions{CPKInfo: &testcommon.TestCPKByValue} + resp, err := dirClient.Create(context.Background(), dirOpts) + _require.NoError(err) + _require.NotNil(resp) + + getPropertiesOpts := &directory.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue} + response, err := dirClient.GetProperties(context.Background(), getPropertiesOpts) + _require.NoError(err) + _require.NotNil(response) + _require.Equal(*(resp.IsServerEncrypted), true) + _require.Equal(resp.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256) +} + func (s *UnrecordedTestSuite) TestDirCreateDeleteUsingOAuth() { _require := require.New(s.T()) testName := s.T().Name() diff --git a/sdk/storage/azdatalake/file/client.go b/sdk/storage/azdatalake/file/client.go index 58b97cd5d342..be8c5a3ec84f 100644 --- a/sdk/storage/azdatalake/file/client.go +++ b/sdk/storage/azdatalake/file/client.go @@ -278,7 +278,7 @@ func (f *Client) Rename(ctx context.Context, destinationPath string, options *Re newPathURL = strings.Split(newPathURL, "?")[0] + "?" + newDestQuery } newBlobURL, _ := shared.GetURLs(newPathURL) - lac, mac, smac, createOpts := path.FormatRenameOptions(options, newSrcPath) + lac, mac, smac, createOpts, cpkOpts := path.FormatRenameOptions(options, newSrcPath) if f.identityCredential() != nil { newBlobClient, err = blockblob.NewClient(newBlobURL, *f.identityCredential(), nil) @@ -293,7 +293,7 @@ func (f *Client) Rename(ctx context.Context, destinationPath string, options *Re return RenameResponse{}, exported.ConvertToDFSError(err) } newFileClient := (*Client)(base.NewPathClient(newPathURL, newBlobURL, newBlobClient, f.generatedFileClientWithDFS().InternalClient().WithClientName(shared.FileClient), f.sharedKey(), f.identityCredential(), f.getClientOptions())) - resp, err := newFileClient.generatedFileClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, nil) + resp, err := newFileClient.generatedFileClientWithDFS().Create(ctx, createOpts, nil, lac, mac, smac, cpkOpts) //return RenameResponse{ // Response: resp, diff --git a/sdk/storage/azdatalake/file/client_test.go b/sdk/storage/azdatalake/file/client_test.go index 7294f001d8c4..bd4eb6b2f710 100644 --- a/sdk/storage/azdatalake/file/client_test.go +++ b/sdk/storage/azdatalake/file/client_test.go @@ -169,6 +169,35 @@ func (s *RecordedTestSuite) TestCreateFileWithNilAccessConditions() { _require.NotNil(resp) } +func (s *RecordedTestSuite) TestCreateFileWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + defer testcommon.DeleteFile(context.Background(), _require, fClient) + + createFileOpts := &file.CreateOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + + resp, err := fClient.Create(context.Background(), createFileOpts) + _require.NoError(err) + _require.NotNil(resp) + _require.Equal(*(resp.IsServerEncrypted), true) + _require.Equal(resp.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256) +} + func (s *RecordedTestSuite) TestCreateFileIfModifiedSinceTrue() { _require := require.New(s.T()) testName := s.T().Name() @@ -1719,6 +1748,37 @@ func (s *RecordedTestSuite) TestFileSetMetadataWithAccessConditions() { _require.NoError(err) } +func (s *RecordedTestSuite) TestFileSetMetadataWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + defer testcommon.DeleteFile(context.Background(), _require, fClient) + + resp, err := fClient.Create(context.Background(), &file.CreateOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.NotNil(resp) + + opts := &file.SetMetadataOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + res, err := fClient.SetMetadata(context.Background(), testcommon.BasicMetadata, opts) + _require.NoError(err) + _require.Equal(*(res.IsServerEncrypted), true) + _require.Equal(res.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256) //TODO: Tanya +} + func validatePropertiesSet(_require *require.Assertions, fileClient *file.Client, disposition string) { resp, err := fileClient.GetProperties(context.Background(), nil) _require.NoError(err) @@ -2043,11 +2103,36 @@ func (s *RecordedTestSuite) TestRenameNoOptions() { _require.NoError(err) _require.NotNil(resp) - //resp1, err := fClient.Rename(context.Background(), "newName", renameFileOpts) _, err = fClient.Rename(context.Background(), "newName", nil) _require.NoError(err) - //_require.NotNil(resp1) - //_require.Contains(resp1.NewFileClient.DFSURL(), "newName") +} + +func (s *RecordedTestSuite) TestRenameFileWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + resp, err := fClient.Create(context.Background(), nil) + _require.NoError(err) + _require.NotNil(resp) + + renameFileOpts := &file.RenameOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + + _, err = fClient.Rename(context.Background(), "newName", renameFileOpts) + _require.NoError(err) } func (s *RecordedTestSuite) TestRenameFileWithNilAccessConditions() { @@ -3109,6 +3194,47 @@ func (s *RecordedTestSuite) TestFileAppendAndFlushDataWithEmptyOpts() { _require.Equal(*gResp2.ContentLength, int64(contentSize)) } +func (s *RecordedTestSuite) TestFileAppendAndFlushDataWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + srcFileName := "src" + testcommon.GenerateFileName(testName) + + srcFClient, err := testcommon.GetFileClient(filesystemName, srcFileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + createOptions := &file.CreateOptions{CPKInfo: &testcommon.TestCPKByValue} + resp, err := srcFClient.Create(context.Background(), createOptions) + _require.NoError(err) + _require.NotNil(resp) + + contentSize := 1024 * 8 // 8KB + rsc, _ := testcommon.GenerateData(contentSize) + + opts := &file.AppendDataOptions{CPKInfo: &testcommon.TestCPKByValue} + opts1 := &file.FlushDataOptions{CPKInfo: &testcommon.TestCPKByValue} + + _, err = srcFClient.AppendData(context.Background(), 0, rsc, opts) + _require.NoError(err) + + _, err = srcFClient.FlushData(context.Background(), int64(contentSize), opts1) + _require.NoError(err) + getPropertiesOptions := &file.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue} + gResp2, err := srcFClient.GetProperties(context.Background(), getPropertiesOptions) + _require.NoError(err) + _require.Equal(*gResp2.ContentLength, int64(contentSize)) + _require.Equal(true, *(gResp2.IsServerEncrypted)) + _require.Equal(testcommon.TestCPKByValue.EncryptionKeySHA256, gResp2.EncryptionKeySHA256) +} + func (s *RecordedTestSuite) TestFileAppendAndFlushDataWithLeasedFile() { _require := require.New(s.T()) testName := s.T().Name() @@ -4187,6 +4313,55 @@ func (s *RecordedTestSuite) TestFileDownloadSmallBufferWithAccessConditions() { _require.Equal(*gResp2.ContentLength, fileSize) } +func (s *RecordedTestSuite) TestFileDownloadBufferWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + var fileSize int64 = 10 * 1024 + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + fileCreateOpts := &file.CreateOptions{CPKInfo: &testcommon.TestCPKByValue} + resp, err := fClient.Create(context.Background(), fileCreateOpts) + _require.NoError(err) + _require.NotNil(resp) + + _, content := testcommon.GenerateData(int(fileSize)) + md5Value := md5.Sum(content[0:fileSize]) + contentMD5 := md5Value[:] + + err = fClient.UploadBuffer(context.Background(), content, &file.UploadBufferOptions{ + CPKInfo: &testcommon.TestCPKByValue, + }) + _require.NoError(err) + + destBuffer := make([]byte, fileSize) + _, err = fClient.DownloadBuffer(context.Background(), destBuffer, &file.DownloadBufferOptions{ + CPKInfo: &testcommon.TestCPKByValue, + }) + _require.NoError(err) + + downloadedMD5Value := md5.Sum(destBuffer) + downloadedContentMD5 := downloadedMD5Value[:] + + _require.EqualValues(downloadedContentMD5, contentMD5) + + gResp2, err := fClient.GetProperties(context.Background(), &file.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.Equal(*gResp2.ContentLength, fileSize) + _require.Equal(*(gResp2.IsServerEncrypted), true) + _require.Equal(gResp2.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256) +} + func (s *RecordedTestSuite) TestFileGetPropertiesResponseCapture() { _require := require.New(s.T()) testName := s.T().Name() @@ -4263,6 +4438,52 @@ func (s *RecordedTestSuite) TestFileGetPropertiesResponseCapture() { _require.Equal("file", respFromCtxDir.Header.Get("x-ms-resource-type")) } +func (s *RecordedTestSuite) TestFileGetPropertiesWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + dirName := testcommon.GenerateDirName(testName) + dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + resp, err := dirClient.Create(context.Background(), nil) + _require.NoError(err) + _require.NotNil(resp) + + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, dirName+"/"+fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + createFileOpts := &file.CreateOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + + resp, err = fClient.Create(context.Background(), createFileOpts) + _require.NoError(err) + _require.NotNil(resp) + + GetPropertiesOpts := &file.GetPropertiesOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + + // This tests file.NewClient + var respFromCtxFile *http.Response + ctxWithRespFile := runtime.WithCaptureResponse(context.Background(), &respFromCtxFile) + response, err := fClient.GetProperties(ctxWithRespFile, GetPropertiesOpts) + _require.NoError(err) + _require.NotNil(response) + _require.NotNil(respFromCtxFile.Header.Get("x-ms-encryption-key-sha256")) // validate that the x-ms-encryption-key-sha256 is actually populated + _require.Equal(testcommon.TestCPKByValue.EncryptionKeySHA256, response.EncryptionKeySHA256) +} + func (s *UnrecordedTestSuite) TestFileCreateDeleteUsingOAuth() { _require := require.New(s.T()) testName := s.T().Name() diff --git a/sdk/storage/azdatalake/internal/path/models.go b/sdk/storage/azdatalake/internal/path/models.go index 3a25acce5a27..64cb04cc1aa5 100644 --- a/sdk/storage/azdatalake/internal/path/models.go +++ b/sdk/storage/azdatalake/internal/path/models.go @@ -38,9 +38,11 @@ type RenameOptions struct { SourceAccessConditions *SourceAccessConditions // AccessConditions contains parameters for accessing the path. AccessConditions *AccessConditions + // CPKInfo contains CPK related information. + CPKInfo *CPKInfo } -func FormatRenameOptions(o *RenameOptions, path string) (*generated.LeaseAccessConditions, *generated.ModifiedAccessConditions, *generated.SourceModifiedAccessConditions, *generated.PathClientCreateOptions) { +func FormatRenameOptions(o *RenameOptions, path string) (*generated.LeaseAccessConditions, *generated.ModifiedAccessConditions, *generated.SourceModifiedAccessConditions, *generated.PathClientCreateOptions, *generated.CPKInfo) { // we don't need sourceModAccCond since this is not rename mode := generated.PathRenameModeLegacy createOpts := &generated.PathClientCreateOptions{ @@ -48,7 +50,15 @@ func FormatRenameOptions(o *RenameOptions, path string) (*generated.LeaseAccessC RenameSource: &path, } if o == nil { - return nil, nil, nil, createOpts + return nil, nil, nil, createOpts, nil + } + var cpkOpts *generated.CPKInfo + if o.CPKInfo != nil { + cpkOpts = &generated.CPKInfo{ + EncryptionAlgorithm: o.CPKInfo.EncryptionAlgorithm, + EncryptionKey: o.CPKInfo.EncryptionKey, + EncryptionKeySHA256: o.CPKInfo.EncryptionKeySHA256, + } } leaseAccessConditions, modifiedAccessConditions := exported.FormatPathAccessConditions(o.AccessConditions) if o.SourceAccessConditions != nil { @@ -62,10 +72,10 @@ func FormatRenameOptions(o *RenameOptions, path string) (*generated.LeaseAccessC SourceIfNoneMatch: o.SourceAccessConditions.SourceModifiedAccessConditions.SourceIfNoneMatch, SourceIfUnmodifiedSince: o.SourceAccessConditions.SourceModifiedAccessConditions.SourceIfUnmodifiedSince, } - return leaseAccessConditions, modifiedAccessConditions, sourceModifiedAccessConditions, createOpts + return leaseAccessConditions, modifiedAccessConditions, sourceModifiedAccessConditions, createOpts, cpkOpts } } - return leaseAccessConditions, modifiedAccessConditions, nil, createOpts + return leaseAccessConditions, modifiedAccessConditions, nil, createOpts, cpkOpts } // GetPropertiesOptions contains the optional parameters for the Client.GetProperties method. diff --git a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go index d79e3c16e2fd..1ef91c058893 100644 --- a/sdk/storage/azdatalake/internal/testcommon/clients_auth.go +++ b/sdk/storage/azdatalake/internal/testcommon/clients_auth.go @@ -53,6 +53,17 @@ var ( DatalakeContentEncoding = "my_encoding" ) +var ( + testEncryptedKey = "MDEyMzQ1NjcwMTIzNDU2NzAxMjM0NTY3MDEyMzQ1Njc=" + testEncryptedHash = "3QFFFpRA5+XANHqwwbT4yXDmrT/2JaLt/FKHjzhOdoE=" + testEncryptionAlgorithm = file.EncryptionAlgorithmTypeAES256 + TestCPKByValue = file.CPKInfo{ + EncryptionKey: &testEncryptedKey, + EncryptionKeySHA256: &testEncryptedHash, + EncryptionAlgorithm: &testEncryptionAlgorithm, + } +) + var BasicHeaders = file.HTTPHeaders{ ContentType: &DatalakeContentType, ContentDisposition: &DatalakeContentDisposition, From 6e558f7ef925f61b6235f2c5b268fc672c3661ad Mon Sep 17 00:00:00 2001 From: tanyasethi-msft <124860586+tanyasethi-msft@users.noreply.github.com> Date: Tue, 19 Dec 2023 15:39:25 +0530 Subject: [PATCH 3/8] Update CHANGELOG.md add newline changelog --- sdk/storage/azfile/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sdk/storage/azfile/CHANGELOG.md b/sdk/storage/azfile/CHANGELOG.md index 16e3633119af..390a176fd2a2 100644 --- a/sdk/storage/azfile/CHANGELOG.md +++ b/sdk/storage/azfile/CHANGELOG.md @@ -1,5 +1,6 @@ # Release History + ## 1.1.2 (Unreleased) ### Features Added @@ -10,6 +11,7 @@ ### Other Changes + ## 1.1.1 (2023-11-15) ### Bugs Fixed From e60de746b206c122ef4b178dfddbc37330d13f34 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Tue, 19 Dec 2023 15:40:10 +0530 Subject: [PATCH 4/8] download file & stream tests --- sdk/storage/azdatalake/assets.json | 2 +- sdk/storage/azdatalake/file/client_test.go | 142 ++++++++++++++++++++- 2 files changed, 142 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azdatalake/assets.json b/sdk/storage/azdatalake/assets.json index ed7099ce4794..a4aaa09d1502 100644 --- a/sdk/storage/azdatalake/assets.json +++ b/sdk/storage/azdatalake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "go", "TagPrefix": "go/storage/azdatalake", - "Tag": "go/storage/azdatalake_d78e1df2e9" + "Tag": "go/storage/azdatalake_5debe7fb7b" } \ No newline at end of file diff --git a/sdk/storage/azdatalake/file/client_test.go b/sdk/storage/azdatalake/file/client_test.go index bd4eb6b2f710..8d4ee9617a7e 100644 --- a/sdk/storage/azdatalake/file/client_test.go +++ b/sdk/storage/azdatalake/file/client_test.go @@ -1776,7 +1776,7 @@ func (s *RecordedTestSuite) TestFileSetMetadataWithCPK() { res, err := fClient.SetMetadata(context.Background(), testcommon.BasicMetadata, opts) _require.NoError(err) _require.Equal(*(res.IsServerEncrypted), true) - _require.Equal(res.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256) //TODO: Tanya + _require.Equal(res.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256) } func validatePropertiesSet(_require *require.Assertions, fileClient *file.Client, disposition string) { @@ -2531,6 +2531,57 @@ func (s *RecordedTestSuite) TestFileUploadTinyStream() { _require.EqualValues(downloadedContentMD5, contentMD5) } +func (s *RecordedTestSuite) TestFileUploadDownloadStreamWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + var fileSize int64 = 1 * 1024 * 1024 + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + resp, err := fClient.Create(context.Background(), &file.CreateOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.NotNil(resp) + + content := make([]byte, fileSize) + _, err = rand.Read(content) + _require.NoError(err) + md5Value := md5.Sum(content) + contentMD5 := md5Value[:] + + err = fClient.UploadStream(context.Background(), streaming.NopCloser(bytes.NewReader(content)), &file.UploadStreamOptions{ + Concurrency: 5, + CPKInfo: &testcommon.TestCPKByValue, + }) + _require.NoError(err) + + gResp2, err := fClient.GetProperties(context.Background(), &file.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.Equal(*gResp2.ContentLength, fileSize) + + dResp, err := fClient.DownloadStream(context.Background(), &file.DownloadStreamOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + + data, err := io.ReadAll(dResp.Body) + _require.NoError(err) + + downloadedMD5Value := md5.Sum(data) + downloadedContentMD5 := downloadedMD5Value[:] + + _require.EqualValues(downloadedContentMD5, contentMD5) + _require.Equal(true, *(dResp.IsServerEncrypted)) + _require.Equal(testcommon.TestCPKByValue.EncryptionKeySHA256, dResp.EncryptionKeySHA256) +} + func (s *UnrecordedTestSuite) TestFileUploadFile() { _require := require.New(s.T()) testName := s.T().Name() @@ -4039,6 +4090,95 @@ func (s *RecordedTestSuite) TestFileUploadDownloadSmallFileWithAccessConditions( _require.Equal(*gResp2.ContentLength, fileSize) } +func (s *RecordedTestSuite) TestFileUploadDownloadSmallFileWithCPK() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + var fileSize int64 = 10 * 1024 + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + resp, err := fClient.Create(context.Background(), &file.CreateOptions{ + CPKInfo: &testcommon.TestCPKByValue, + }) + _require.NoError(err) + _require.NotNil(resp) + + // create local file + _, content := testcommon.GenerateData(int(fileSize)) + srcFileName := "testFileUpload" + err = os.WriteFile(srcFileName, content, 0644) + _require.NoError(err) + defer func() { + err = os.Remove(srcFileName) + _require.NoError(err) + }() + fh, err := os.Open(srcFileName) + _require.NoError(err) + defer func(fh *os.File) { + err := fh.Close() + _require.NoError(err) + }(fh) + + srcHash := md5.New() + _, err = io.Copy(srcHash, fh) + _require.NoError(err) + contentMD5 := srcHash.Sum(nil) + + err = fClient.UploadFile(context.Background(), fh, &file.UploadFileOptions{ + Concurrency: 5, + ChunkSize: 2 * 1024, + CPKInfo: &testcommon.TestCPKByValue, + }) + _require.NoError(err) + + destFileName := "SmallFile-downloaded.bin" + destFile, err := os.Create(destFileName) + _require.NoError(err) + defer func(name string) { + err = os.Remove(name) + _require.NoError(err) + }(destFileName) + defer func(destFile *os.File) { + err = destFile.Close() + _require.NoError(err) + }(destFile) + + cnt, err := fClient.DownloadFile(context.Background(), destFile, &file.DownloadFileOptions{ + ChunkSize: 2 * 1024, + Concurrency: 5, + Range: &file.HTTPRange{ + Offset: 0, + Count: 10 * 1024, + }, + CPKInfo: &testcommon.TestCPKByValue, + }) + _require.NoError(err) + _require.Equal(cnt, fileSize) + + destHash := md5.New() + _, err = io.Copy(destHash, destFile) + _require.NoError(err) + downloadedContentMD5 := destHash.Sum(nil) + + _require.EqualValues(downloadedContentMD5, contentMD5) + + gResp2, err := fClient.GetProperties(context.Background(), &file.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.Equal(*gResp2.ContentLength, fileSize) + _require.Equal(*(gResp2.IsServerEncrypted), true) + _require.Equal(gResp2.EncryptionKeySHA256, testcommon.TestCPKByValue.EncryptionKeySHA256) +} + func (s *RecordedTestSuite) TestFileUploadDownloadWithProgress() { _require := require.New(s.T()) testName := s.T().Name() From 0692497f849618f045e3a8162b641da09306ccbe Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Wed, 20 Dec 2023 10:14:21 +0530 Subject: [PATCH 5/8] recordings --- sdk/storage/azdatalake/assets.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/storage/azdatalake/assets.json b/sdk/storage/azdatalake/assets.json index a4aaa09d1502..c5e203f6a8c2 100644 --- a/sdk/storage/azdatalake/assets.json +++ b/sdk/storage/azdatalake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "go", "TagPrefix": "go/storage/azdatalake", - "Tag": "go/storage/azdatalake_5debe7fb7b" + "Tag": "go/storage/azdatalake_08c1840c4f" } \ No newline at end of file From 53fb694bea4d682dbf04d67ea56068f93ca35e69 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Wed, 20 Dec 2023 10:50:23 +0530 Subject: [PATCH 6/8] recordings --- sdk/storage/azdatalake/assets.json | 2 +- sdk/storage/azdatalake/file/client_test.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/sdk/storage/azdatalake/assets.json b/sdk/storage/azdatalake/assets.json index c5e203f6a8c2..933e3434bc5e 100644 --- a/sdk/storage/azdatalake/assets.json +++ b/sdk/storage/azdatalake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "go", "TagPrefix": "go/storage/azdatalake", - "Tag": "go/storage/azdatalake_08c1840c4f" + "Tag": "go/storage/azdatalake_bc642dba20" } \ No newline at end of file diff --git a/sdk/storage/azdatalake/file/client_test.go b/sdk/storage/azdatalake/file/client_test.go index cebc7717b15c..c38e7041b4fe 100644 --- a/sdk/storage/azdatalake/file/client_test.go +++ b/sdk/storage/azdatalake/file/client_test.go @@ -2733,8 +2733,7 @@ func (s *RecordedTestSuite) TestFileUploadDownloadStreamWithCPK() { contentMD5 := md5Value[:] err = fClient.UploadStream(context.Background(), streaming.NopCloser(bytes.NewReader(content)), &file.UploadStreamOptions{ - Concurrency: 5, - CPKInfo: &testcommon.TestCPKByValue, + CPKInfo: &testcommon.TestCPKByValue, }) _require.NoError(err) From 307c78b8866785319b7724e461c68068801e8531 Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Wed, 20 Dec 2023 12:57:00 +0530 Subject: [PATCH 7/8] change to unrecorded test --- sdk/storage/azdatalake/assets.json | 2 +- sdk/storage/azdatalake/file/client_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/storage/azdatalake/assets.json b/sdk/storage/azdatalake/assets.json index 933e3434bc5e..648392673fcd 100644 --- a/sdk/storage/azdatalake/assets.json +++ b/sdk/storage/azdatalake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "go", "TagPrefix": "go/storage/azdatalake", - "Tag": "go/storage/azdatalake_bc642dba20" + "Tag": "go/storage/azdatalake_b9c315a9bf" } \ No newline at end of file diff --git a/sdk/storage/azdatalake/file/client_test.go b/sdk/storage/azdatalake/file/client_test.go index c38e7041b4fe..94088978ba32 100644 --- a/sdk/storage/azdatalake/file/client_test.go +++ b/sdk/storage/azdatalake/file/client_test.go @@ -2705,7 +2705,7 @@ func (s *RecordedTestSuite) TestFileUploadTinyStream() { _require.EqualValues(downloadedContentMD5, contentMD5) } -func (s *RecordedTestSuite) TestFileUploadDownloadStreamWithCPK() { +func (s *UnrecordedTestSuite) TestFileUploadDownloadStreamWithCPK() { _require := require.New(s.T()) testName := s.T().Name() From fb3675d301415abbf379ad89527cecc62dee06be Mon Sep 17 00:00:00 2001 From: tanyasethi-msft Date: Wed, 20 Dec 2023 16:06:24 +0530 Subject: [PATCH 8/8] add negative tests for CPK --- sdk/storage/azdatalake/assets.json | 2 +- .../azdatalake/directory/client_test.go | 30 ++++++++++++++ sdk/storage/azdatalake/file/client_test.go | 39 +++++++++++++++++++ 3 files changed, 70 insertions(+), 1 deletion(-) diff --git a/sdk/storage/azdatalake/assets.json b/sdk/storage/azdatalake/assets.json index 648392673fcd..a3472b483893 100644 --- a/sdk/storage/azdatalake/assets.json +++ b/sdk/storage/azdatalake/assets.json @@ -2,5 +2,5 @@ "AssetsRepo": "Azure/azure-sdk-assets", "AssetsRepoPrefixPath": "go", "TagPrefix": "go/storage/azdatalake", - "Tag": "go/storage/azdatalake_b9c315a9bf" + "Tag": "go/storage/azdatalake_36b1978c0d" } \ No newline at end of file diff --git a/sdk/storage/azdatalake/directory/client_test.go b/sdk/storage/azdatalake/directory/client_test.go index 53ef88fd0349..9a43fa362733 100644 --- a/sdk/storage/azdatalake/directory/client_test.go +++ b/sdk/storage/azdatalake/directory/client_test.go @@ -1966,6 +1966,36 @@ func (s *RecordedTestSuite) TestDirSetMetadataWithCPK() { _require.NoError(err) } +func (s *RecordedTestSuite) TestDirSetMetadataWithCPKNegative() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + dirName := testcommon.GenerateDirName(testName) + dirClient, err := testcommon.GetDirClient(filesystemName, dirName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + defer testcommon.DeleteDir(context.Background(), _require, dirClient) + + resp, err := dirClient.Create(context.Background(), nil) + _require.NoError(err) + _require.NotNil(resp) + + opts := &directory.SetMetadataOptions{ + CPKInfo: &testcommon.TestCPKByValue, + } + _, err = dirClient.SetMetadata(context.Background(), testcommon.BasicMetadata, opts) + _require.Error(err) + _require.ErrorContains(err, "PathDoesNotUseCustomerSpecifiedEncryption") +} + func validatePropertiesSet(_require *require.Assertions, dirClient *directory.Client, disposition string) { resp, err := dirClient.GetProperties(context.Background(), nil) _require.NoError(err) diff --git a/sdk/storage/azdatalake/file/client_test.go b/sdk/storage/azdatalake/file/client_test.go index 94088978ba32..33794a13a3ba 100644 --- a/sdk/storage/azdatalake/file/client_test.go +++ b/sdk/storage/azdatalake/file/client_test.go @@ -2755,6 +2755,45 @@ func (s *UnrecordedTestSuite) TestFileUploadDownloadStreamWithCPK() { _require.Equal(testcommon.TestCPKByValue.EncryptionKeySHA256, dResp.EncryptionKeySHA256) } +func (s *UnrecordedTestSuite) TestFileUploadDownloadStreamWithCPKNegative() { + _require := require.New(s.T()) + testName := s.T().Name() + + filesystemName := testcommon.GenerateFileSystemName(testName) + fsClient, err := testcommon.GetFileSystemClient(filesystemName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + defer testcommon.DeleteFileSystem(context.Background(), _require, fsClient) + + _, err = fsClient.Create(context.Background(), nil) + _require.NoError(err) + + var fileSize int64 = 1 * 1024 * 1024 + fileName := testcommon.GenerateFileName(testName) + fClient, err := testcommon.GetFileClient(filesystemName, fileName, s.T(), testcommon.TestAccountDatalake, nil) + _require.NoError(err) + + resp, err := fClient.Create(context.Background(), &file.CreateOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.NotNil(resp) + + content := make([]byte, fileSize) + _, err = rand.Read(content) + _require.NoError(err) + + err = fClient.UploadStream(context.Background(), streaming.NopCloser(bytes.NewReader(content)), &file.UploadStreamOptions{ + CPKInfo: &testcommon.TestCPKByValue, + }) + _require.NoError(err) + + gResp2, err := fClient.GetProperties(context.Background(), &file.GetPropertiesOptions{CPKInfo: &testcommon.TestCPKByValue}) + _require.NoError(err) + _require.Equal(*gResp2.ContentLength, fileSize) + + _, err = fClient.DownloadStream(context.Background(), &file.DownloadStreamOptions{}) + _require.Error(err) + _require.ErrorContains(err, "PathUsesCustomerSpecifiedEncryption") +} + func (s *UnrecordedTestSuite) TestFileUploadFile() { _require := require.New(s.T()) testName := s.T().Name()