{storage-preview} Add hns soft delete (#2877)

* draft for hns

* fix style

* import pass

* show work

* draft update

* use stable blob SDK

* support list deleted and undelete

* add readme

* update sdk

* remove personal info

* fxi typo

* test pass

* remove validator

* update SDK

* update SDK

* resolve SDKissue

* resolve SDKissue

* add more argument for list

* test pass with new SDK

* test pass

* update help and release note

* fiix linter and test

* refine code

* Apply suggestions from code review

Co-authored-by: Yishi Wang <yishiwang@microsoft.com>

Co-authored-by: Yishi Wang <yishiwang@microsoft.com>

src/storage-preview/HISTORY.rst

@@ -2,6 +2,10 @@
 
 Release History
 ===============
+0.7.3(2021-05-20)
+++++++++++++++++++
+* Support soft delete for ADLS Gen2 account
+
 0.7.2(2021-04-09)
 ++++++++++++++++++
 * Remove `az storage blob service-properties` as it is supported in storage-blob-preview extension and Azure CLI

src/storage-preview/README.md

@@ -306,4 +306,69 @@ az storage account file-service-properties update \
     -g MyResourceGroup
 ```
 
+#### Soft Delete for ADLS Gen2 storage
+##### Prepare resource
+1. ADLS Gen2 storage account with soft delete support
+```
+az storage account create \
+    -n myadls \
+    -g myresourcegroup \
+    --hns
+``` 
+To get connection string, you could use the following command:
+```
+az storage account show-connection-string \
+    -n myadls \
+    -g myresourcegroup
+``` 
+2. Prepare file system in the ADLS Gen2 storage account
+```
+az storage fs create \
+    -n myfilesystem \
+    --connection-string myconnectionstring 
+```
+##### Enable delete retention
+```
+az storage fs service-properties update \
+    --delete-retention \
+    --delete-retention-period 5 \
+    --connection-string myconnectionstring
+```
+##### Upload file to file system
+```
+az storage fs file upload \
+    -s ".\test.txt" \
+    -p test \
+    -f filesystemcetk2triyptlaa \
+    --connection-string $con
+```
+##### List deleted path
+```
+az storage fs file delete \
+    -p test \
+    -f filesystemcetk2triyptlaa \ 
+    --connection-string $con
+```
+##### List deleted path
+```
+az storage fs list-deleted-path \
+    -f filesystemcetk2triyptlaa \
+    --connection-string $con
+```
+##### Undelete deleted path
+```
+az storage fs undelete-path \
+    -f filesystemcetk2triyptlaa \
+    -f filesystemcetk2triyptlaa \
+    --deleted-path-name test \
+    --deleted-path-version 132549163 \
+    --connection-string $con
+```
+##### Disable delete retention
+```
+az storage fs service-properties update \
+    --delete-retention false \
+    --connection-string $con
+```
+
 If you have issues, please give feedback by opening an issue at https://github.com/Azure/azure-cli-extensions/issues.

src/storage-preview/azext_storage_preview/__init__.py

@@ -8,8 +8,8 @@
 from azure.cli.core.commands import AzCommandGroup, AzArgumentContext
 
 import azext_storage_preview._help  # pylint: disable=unused-import
-from .profiles import (CUSTOM_DATA_STORAGE, CUSTOM_MGMT_PREVIEW_STORAGE, CUSTOM_DATA_STORAGE_ADLS,
-                       CUSTOM_DATA_STORAGE_FILESHARE)
+from .profiles import CUSTOM_DATA_STORAGE, CUSTOM_MGMT_PREVIEW_STORAGE, CUSTOM_DATA_STORAGE_ADLS, \
+    CUSTOM_DATA_STORAGE_FILESHARE, CUSTOM_DATA_STORAGE_FILEDATALAKE
 
 
 class StorageCommandsLoader(AzCommandsLoader):
@@ -20,6 +20,8 @@ def __init__(self, cli_ctx=None):
         register_resource_type('latest', CUSTOM_DATA_STORAGE_ADLS, '2019-02-02-preview')
         register_resource_type('latest', CUSTOM_MGMT_PREVIEW_STORAGE, '2020-08-01-preview')
         register_resource_type('latest', CUSTOM_DATA_STORAGE_FILESHARE, '2020-02-10')
+        register_resource_type('latest', CUSTOM_DATA_STORAGE_FILEDATALAKE, '2020-06-12')
+
         storage_custom = CliCommandType(operations_tmpl='azext_storage_preview.custom#{}')
 
         super(StorageCommandsLoader, self).__init__(cli_ctx=cli_ctx,
@@ -63,8 +65,9 @@ def register_content_settings_argument(self, settings_class, update, arg_group=N
 
         self.ignore('content_settings')
 
-        # The parameter process_md5 is used to determine whether it is compatible with the process_md5 parameter type of Python SDK
-        # When the Python SDK is fixed (Issue: https://github.com/Azure/azure-sdk-for-python/issues/15919),
+        # The parameter process_md5 is used to determine whether it is compatible with the process_md5 parameter
+        # type of Python SDK When the Python SDK is fixed
+        # (Issue: https://github.com/Azure/azure-sdk-for-python/issues/15919),
         # this parameter should not be passed in any more
         self.extra('content_type', default=None, help='The content MIME type.', arg_group=arg_group,
                    validator=get_content_setting_validator(settings_class, update, guess_from_file=guess_from_file,

src/storage-preview/azext_storage_preview/_client_factory.py

@@ -8,7 +8,9 @@
 from knack.util import CLIError
 from knack.log import get_logger
 
-from .profiles import CUSTOM_DATA_STORAGE, CUSTOM_MGMT_PREVIEW_STORAGE, CUSTOM_DATA_STORAGE_FILESHARE
+
+from .profiles import CUSTOM_DATA_STORAGE, CUSTOM_MGMT_PREVIEW_STORAGE, CUSTOM_DATA_STORAGE_FILESHARE, \
+    CUSTOM_DATA_STORAGE_FILEDATALAKE
 
 MISSING_CREDENTIALS_ERROR_MESSAGE = """
 Missing credentials to access storage service. The following variations are accepted:
@@ -136,7 +138,6 @@ def cf_mgmt_file_services(cli_ctx, _):
 
 
 def get_account_url(cli_ctx, account_name, service):
-    from knack.util import CLIError
     if account_name is None:
         raise CLIError("Please provide storage account name or connection string.")
     storage_endpoint = cli_ctx.cloud.suffixes.storage_endpoint
@@ -172,3 +173,29 @@ def cf_share_directory_client(cli_ctx, kwargs):
 
 def cf_share_file_client(cli_ctx, kwargs):
     return cf_share_client(cli_ctx, kwargs).get_file_client(file_path=kwargs.pop('file_path'))
+
+
+def cf_adls_service(cli_ctx, kwargs):
+    client_kwargs = {}
+    t_adls_service = get_sdk(cli_ctx, CUSTOM_DATA_STORAGE_FILEDATALAKE,
+                             '_data_lake_service_client#DataLakeServiceClient')
+    connection_string = kwargs.pop('connection_string', None)
+    account_name = kwargs.pop('account_name', None)
+    account_key = kwargs.pop('account_key', None)
+    token_credential = kwargs.pop('token_credential', None)
+    sas_token = kwargs.pop('sas_token', None)
+    # Enable NetworkTraceLoggingPolicy which logs all headers (except Authorization) without being redacted
+    client_kwargs['logging_enable'] = True
+    if connection_string:
+        return t_adls_service.from_connection_string(conn_str=connection_string, **client_kwargs)
+
+    account_url = get_account_url(cli_ctx, account_name=account_name, service='dfs')
+    credential = account_key or sas_token or token_credential
+
+    if account_url and credential:
+        return t_adls_service(account_url=account_url, credential=credential, **client_kwargs)
+    return None
+
+
+def cf_adls_file_system(cli_ctx, kwargs):
+    return cf_adls_service(cli_ctx, kwargs).get_file_system_client(file_system=kwargs.pop('file_system_name'))

src/storage-preview/azext_storage_preview/_help.py

@@ -394,3 +394,45 @@
         - name: Upload a set of files in a local directory to a storage blob directory.
           text: az storage blob directory upload -c MyContainer --account-name MyStorageAccount -s "path/to/file*" -d directory --recursive
 """
+
+helps['storage fs list-deleted-path'] = """
+type: command
+short-summary: List the deleted (file or directory) paths under the specified file system.
+examples:
+  - name: List the deleted (file or directory) paths under the specified file system..
+    text: |
+        az storage fs list-deleted-path -f myfilesystem --account-name mystorageccount --account-key 00000000
+"""
+
+helps['storage fs service-properties'] = """
+type: group
+short-summary: Manage storage datalake service properties.
+"""
+
+helps['storage fs service-properties show'] = """
+type: command
+short-summary: Show the properties of a storage account's datalake service, including Azure Storage Analytics.
+examples:
+  - name: Show the properties of a storage account's datalake service
+    text: |
+        az storage fs service-properties show --account-name mystorageccount --account-key 00000000
+"""
+
+helps['storage fs service-properties update'] = """
+type: command
+short-summary: Update the properties of a storage account's datalake service, including Azure Storage Analytics.
+examples:
+  - name: Update the properties of a storage account's datalake service
+    text: |
+        az storage fs service-properties update --delete-retention --delete-retention-period 7 --account-name mystorageccount --account-key 00000000
+"""
+
+helps['storage fs undelete-path'] = """
+type: command
+short-summary: Restore soft-deleted path.
+long-summary: Operation will only be successful if used within the specified number of days set in the delete retention policy.
+examples:
+  - name: Restore soft-deleted path.
+    text: |
+        az storage fs undelete-path -f myfilesystem --deleted-path-name dir --deletion-id 0000 --account-name mystorageccount --account-key 00000000
+"""

src/storage-preview/azext_storage_preview/_params.py

@@ -13,7 +13,7 @@
                           validate_storage_data_plane_list,
                           process_resource_group, add_upload_progress_callback)
 
-from .profiles import CUSTOM_MGMT_PREVIEW_STORAGE
+from .profiles import CUSTOM_MGMT_PREVIEW_STORAGE, CUSTOM_DATA_STORAGE_FILEDATALAKE
 
 
 def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statements
@@ -54,6 +54,9 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
                                     'e.g."user::rwx,user:john.doe@contoso:rwx,group::r--,other::---,mask::rwx".')
     progress_type = CLIArgumentType(help='Include this flag to disable progress reporting for the command.',
                                     action='store_true', validator=add_upload_progress_callback)
+    timeout_type = CLIArgumentType(
+        help='Request timeout in seconds. Applies to each call to the service.', type=int
+    )
 
     with self.argument_context('storage') as c:
         c.argument('container_name', container_name_type)
@@ -157,15 +160,15 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
 
     with self.argument_context('storage blob service-properties update') as c:
         c.argument('delete_retention', arg_type=get_three_state_flag(), arg_group='Soft Delete',
-                   help='Enables soft-delete.')
+                   help='Enable soft-delete.')
         c.argument('days_retained', type=int, arg_group='Soft Delete',
                    help='Number of days that soft-deleted blob will be retained. Must be in range [1,365].')
         c.argument('static_website', arg_group='Static Website', arg_type=get_three_state_flag(),
-                   help='Enables static-website.')
+                   help='Enable static-website.')
         c.argument('index_document', help='Represents the name of the index document. This is commonly "index.html".',
                    arg_group='Static Website')
         c.argument('error_document_404_path', options_list=['--404-document'], arg_group='Static Website',
-                   help='Represents the path to the error document that should be shown when an error 404 is issued,'
+                   help='Represent the path to the error document that should be shown when an error 404 is issued,'
                         ' in other words, when a browser requests a page that does not exist.')
 
     with self.argument_context('storage azcopy blob upload') as c:
@@ -374,3 +377,34 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
         c.register_content_settings_argument(t_file_content_settings, update=False, arg_group='Content Settings',
                                              process_md5=True)
         c.extra('no_progress', progress_type)
+
+    with self.argument_context('storage fs service-properties update', resource_type=CUSTOM_DATA_STORAGE_FILEDATALAKE,
+                               min_api='2020-06-12') as c:
+        c.argument('delete_retention', arg_type=get_three_state_flag(), arg_group='Soft Delete',
+                   help='Enable soft-delete.')
+        c.argument('delete_retention_period', type=int, arg_group='Soft Delete',
+                   options_list=['--delete-retention-period', '--period'],
+                   help='Number of days that soft-deleted fs will be retained. Must be in range [1,365].')
+        c.argument('enable_static_website', options_list=['--static-website'], arg_group='Static Website',
+                   arg_type=get_three_state_flag(),
+                   help='Enable static-website.')
+        c.argument('index_document', help='Represent the name of the index document. This is commonly "index.html".',
+                   arg_group='Static Website')
+        c.argument('error_document_404_path', options_list=['--404-document'], arg_group='Static Website',
+                   help='Represent the path to the error document that should be shown when an error 404 is issued,'
+                        ' in other words, when a browser requests a page that does not exist.')
+
+    for item in ['list-deleted-path', 'undelete-path']:
+        with self.argument_context('storage fs {}'.format(item)) as c:
+            c.extra('file_system_name', options_list=['--file-system', '-f'],
+                    help="File system name.", required=True)
+            c.extra('timeout', timeout_type)
+
+    with self.argument_context('storage fs list-deleted-path') as c:
+        c.argument('path_prefix', help='Filter the results to return only paths under the specified path.')
+        c.argument('num_results', type=int, help='Specify the maximum number to return.')
+        c.argument('marker', help='A string value that identifies the portion of the list of containers to be '
+                   'returned with the next listing operation. The operation returns the NextMarker value within '
+                   'the response body if the listing operation did not return all containers remaining to be listed '
+                   'with the current page. If specified, this generator will begin returning results from the point '
+                   'where the previous generator stopped.')

src/storage-preview/azext_storage_preview/_transformers.py

@@ -122,5 +122,6 @@ def transform_storage_list_output(result):
     return list(result)
 
 
+# pylint: disable=unused-argument
 def transform_file_upload(result):
     return None

src/storage-preview/azext_storage_preview/_validators.py

@@ -76,59 +76,85 @@ def validate_bypass(namespace):
         namespace.bypass = ', '.join(namespace.bypass) if isinstance(namespace.bypass, list) else namespace.bypass
 
 
+def get_config_value(cmd, section, key, default):
+    return cmd.cli_ctx.config.get(section, key, default)
+
+
+def is_storagev2(import_prefix):
+    return import_prefix.startswith('azure.multiapi.storagev2.') or 'datalake' in import_prefix
+
+
 def validate_client_parameters(cmd, namespace):
     """ Retrieves storage connection parameters from environment variables and parses out connection string into
     account name and key """
     n = namespace
 
-    def get_config_value(section, key, default):
-        return cmd.cli_ctx.config.get(section, key, default)
-
     if hasattr(n, 'auth_mode'):
-        auth_mode = n.auth_mode or get_config_value('storage', 'auth_mode', None)
+        auth_mode = n.auth_mode or get_config_value(cmd, 'storage', 'auth_mode', None)
         del n.auth_mode
         if not n.account_name:
-            n.account_name = get_config_value('storage', 'account', None)
+            n.account_name = get_config_value(cmd, 'storage', 'account', None)
         if auth_mode == 'login':
-            n.token_credential = _create_token_credential(cmd.cli_ctx)
-
-            # give warning if there are account key args being ignored
-            account_key_args = [n.account_key and "--account-key", n.sas_token and "--sas-token",
-                                n.connection_string and "--connection-string"]
-            account_key_args = [arg for arg in account_key_args if arg]
-
-            if account_key_args:
-                logger.warning('In "login" auth mode, the following arguments are ignored: %s',
-                               ' ,'.join(account_key_args))
-            return
+            prefix = cmd.command_kwargs['resource_type'].import_prefix
+            # is_storagv2() is used to distinguish if the command is in track2 SDK
+            # If yes, we will use get_login_credentials() as token credential
+            if is_storagev2(prefix):
+                from azure.cli.core._profile import Profile
+                profile = Profile(cli_ctx=cmd.cli_ctx)
+                n.token_credential, _, _ = profile.get_login_credentials(subscription_id=n._subscription)
+            # Otherwise, we will assume it is in track1 and keep previous token updater
+            else:
+                n.token_credential = _create_token_credential(cmd.cli_ctx)
+
+    if hasattr(n, 'token_credential') and n.token_credential:
+        # give warning if there are account key args being ignored
+        account_key_args = [n.account_key and "--account-key", n.sas_token and "--sas-token",
+                            n.connection_string and "--connection-string"]
+        account_key_args = [arg for arg in account_key_args if arg]
+
+        if account_key_args:
+            logger.warning('In "login" auth mode, the following arguments are ignored: %s',
+                           ' ,'.join(account_key_args))
+        return
 
     if not n.connection_string:
-        n.connection_string = get_config_value('storage', 'connection_string', None)
+        n.connection_string = get_config_value(cmd, 'storage', 'connection_string', None)
 
     # if connection string supplied or in environment variables, extract account key and name
     if n.connection_string:
         conn_dict = validate_key_value_pairs(n.connection_string)
         n.account_name = conn_dict.get('AccountName')
         n.account_key = conn_dict.get('AccountKey')
-        if not n.account_name or not n.account_key:
-            raise CLIError('Connection-string: %s, is malformed. Some shell environments require the '
-                           'connection string to be surrounded by quotes.' % n.connection_string)
+        n.sas_token = conn_dict.get('SharedAccessSignature')
 
     # otherwise, simply try to retrieve the remaining variables from environment variables
     if not n.account_name:
-        n.account_name = get_config_value('storage', 'account', None)
+        n.account_name = get_config_value(cmd, 'storage', 'account', None)
     if not n.account_key:
-        n.account_key = get_config_value('storage', 'key', None)
+        n.account_key = get_config_value(cmd, 'storage', 'key', None)
     if not n.sas_token:
-        n.sas_token = get_config_value('storage', 'sas_token', None)
+        n.sas_token = get_config_value(cmd, 'storage', 'sas_token', None)
 
     # strip the '?' from sas token. the portal and command line are returns sas token in different
     # forms
     if n.sas_token:
         n.sas_token = n.sas_token.lstrip('?')
 
+    # account name with secondary
+    if n.account_name and n.account_name.endswith('-secondary'):
+        n.location_mode = 'secondary'
+        n.account_name = n.account_name[:-10]
+
     # if account name is specified but no key, attempt to query
     if n.account_name and not n.account_key and not n.sas_token:
+        logger.warning('There are no credentials provided in your command and environment, we will query for the '
+                       'account key inside your storage account. \nPlease provide --connection-string, '
+                       '--account-key or --sas-token as credentials, or use `--auth-mode login` if you '
+                       'have required RBAC roles in your command. For more information about RBAC roles '
+                       'in storage, visit '
+                       'https://docs.microsoft.com/en-us/azure/storage/common/storage-auth-aad-rbac-cli. \n'
+                       'Setting the corresponding environment variables can avoid inputting credentials in '
+                       'your command. Please use --help to get more information.')
         n.account_key = _query_account_key(cmd.cli_ctx, n.account_name)