Static web and hierarchical namespace features. (#218)

* updated sdk

* service-properties update

* initial generic update for blob service-properties

* implemented generic update for service props

* added testing and recording update

* fixed bug

* added test

* recording

* updated mgmt sdk for storage

* added hierarchical namespace feature to account create

* version bump

* ci

* pylint and re-recorded all tests

* updated all recordings and tests for new sdks

src/storage-preview/azext_storage_preview/__init__.py

@@ -15,7 +15,7 @@ class StorageCommandsLoader(AzCommandsLoader):
     def __init__(self, cli_ctx=None):
         from azure.cli.core.commands import CliCommandType
 
-        register_resource_type('latest', CUSTOM_DATA_STORAGE, '2017-11-09')
+        register_resource_type('latest', CUSTOM_DATA_STORAGE, '2018-03-28')
         register_resource_type('latest', CUSTOM_MGMT_STORAGE, '2018-03-01-preview')
         storage_custom = CliCommandType(operations_tmpl='azext_storage_preview.custom#{}')
 
@@ -132,10 +132,13 @@ def register_common_storage_account_options(self):
 
 
 class StorageCommandGroup(AzCommandGroup):
-    def storage_command(self, name, method_name=None, command_type=None, oauth=False, **kwargs):
+    def storage_command(self, name, method_name=None, command_type=None, oauth=False, generic_update=None, **kwargs):
         """ Registers an Azure CLI Storage Data Plane command. These commands always include the four parameters which
         can be used to obtain a storage client: account-name, account-key, connection-string, and sas-token. """
-        if command_type:
+        if generic_update:
+            command_name = '{} {}'.format(self.group_name, name) if self.group_name else name
+            self.generic_update_command(name, **kwargs)
+        elif command_type:
             command_name = self.command(name, method_name, command_type=command_type, **kwargs)
         else:
             command_name = self.command(name, method_name, **kwargs)
@@ -241,16 +244,16 @@ def _register_data_plane_oauth_arguments(self, command_name):
 
 
 def _merge_new_exception_handler(kwargs, handler):
-    if kwargs.get('exception_handler'):
-        def new_handler(ex):
-            first = kwargs['exception_handler']
-            try:
-                first(ex)
-            except Exception as raised_ex:  # pylint: disable=broad-except
-                handler(raised_ex)
-        kwargs['exception_handler'] = new_handler
-    else:
-        kwargs['exception_handler'] = handler
+    first = kwargs.get('exception_handler')
+
+    def new_handler(ex):
+        try:
+            handler(ex)
+        except Exception:  # pylint: disable=broad-except
+            if not first:
+                raise
+            first(ex)
+    kwargs['exception_handler'] = new_handler
 
 
 COMMAND_LOADER_CLS = StorageCommandsLoader

src/storage-preview/azext_storage_preview/_client_factory.py

@@ -168,3 +168,7 @@ def cf_sa(cli_ctx, _):
 
 def cf_blob_container_mgmt(cli_ctx, _):
     return storage_client_factory(cli_ctx).blob_containers
+
+
+def cf_blob_data_gen_update(cli_ctx, kwargs):
+    return blob_data_service_factory(cli_ctx, kwargs.copy())

src/storage-preview/azext_storage_preview/_help.py

@@ -224,6 +224,11 @@
     short-summary: Manage storage blob service properties.
 """
 
+helps['storage blob service-properties update'] = """
+    type: command
+    short-summary: Update storage blob service properties.
+"""
+
 helps['storage blob service-properties delete-policy'] = """
     type: group
     short-summary: Manage storage blob delete-policy service properties.

src/storage-preview/azext_storage_preview/_params.py

@@ -4,7 +4,8 @@
 # --------------------------------------------------------------------------------------------
 
 from azure.cli.core.commands.validators import get_default_location_from_resource_group
-from azure.cli.core.commands.parameters import (tags_type, file_type, get_location_type, get_enum_type)
+from azure.cli.core.commands.parameters import (tags_type, file_type, get_location_type, get_enum_type,
+                                                get_three_state_flag)
 
 from ._validators import (get_datetime_type, validate_metadata, get_permission_validator, get_permission_help_string,
                           resource_type_type, services_type, validate_entity, validate_select, validate_blob_type,
@@ -94,6 +95,8 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
         t_account_type, t_sku_name, t_kind = self.get_models('AccountType', 'SkuName', 'Kind',
                                                              resource_type=CUSTOM_MGMT_STORAGE)
         c.register_common_storage_account_options()
+        c.argument('hierarchical_namespace', help='Allows the blob service to exhibit filesystem semantics.',
+                   arg_type=get_three_state_flag())
         c.argument('location', get_location_type(self.cli_ctx), validator=get_default_location_from_resource_group)
         c.argument('account_type', help='The storage account type', arg_type=get_enum_type(t_account_type))
         c.argument('account_name', acct_name_type, options_list=['--name', '-n'], completer=None)
@@ -242,6 +245,19 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
         c.argument('days_retained', type=int,
                    help='Number of days that soft-deleted blob will be retained. Must be in range [1,365].')
 
+    with self.argument_context('storage blob service-properties update') as c:
+        c.argument('delete_retention', arg_type=get_three_state_flag(), arg_group='Soft Delete',
+                   help='Enables soft-delete.')
+        c.argument('days_retained', type=int, arg_group='Soft Delete',
+                   help='Number of days that soft-deleted blob will be retained. Must be in range [1,365].')
+        c.argument('static_website', arg_group='Static Website', arg_type=get_three_state_flag(),
+                   help='Enables static-website.')
+        c.argument('index_document', help='Represents the name of the index document. This is commonly "index.html".',
+                   arg_group='Static Website')
+        c.argument('error_document_404_path', options_list=['--404-document'], arg_group='Static Website',
+                   help='Represents the path to the error document that should be shown when an error 404 is issued,'
+                        ' in other words, when a browser requests a page that does not exist.')
+
     with self.argument_context('storage blob upload') as c:
         from ._validators import page_blob_tier_validator
         from .sdkutil import get_blob_types, get_blob_tier_names

src/storage-preview/azext_storage_preview/_validators.py

@@ -50,7 +50,7 @@ def _create_token_credential(cli_ctx):
 
     TokenCredential = get_sdk(cli_ctx, CUSTOM_DATA_STORAGE, 'common#TokenCredential')
 
-    token_credential = TokenCredential(None)
+    token_credential = TokenCredential()
     updater = TokenUpdater(token_credential, cli_ctx)
 
     def _cancel_timer_event_handler(_, **__):

src/storage-preview/azext_storage_preview/commands.py

@@ -5,7 +5,7 @@
 
 from azure.cli.core.commands import CliCommandType
 from azure.cli.core.profiles import ResourceType
-from ._client_factory import (cf_sa, cf_blob_container_mgmt, blob_data_service_factory,
+from ._client_factory import (cf_sa, cf_blob_container_mgmt, cf_blob_data_gen_update, blob_data_service_factory,
                               page_blob_service_factory, file_data_service_factory,
                               queue_data_service_factory, table_data_service_factory,
                               cloud_storage_account_service_factory,
@@ -28,7 +28,7 @@ def load_command_table(self, _):  # pylint: disable=too-many-locals, too-many-st
         client_factory=cf_sa)
 
     cloud_data_plane_sdk = CliCommandType(
-        operations_tmpl='azure.multiapi.storage.common#CloudStorageAccount.{}',
+        operations_tmpl='azext_storage_preview.vendored_sdks.azure_storage.common#CloudStorageAccount.{}',
         client_factory=cloud_storage_account_service_factory
     )
 
@@ -88,12 +88,12 @@ def get_custom_sdk(custom_module, client_factory, resource_type=CUSTOM_DATA_STOR
                           exception_handler=g.get_handler_suppress_404())
 
     block_blob_sdk = CliCommandType(
-        operations_tmpl='azure.multiapi.storage.blob.blockblobservice#BlockBlobService.{}',
+        operations_tmpl='azext_storage_preview.vendored_sdks.azure_storage.blob.blockblobservice#BlockBlobService.{}',
         client_factory=blob_data_service_factory,
         resource_type=CUSTOM_DATA_STORAGE)
 
     base_blob_sdk = CliCommandType(
-        operations_tmpl='azure.multiapi.storage.blob.baseblobservice#BaseBlobService.{}',
+        operations_tmpl='azext_storage_preview.vendored_sdks.azure_storage.blob.baseblobservice#BaseBlobService.{}',
         client_factory=blob_data_service_factory,
         resource_type=CUSTOM_DATA_STORAGE)
 
@@ -140,14 +140,16 @@ def get_custom_sdk(custom_module, client_factory, resource_type=CUSTOM_DATA_STOR
         g.storage_custom_command_oauth('copy start-batch', 'storage_blob_copy_batch')
 
     with self.command_group('storage blob incremental-copy',
-                            operations_tmpl='azure.multiapi.storage.blob.pageblobservice#PageBlobService.{}',
+                            operations_tmpl='azext_storage_preview.vendored_sdks.azure_storage.blob.pageblobservice'
+                                            '#PageBlobService.{}',
                             client_factory=page_blob_service_factory,
                             resource_type=CUSTOM_DATA_STORAGE,
                             min_api='2016-05-31') as g:
         g.storage_command_oauth('start', 'incremental_copy_blob')
 
     with self.command_group('storage blob incremental-copy',
-                            operations_tmpl='azure.multiapi.storage.blob.blockblobservice#BlockBlobService.{}',
+                            operations_tmpl='azext_storage_preview.vendored_sdks.azure_storage.blob.blockblobservice'
+                                            '#BlockBlobService.{}',
                             client_factory=page_blob_service_factory,
                             resource_type=CUSTOM_DATA_STORAGE,
                             min_api='2016-05-31') as g:
@@ -163,6 +165,10 @@ def get_custom_sdk(custom_module, client_factory, resource_type=CUSTOM_DATA_STOR
 
     with self.command_group('storage blob service-properties', command_type=base_blob_sdk) as g:
         g.storage_command_oauth('show', 'get_blob_service_properties', exception_handler=g.get_handler_suppress_404())
+        g.storage_command_oauth('update', generic_update=True, getter_name='get_blob_service_properties',
+                                setter_type=get_custom_sdk('blob', cf_blob_data_gen_update),
+                                setter_name='set_service_properties',
+                                client_factory=cf_blob_data_gen_update)
 
     with self.command_group('storage container', command_type=block_blob_sdk,
                             custom_command_type=get_custom_sdk('acl', blob_data_service_factory)) as g:
@@ -221,7 +227,7 @@ def get_custom_sdk(custom_module, client_factory, resource_type=CUSTOM_DATA_STOR
         g.command('show', 'get', transform=lambda x: getattr(x, 'legal_hold', x))
 
     file_sdk = CliCommandType(
-        operations_tmpl='azure.multiapi.storage.file.fileservice#FileService.{}',
+        operations_tmpl='azext_storage_preview.vendored_sdks.azure_storage.file.fileservice#FileService.{}',
         client_factory=file_data_service_factory,
         resource_type=CUSTOM_DATA_STORAGE)
 
@@ -302,9 +308,10 @@ def get_custom_sdk(custom_module, client_factory, resource_type=CUSTOM_DATA_STOR
         g.storage_command('clear', 'clear_cors')
         g.storage_command('list', 'list_cors', transform=transform_cors_list_output)
 
-    queue_sdk = CliCommandType(operations_tmpl='azure.multiapi.storage.queue.queueservice#QueueService.{}',
-                               client_factory=queue_data_service_factory,
-                               resource_type=CUSTOM_DATA_STORAGE)
+    queue_sdk = CliCommandType(
+        operations_tmpl='azext_storage_preview.vendored_sdks.azure_storage.queue.queueservice#QueueService.{}',
+        client_factory=queue_data_service_factory,
+        resource_type=CUSTOM_DATA_STORAGE)
 
     with self.command_group('storage queue', queue_sdk,
                             custom_command_type=get_custom_sdk('acl', queue_data_service_factory)) as g:
@@ -343,13 +350,15 @@ def get_custom_sdk(custom_module, client_factory, resource_type=CUSTOM_DATA_STOR
         g.storage_command_oauth('update', 'update_message')
 
     if cosmosdb_table_exists(self.cli_ctx):
-        table_sdk = CliCommandType(operations_tmpl='azure.multiapi.cosmosdb.table.tableservice#TableService.{}',
-                                   client_factory=table_data_service_factory,
-                                   resource_type=ResourceType.DATA_COSMOS_TABLE)
+        table_sdk = CliCommandType(
+            operations_tmpl='azure.multiapi.cosmosdb.table.tableservice#TableService.{}',
+            client_factory=table_data_service_factory,
+            resource_type=ResourceType.DATA_COSMOS_TABLE)
     else:
-        table_sdk = CliCommandType(operations_tmpl='azure.multiapi.storage.table.tableservice#TableService.{}',
-                                   client_factory=table_data_service_factory,
-                                   resource_type=ResourceType.DATA_COSMOS_TABLE)
+        table_sdk = CliCommandType(
+            operations_tmpl='azext_storage_preview.vendored_sdks.azure_storage.table.tableservice#TableService.{}',
+            client_factory=table_data_service_factory,
+            resource_type=ResourceType.DATA_COSMOS_TABLE)
 
     with self.command_group('storage table', table_sdk,
                             custom_command_type=get_custom_sdk('acl', table_data_service_factory)) as g:

src/storage-preview/azext_storage_preview/oauth_token_util.py

@@ -26,7 +26,7 @@ def timer_callback(self):
         from datetime import datetime
         # should give back token that is valid for at least 5 mins
         token = Profile(cli_ctx=self.cli_ctx).get_raw_token(resource="https://storage.azure.com")[0][2]
-        self.token_credential.update_token(token['accessToken'])
+        self.token_credential.token = token['accessToken']
         seconds_left = (datetime.strptime(token['expiresOn'], "%Y-%m-%d %H:%M:%S.%f") - datetime.now()).seconds
         if seconds_left < 240:
             # acquired token expires in less than 4 mins

src/storage-preview/azext_storage_preview/operations/account.py

@@ -10,9 +10,10 @@
 from .._client_factory import storage_client_factory
 
 
+# pylint: disable=too-many-locals
 def create_storage_account(cmd, resource_group_name, account_name, sku=None, location=None, kind=None,
                            tags=None, custom_domain=None, encryption_services=None, access_tier=None, https_only=None,
-                           bypass=None, default_action=None, assign_identity=False):
+                           hierarchical_namespace=None, bypass=None, default_action=None, assign_identity=False):
     StorageAccountCreateParameters, Kind, Sku, CustomDomain, AccessTier, Identity, Encryption, NetworkRuleSet = \
         cmd.get_models('StorageAccountCreateParameters', 'Kind', 'Sku', 'CustomDomain', 'AccessTier', 'Identity',
                        'Encryption', 'NetworkRuleSet')
@@ -28,6 +29,8 @@ def create_storage_account(cmd, resource_group_name, account_name, sku=None, loc
         params.identity = Identity()
     if https_only:
         params.enable_https_traffic_only = https_only
+    if hierarchical_namespace:
+        params.is_hns_enabled = hierarchical_namespace
 
     if NetworkRuleSet and (bypass or default_action):
         if bypass and not default_action:

src/storage-preview/azext_storage_preview/operations/blob.py

@@ -45,6 +45,36 @@ def set_delete_policy(client, enable=None, days_retained=None):
     return client.get_blob_service_properties().delete_retention_policy
 
 
+def set_service_properties(client, parameters, delete_retention=None, days_retained=None, static_website=None,
+                           index_document=None, error_document_404_path=None):
+    # update
+    kwargs = {}
+    if any([delete_retention, days_retained]):
+        kwargs['delete_retention_policy'] = parameters.delete_retention_policy
+    if delete_retention is not None:
+        parameters.delete_retention_policy.enabled = delete_retention
+    if days_retained is not None:
+        parameters.delete_retention_policy.days = days_retained
+
+    if any([static_website, index_document, error_document_404_path]):
+        kwargs['static_website'] = parameters.static_website
+    if static_website is not None:
+        parameters.static_website.enabled = static_website
+    if index_document is not None:
+        parameters.static_website.index_document = index_document
+    if error_document_404_path is not None:
+        parameters.static_website.error_document_404_path = error_document_404_path
+
+    # checks
+    policy = parameters.delete_retention_policy
+    if policy.enabled and not policy.days:
+        from knack.util import CLIError
+        raise CLIError("must specify days-retained")
+
+    client.set_blob_service_properties(**kwargs)
+    return client.get_blob_service_properties()
+
+
 def storage_blob_copy_batch(cmd, client, source_client, destination_container=None,
                             destination_path=None, source_container=None, source_share=None,
                             source_sas=None, pattern=None, dryrun=False):

src/storage-preview/azext_storage_preview/tests/latest/__init__.py

@@ -5,4 +5,4 @@
 
 from azure.cli.core.profiles import register_resource_type
 from ...profiles import CUSTOM_MGMT_STORAGE
-register_resource_type('latest', CUSTOM_MGMT_STORAGE, '2018-02-01')
+register_resource_type('latest', CUSTOM_MGMT_STORAGE, '2018-03-01-preview')