Worm feature and listing usage by region (#181)

* vendored mgmt sdk

* sdk

* updated old recordings for new sdk

* added worm feature

enable listing usage by region

ci fixes

fixes

fixes

flake8 fixes

removed prints

* address feedback

* updated index with new extension path

* version bump in index

src/index.json

@@ -829,9 +829,9 @@
         ],
         "storage-preview": [
             {
-                "filename": "storage_preview-0.1.0-py2.py3-none-any.whl",
-                "sha256Digest": "36768962d09c65b9668581f5bc01f1ad252acf832f22afdb04ed00c3333379cc",
-                "downloadUrl": "https://azurecliprod.blob.core.windows.net/cli-extensions/storage_preview-0.1.0-py2.py3-none-any.whl",
+                "filename": "storage_preview-0.1.1-py2.py3-none-any.whl",
+                "sha256Digest": "4447f8b825559e2b94bbdcaa2a1af1723a2151fad670ff88162e8d2567e715cf",
+                "downloadUrl": "https://azurecliprod.blob.core.windows.net/cli-extensions/storage_preview-0.1.1-py2.py3-none-any.whl",
                 "metadata": {
                     "azext.isPreview": true,
                     "azext.minCliCoreVersion": "2.0.32.dev0",
@@ -870,7 +870,7 @@
                     "metadata_version": "2.0",
                     "name": "storage-preview",
                     "summary": "Provides a preview for upcoming storage features.",
-                    "version": "0.1.0"
+                    "version": "0.1.1"
                 }
             }
         ]

src/storage-preview/azext_storage_preview/__init__.py

@@ -4,21 +4,23 @@
 # --------------------------------------------------------------------------------------------
 
 from azure.cli.core import AzCommandsLoader
-from azure.cli.core.profiles import ResourceType, register_resource_type
+from azure.cli.core.profiles import register_resource_type
 from azure.cli.core.commands import AzCommandGroup, AzArgumentContext
 
 import azext_storage_preview._help  # pylint: disable=unused-import
-from .profiles import CUSTOM_DATA_STORAGE
+from .profiles import CUSTOM_DATA_STORAGE, CUSTOM_MGMT_STORAGE
 
 
 class StorageCommandsLoader(AzCommandsLoader):
     def __init__(self, cli_ctx=None):
         from azure.cli.core.commands import CliCommandType
 
         register_resource_type('latest', CUSTOM_DATA_STORAGE, '2017-11-09')
+        register_resource_type('latest', CUSTOM_MGMT_STORAGE, '2018-02-01')
         storage_custom = CliCommandType(operations_tmpl='azext_storage_preview.custom#{}')
 
         super(StorageCommandsLoader, self).__init__(cli_ctx=cli_ctx,
+                                                    min_profile='2017-03-10-profile',
                                                     resource_type=CUSTOM_DATA_STORAGE,
                                                     custom_command_type=storage_custom,
                                                     command_group_cls=StorageCommandGroup,
@@ -107,12 +109,12 @@ def register_common_storage_account_options(self):
         from ._validators import validate_encryption_services
 
         t_access_tier, t_sku_name, t_encryption_services = self.command_loader.get_models(
-            'AccessTier', 'SkuName', 'EncryptionServices', resource_type=ResourceType.MGMT_STORAGE)
+            'AccessTier', 'SkuName', 'EncryptionServices', resource_type=CUSTOM_MGMT_STORAGE)
 
         self.argument('https_only', help='Allows https traffic only to storage service.',
                       arg_type=get_three_state_flag())
         self.argument('sku', help='The storage account SKU.', arg_type=get_enum_type(t_sku_name))
-        self.argument('assign_identity', action='store_true', resource_type=ResourceType.MGMT_STORAGE,
+        self.argument('assign_identity', action='store_true', resource_type=CUSTOM_MGMT_STORAGE,
                       min_api='2017-06-01',
                       help='Generate and assign a new Storage Account Identity for this storage account for use '
                            'with key management services like Azure KeyVault.')
@@ -125,7 +127,7 @@ def register_common_storage_account_options(self):
             encryption_choices = list(
                 t_encryption_services._attribute_map.keys())  # pylint: disable=protected-access
             self.argument('encryption_services', arg_type=get_enum_type(encryption_choices),
-                          resource_type=ResourceType.MGMT_STORAGE, min_api='2016-12-01', nargs='+',
+                          resource_type=CUSTOM_MGMT_STORAGE, min_api='2016-12-01', nargs='+',
                           validator=validate_encryption_services, help='Specifies which service(s) to encrypt.')
 
 
@@ -196,14 +198,17 @@ def handler(ex):
 
     def _register_data_plane_account_arguments(self, command_name):
         """ Add parameters required to create a storage client """
+        from azure.cli.core.commands.parameters import get_resource_name_completion_list
         from ._validators import validate_client_parameters
         command = self.command_loader.command_table.get(command_name, None)
         if not command:
             return
 
         group_name = 'Storage Account'
+
         command.add_argument('account_name', '--account-name', required=False, default=None,
                              arg_group=group_name,
+                             completer=get_resource_name_completion_list('Microsoft.Storage/storageAccounts'),
                              help='Storage account name. Related environment variable: AZURE_STORAGE_ACCOUNT. Must be '
                                   'used in conjunction with either storage account key or a SAS token. If neither are '
                                   'present, the command will try to query the storage account key using the '

src/storage-preview/azext_storage_preview/_client_factory.py

@@ -4,12 +4,12 @@
 # --------------------------------------------------------------------------------------------
 
 from azure.cli.core.commands.client_factory import get_mgmt_service_client, _get_add_headers_callback
-from azure.cli.core.profiles import ResourceType, get_sdk
+from azure.cli.core.profiles import get_sdk
 from knack.util import CLIError
 from knack.log import get_logger
 
 from .sdkutil import get_table_data_type
-from .profiles import CUSTOM_DATA_STORAGE
+from .profiles import CUSTOM_DATA_STORAGE, CUSTOM_MGMT_STORAGE
 
 NO_CREDENTIALS_ERROR_MESSAGE = """
 No credentials specified to access storage service. Please provide any of the following:
@@ -75,7 +75,7 @@ def generic_data_service_factory(cli_ctx, service, name=None, key=None, connecti
 
 
 def storage_client_factory(cli_ctx, **_):
-    return get_mgmt_service_client(cli_ctx, ResourceType.MGMT_STORAGE)
+    return get_mgmt_service_client(cli_ctx, CUSTOM_MGMT_STORAGE)
 
 
 def file_data_service_factory(cli_ctx, kwargs):
@@ -164,3 +164,7 @@ def get_creator(name, service_type):
 
 def cf_sa(cli_ctx, _):
     return storage_client_factory(cli_ctx).storage_accounts
+
+
+def cf_blob_container_mgmt(cli_ctx, _):
+    return storage_client_factory(cli_ctx).blob_containers

src/storage-preview/azext_storage_preview/_format.py

@@ -148,3 +148,11 @@ def transformer(result):
 
         return return_list
     return transformer
+
+
+def transform_immutability_policy(result):
+    # service returns policy with period value of "0" after it has been deleted
+    # this only shows the policy if the property value is greater than 0
+    if result.immutability_period_since_creation_in_days:
+        return result
+    return None

src/storage-preview/azext_storage_preview/_help.py

@@ -403,6 +403,21 @@
     short-summary: Manage container stored access policies.
 """
 
+helps['storage container immutability-policy'] = """
+    type: group
+    short-summary: Manage container immutability policies.
+"""
+
+helps['storage container legal-hold'] = """
+    type: group
+    short-summary: Manage container legal holds.
+"""
+
+helps['storage container legal-hold show'] = """
+    type: command
+    short-summary: Get the legal hold properties of a container.
+"""
+
 helps['storage cors'] = """
     type: group
     short-summary: Manage storage service Cross-Origin Resource Sharing (CORS).

src/storage-preview/azext_storage_preview/_params.py

@@ -3,7 +3,6 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
-from azure.cli.core.profiles import ResourceType
 from azure.cli.core.commands.validators import get_default_location_from_resource_group
 from azure.cli.core.commands.parameters import (tags_type, file_type, get_location_type, get_enum_type)
 
@@ -13,6 +12,7 @@
                           validate_table_payload_format, validate_key, add_progress_callback,
                           storage_account_key_options, process_file_download_namespace, process_metric_update_namespace,
                           get_char_options_validator, validate_bypass, validate_encryption_source)
+from .profiles import CUSTOM_MGMT_STORAGE
 
 
 def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statements
@@ -24,7 +24,7 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
     from azure.cli.core.commands.parameters import get_resource_name_completion_list
 
     from .sdkutil import get_table_data_type
-    from .completers import get_storage_name_completion_list
+    from .completers import get_storage_name_completion_list, get_container_name_completions
 
     t_base_blob_service = self.get_sdk('blob.baseblobservice#BaseBlobService')
     t_file_service = self.get_sdk('file#FileService')
@@ -39,8 +39,7 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
                                                                                 parent='container_name'))
 
     container_name_type = CLIArgumentType(options_list=['--container-name', '-c'], help='The container name.',
-                                          completer=get_storage_name_completion_list(t_base_blob_service,
-                                                                                     'list_containers'))
+                                          completer=get_container_name_completions)
     directory_type = CLIArgumentType(options_list=['--directory-name', '-d'], help='The directory name.',
                                      completer=get_storage_name_completion_list(t_file_service,
                                                                                 'list_directories_and_files',
@@ -84,7 +83,7 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
         c.argument('if_match')
         c.argument('if_none_match')
 
-    for item in ['delete', 'list', 'show', 'show-usage', 'update', 'keys']:
+    for item in ['delete', 'list', 'show', 'update', 'keys']:
         with self.argument_context('storage account {}'.format(item)) as c:
             c.argument('account_name', acct_name_type, options_list=['--name', '-n'])
 
@@ -93,7 +92,7 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
 
     with self.argument_context('storage account create') as c:
         t_account_type, t_sku_name, t_kind = self.get_models('AccountType', 'SkuName', 'Kind',
-                                                             resource_type=ResourceType.MGMT_STORAGE)
+                                                             resource_type=CUSTOM_MGMT_STORAGE)
 
         c.register_common_storage_account_options()
         c.argument('location', get_location_type(self.cli_ctx), validator=get_default_location_from_resource_group)
@@ -126,10 +125,10 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
         c.ignore('encryption_key_vault_properties')
 
     for scope in ['storage account create', 'storage account update']:
-        with self.argument_context(scope, resource_type=ResourceType.MGMT_STORAGE, min_api='2017-06-01',
+        with self.argument_context(scope, resource_type=CUSTOM_MGMT_STORAGE, min_api='2017-06-01',
                                    arg_group='Network Rule') as c:
             t_bypass, t_default_action = self.get_models('Bypass', 'DefaultAction',
-                                                         resource_type=ResourceType.MGMT_STORAGE)
+                                                         resource_type=CUSTOM_MGMT_STORAGE)
 
             c.argument('bypass', nargs='+', validator=validate_bypass, arg_type=get_enum_type(t_bypass),
                        help='Bypass traffic for space-separated uses.')
@@ -417,6 +416,17 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
         c.argument('lease_duration', type=int)
         c.argument('lease_break_period', type=int)
 
+    with self.argument_context('storage container immutability-policy') as c:
+        c.argument('immutability_period_since_creation_in_days', options_list='--period')
+        c.argument('container_name', container_name_type)
+        c.argument('account_name', completer=get_resource_name_completion_list('Microsoft.Storage/storageAccounts'))
+
+    with self.argument_context('storage container legal-hold') as c:
+        c.argument('container_name', container_name_type)
+        c.argument('account_name', completer=get_resource_name_completion_list('Microsoft.Storage/storageAccounts'))
+        c.argument('tags', nargs='+', help='Each tag should be 3 to 23 alphanumeric characters and is '
+                                           'normalized to lower case')
+
     with self.argument_context('storage share') as c:
         c.argument('share_name', share_name_type, options_list=('--name', '-n'))
 

src/storage-preview/azext_storage_preview/_validators.py

@@ -6,13 +6,14 @@
 # pylint: disable=protected-access
 from azure.cli.core.commands.client_factory import get_mgmt_service_client
 from azure.cli.core.commands.validators import validate_key_value_pairs
-from azure.cli.core.profiles import ResourceType, get_sdk
+from azure.cli.core.profiles import get_sdk
 
 from ._client_factory import get_storage_data_service_client
 from .util import glob_files_locally, guess_content_type
 from .sdkutil import get_table_data_type
 from .url_quote_util import encode_for_url
 from .oauth_token_util import TokenUpdater
+from .profiles import CUSTOM_MGMT_STORAGE
 
 storage_account_key_options = {'primary': 'key1', 'secondary': 'key2'}
 
@@ -23,15 +24,15 @@
 # pylint: disable=inconsistent-return-statements
 def _query_account_key(cli_ctx, account_name):
     """Query the storage account key. This is used when the customer doesn't offer account key but name."""
-    scf = get_mgmt_service_client(cli_ctx, ResourceType.MGMT_STORAGE)
+    scf = get_mgmt_service_client(cli_ctx, CUSTOM_MGMT_STORAGE)
     acc = next((x for x in scf.storage_accounts.list() if x.name == account_name), None)
     if acc:
         from msrestazure.tools import parse_resource_id
         rg = parse_resource_id(acc.id)['resource_group']
 
         t_storage_account_keys, t_storage_account_list_keys_results = get_sdk(
             cli_ctx,
-            ResourceType.MGMT_STORAGE,
+            CUSTOM_MGMT_STORAGE,
             'models.storage_account_keys#StorageAccountKeys',
             'models.storage_account_list_keys_result#StorageAccountListKeysResult')
 
@@ -398,7 +399,7 @@ def validate_encryption_services(cmd, namespace):
     Builds up the encryption services object for storage account operations based on the list of services passed in.
     """
     if namespace.encryption_services:
-        t_encryption_services, t_encryption_service = get_sdk(cmd.cli_ctx, ResourceType.MGMT_STORAGE,
+        t_encryption_services, t_encryption_service = get_sdk(cmd.cli_ctx, CUSTOM_MGMT_STORAGE,
                                                               'EncryptionServices', 'EncryptionService', mod='models')
         services = {service: t_encryption_service(enabled=True) for service in namespace.encryption_services}
 
@@ -420,7 +421,7 @@ def validate_encryption_source(cmd, namespace):
         if namespace.encryption_key_source != 'Microsoft.Keyvault':
             raise ValueError('--encryption-key-name, --encryption-key-vault, and --encryption-key-version are not '
                              'applicable when --encryption-key-source=Microsoft.Keyvault is not specified.')
-        KeyVaultProperties = get_sdk(cmd.cli_ctx, ResourceType.MGMT_STORAGE, 'KeyVaultProperties',
+        KeyVaultProperties = get_sdk(cmd.cli_ctx, CUSTOM_MGMT_STORAGE, 'KeyVaultProperties',
                                      mod='models')
         if not KeyVaultProperties:
             return

src/storage-preview/azext_storage_preview/commands.py

@@ -5,20 +5,21 @@
 
 from azure.cli.core.commands import CliCommandType
 from azure.cli.core.profiles import ResourceType
-from ._client_factory import (cf_sa, blob_data_service_factory,
+from ._client_factory import (cf_sa, cf_blob_container_mgmt, blob_data_service_factory,
                               page_blob_service_factory, file_data_service_factory,
                               queue_data_service_factory, table_data_service_factory,
                               cloud_storage_account_service_factory,
                               multi_service_properties_factory)
 from .sdkutil import cosmosdb_table_exists
-from .profiles import CUSTOM_DATA_STORAGE
+from .profiles import CUSTOM_DATA_STORAGE, CUSTOM_MGMT_STORAGE
+from ._format import transform_immutability_policy
 
 
 def load_command_table(self, _):  # pylint: disable=too-many-locals, too-many-statements
     storage_account_sdk = CliCommandType(
         operations_tmpl='azure.mgmt.storage.operations.storage_accounts_operations#StorageAccountsOperations.{}',
         client_factory=cf_sa,
-        resource_type=ResourceType.MGMT_STORAGE
+        resource_type=CUSTOM_MGMT_STORAGE
     )
 
     storage_account_custom_type = CliCommandType(
@@ -40,7 +41,7 @@ def get_custom_sdk(custom_module, client_factory, resource_type=CUSTOM_DATA_STOR
             resource_type=resource_type
         )
 
-    with self.command_group('storage account', storage_account_sdk, resource_type=ResourceType.MGMT_STORAGE,
+    with self.command_group('storage account', storage_account_sdk, resource_type=CUSTOM_MGMT_STORAGE,
                             custom_command_type=storage_account_custom_type) as g:
         g.command('check-name', 'check_name_availability')
         g.custom_command('create', 'create_storage_account', min_api='2016-01-01')
@@ -57,7 +58,7 @@ def get_custom_sdk(custom_module, client_factory, resource_type=CUSTOM_DATA_STOR
 
     with self.command_group('storage account network-rule', storage_account_sdk,
                             custom_command_type=storage_account_custom_type,
-                            resource_type=ResourceType.MGMT_STORAGE, min_api='2017-06-01') as g:
+                            resource_type=CUSTOM_MGMT_STORAGE, min_api='2017-06-01') as g:
         g.custom_command('add', 'add_network_rule')
         g.custom_command('list', 'list_network_rules')
         g.custom_command('remove', 'remove_network_rule')
@@ -190,6 +191,25 @@ def get_custom_sdk(custom_module, client_factory, resource_type=CUSTOM_DATA_STOR
         g.storage_custom_command_oauth('policy show', 'get_acl_policy', exception_handler=g.get_handler_suppress_404())
         g.storage_custom_command_oauth('policy list', 'list_acl_policies', table_transformer=transform_acl_list_output)
 
+    blob_container_mgmt_sdk = CliCommandType(
+        operations_tmpl='azext_storage_preview.vendored_sdks.azure_mgmt_storage.operations.blob_containers_operations'
+                        '#BlobContainersOperations.{}',
+        client_factory=cf_blob_container_mgmt,
+        resource_type=CUSTOM_MGMT_STORAGE
+    )
+
+    with self.command_group('storage container immutability-policy', command_type=blob_container_mgmt_sdk) as g:
+        g.command('show', 'get_immutability_policy', transform=transform_immutability_policy)
+        g.command('create', 'create_or_update_immutability_policy')
+        g.command('delete', 'delete_immutability_policy', transform=lambda x: None)
+        g.command('lock', 'lock_immutability_policy')
+        g.command('extend', 'extend_immutability_policy')
+
+    with self.command_group('storage container legal-hold', command_type=blob_container_mgmt_sdk) as g:
+        g.command('set', 'set_legal_hold')
+        g.command('clear', 'clear_legal_hold')
+        g.command('show', 'get', transform=lambda x: getattr(x, 'legal_hold', x))
+
     file_sdk = CliCommandType(
         operations_tmpl='azure.multiapi.storage.file.fileservice#FileService.{}',
         client_factory=file_data_service_factory,