[Storage] Upgrade Azcopy for storage copy/remove and blob sync (#11874)

[Storage] Integrate Azcopy 10.3.3 and support Win32.
[Storage] `az storage copy`: Add `--include-path`, `--include-pattern`, `--exclude-path` and`--exclude-pattern` parameters
[Storage] `az storage remove`: Change `--inlcude` and `--exclude` parameters to `--include-path`, `--include-pattern`, `--exclude-path` and`--exclude-pattern` parameters
[Storage] `az storage sync`: Add `--include-pattern`, `--exclude-path` and`--exclude-pattern` parameters

src/azure-cli/HISTORY.rst

@@ -44,6 +44,10 @@ Release History
 
 * Add a new command group `az storage share-rm` to use the Microsoft.Storage resource provider for Azure file share management operations.
 * Fix issue #11415: permission error for `az storage blob update`
+* Integrate Azcopy 10.3.3 and support Win32.
+* `az storage copy`: Add `--include-path`, `--include-pattern`, `--exclude-path` and`--exclude-pattern` parameters
+* `az storage remove`: Change `--inlcude` and `--exclude` parameters to `--include-path`, `--include-pattern`, `--exclude-path` and`--exclude-pattern` parameters
+* `az storage sync`: Add `--include-pattern`, `--exclude-path` and`--exclude-pattern` parameters
 
 2.0.80
 ++++++

src/azure-cli/azure/cli/command_modules/storage/_help.py

@@ -534,7 +534,6 @@
 helps['storage blob sync'] = """
 type: command
 short-summary: Sync blobs recursively to a storage blob container.
-long-summary: Sync command depends on Azcopy, which will be upgraded to v10.3 soon to support 32-bit Operating System and utilize new features.
 examples:
   - name: Sync a single blob to a container.
     text: az storage blob sync -c mycontainer -s "path/to/file" -d NewBlob
@@ -721,12 +720,6 @@
 helps['storage copy'] = """
 type: command
 short-summary: Copy files or directories to or from Azure storage.
-long-summary: >
-    Copy command depends on Azcopy, which will be upgraded to v10.3 soon to support 32-bit Operating System and
-    utilize new features.
-
-    [COMING BREAKING CHANGE] With Azcopy v10.3, `*` character is no longer supported as a wildcard in URL, but new
-    parameters --include-pattern and --exclude-pattern will be added with `*` wildcard support.
 examples:
   - name: Upload a single file to Azure Blob using url.
     text: az storage copy -s /path/to/file.txt -d https://[account].blob.core.windows.net/[container]/[path/to/blob]
@@ -750,8 +743,10 @@
     text: az storage copy -s https://[account].blob.core.windows.net/[container]/[path/to/blob] -d /path/to/file.txt
   - name: Download an entire directory from Azure Blob, and you can also specify your storage account and container information as above.
     text: az storage copy -s https://[account].blob.core.windows.net/[container]/[path/to/directory] -d /path/to/dir --recursive
-  - name: Download a set of files from Azure Blob using wildcards, and you can also specify your storage account and container information as above.
-    text: az storage copy -s https://[account].blob.core.windows.net/[container]/foo* -d /path/to/dir --recursive
+  - name: Download a subset of containers within a storage account by using a wildcard symbol (*) in the container name, and you can also specify your storage account and container information as above.
+    text: az storage copy -s https://[account].blob.core.windows.net/[container*name] -d /path/to/dir --recursive
+  - name: Download a subset of files from Azure Blob. (Only jpg files and file names don't start with test will be included.)
+    text: az storage copy -s https://[account].blob.core.windows.net/[container] --include-pattern "*.jpg" --exclude-pattern test* -d /path/to/dir --recursive
   - name: Copy a single blob to another blob, and you can also specify the storage account and container information of source and destination as above.
     text: az storage copy -s https://[srcaccount].blob.core.windows.net/[container]/[path/to/blob] -d https://[destaccount].blob.core.windows.net/[container]/[path/to/blob]
   - name: Copy an entire account data from blob account to another blob account, and you can also specify the storage account and container information of source and destination as above.
@@ -783,7 +778,7 @@
   - name: Download an entire directory from Azure File Share, and you can also specify your storage account and share information as above.
     text: az storage copy -s https://[account].file.core.windows.net/[share]/[path/to/directory] -d /path/to/dir --recursive
   - name: Download a set of files from Azure File Share using wildcards, and you can also specify your storage account and share information as above.
-    text: az storage copy -s https://[account].file.core.windows.net/[share]/foo* -d /path/to/dir --recursive
+    text: az storage copy -s https://[account].file.core.windows.net/[share]/ --include-pattern foo* -d /path/to/dir --recursive
 """
 
 helps['storage cors'] = """
@@ -1204,10 +1199,6 @@
 helps['storage remove'] = """
 type: command
 short-summary: Delete blobs or files from Azure Storage.
-long-summary: >
-    To delete blobs, both the source must either be public or be authenticated by using a shared access signature.
-    Remove command depends on Azcopy, which will be upgraded to v10.3 soon to support 32-bit Operating System and
-    utilize new features.
 examples:
   - name: Remove a single blob.
     text: az storage remove -c MyContainer -n MyBlob
@@ -1217,10 +1208,10 @@
     text: az storage remove -c MyContainer --recursive
   - name: Remove all the blobs in a Storage Container.
     text: az storage remove -c MyContainer -n path/to/directory
-  - name: Remove a subset of blobs in a virtual directory (For example, only jpg and pdf files, or if the blob name is "exactName").
-    text: az storage remove -c MyContainer -n path/to/directory --recursive --include "*.jpg;*.pdf;exactName"
+  - name: Remove a subset of blobs in a virtual directory (For example, only jpg and pdf files, or if the blob name is "exactName" and file names don't start with "test").
+    text: az storage remove -c MyContainer --include-path path/to/directory --include-pattern "*.jpg;*.pdf;exactName" --exclude-pattern "test*" --recursive
   - name: Remove an entire virtual directory but exclude certain blobs from the scope (For example, every blob that starts with foo or ends with bar).
-    text: az storage remove -c MyContainer -n path/to/directory --recursive --include "foo*;*bar"
+    text: az storage remove -c MyContainer --include-path path/to/directory --exclude-pattern "foo*;*bar" --recursive
   - name: Remove a single file.
     text: az storage remove -s MyShare -p MyFile
   - name: Remove an entire directory.

src/azure-cli/azure/cli/command_modules/storage/_params.py

@@ -99,6 +99,20 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
     azure_storage_sid_type = CLIArgumentType(min_api='2019-04-01', arg_group="Azure Active Directory Properties",
                                              help="Specify the security identifier (SID) for Azure Storage. "
                                                   "Required when --enable-files-adds is set to True")
+    exclude_pattern_type = CLIArgumentType(arg_group='Additional Flags', help='Exclude these files where the name '
+                                           'matches the pattern list. For example: *.jpg;*.pdf;exactName. This '
+                                           'option supports wildcard characters (*)')
+    include_pattern_type = CLIArgumentType(arg_group='Additional Flags', help='Include only these files where the name '
+                                           'matches the pattern list. For example: *.jpg;*.pdf;exactName. This '
+                                           'option supports wildcard characters (*)')
+    exclude_path_type = CLIArgumentType(arg_group='Additional Flags', help='Exclude these paths. This option does not '
+                                        'support wildcard characters (*). Checks relative path prefix. For example: '
+                                        'myFolder;myFolder/subDirName/file.pdf.')
+    include_path_type = CLIArgumentType(arg_group='Additional Flags', help='Include only these paths. This option does '
+                                        'not support wildcard characters (*). Checks relative path prefix. For example:'
+                                        'myFolder;myFolder/subDirName/file.pdf')
+    recursive_type = CLIArgumentType(options_list=['--recursive', '-r'], action='store_true',
+                                     help='Look into sub-directories recursively.')
     sas_help = 'The permissions the SAS grants. Allowed values: {}. Do not use if a stored access policy is ' \
                'referenced with --id that specifies this value. Can be combined.'
 
@@ -474,8 +488,6 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
                        help='File path in file share of copy {} storage account'.format(item))
             c.argument('{}_local_path'.format(item), arg_group='Copy {}'.format(item),
                        help='Local file path')
-        c.argument('recursive', arg_group='Additional Flags', action='store_true', help='Look into sub-directories \
-                    recursively when uploading from local file system.')
         c.argument('put_md5', arg_group='Additional Flags', action='store_true',
                    help='Create an MD5 hash of each file, and save the hash as the Content-MD5 property of the '
                    'destination blob/file.Only available when uploading.')
@@ -488,6 +500,11 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
                    'to ensure destination storage account support setting access tier. In the cases that setting '
                    'access tier is not supported, please use `--preserve-s2s-access-tier false` to bypass copying '
                    'access tier. (Default true)')
+        c.argument('exclude_pattern', exclude_pattern_type)
+        c.argument('include_pattern', include_pattern_type)
+        c.argument('exclude_path', exclude_path_type)
+        c.argument('include_path', include_path_type)
+        c.argument('recursive', recursive_type)
 
     with self.argument_context('storage blob copy') as c:
         for item in ['destination', 'source']:
@@ -539,6 +556,9 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
         c.argument('source', options_list=['--source', '-s'],
                    help='The source file path to sync from.')
         c.ignore('destination')
+        c.argument('exclude_pattern', exclude_pattern_type)
+        c.argument('include_pattern', include_pattern_type)
+        c.argument('exclude_path', exclude_path_type)
 
     with self.argument_context('storage container') as c:
         from .sdkutil import get_container_access_type_names
@@ -895,10 +915,11 @@ def load_arguments(self, _):  # pylint: disable=too-many-locals, too-many-statem
         c.extra('path', options_list=('--path', '-p'),
                 help='The path to the file within the file share.',
                 completer=file_path_completer)
-        c.argument('exclude', help='Exclude files whose name matches the pattern list.')
-        c.argument('include', help='Only include files whose name matches the pattern list.')
-        c.argument('recursive', options_list=['--recursive', '-r'], action='store_true',
-                   help='Look into sub-directories recursively when deleting between directories.')
+        c.argument('exclude_pattern', exclude_pattern_type)
+        c.argument('include_pattern', include_pattern_type)
+        c.argument('exclude_path', exclude_path_type)
+        c.argument('include_path', include_path_type)
+        c.argument('recursive', recursive_type)
         c.ignore('destination')
         c.ignore('service')
         c.ignore('target')

src/azure-cli/azure/cli/command_modules/storage/azcopy/util.py

@@ -23,35 +23,52 @@
 
 STORAGE_RESOURCE_ENDPOINT = "https://storage.azure.com"
 SERVICES = {'blob', 'file'}
-AZCOPY_VERSION = '10.1.0'
+AZCOPY_VERSION = '10.3.3'
 
 
 class AzCopy(object):
     def __init__(self, creds=None):
         self.system = platform.system()
         install_location = _get_default_install_location()
-        if not os.path.isfile(install_location):
-            install_dir = os.path.dirname(install_location)
-            if not os.path.exists(install_dir):
-                os.makedirs(install_dir)
-            base_url = 'https://azcopyvnext.azureedge.net/release20190423/azcopy_{}_amd64_10.1.0.{}'
-            if self.system == 'Windows':
-                file_url = base_url.format('windows', 'zip')
-            elif self.system == 'Linux':
-                file_url = base_url.format('linux', 'tar.gz')
-            elif self.system == 'Darwin':
-                file_url = base_url.format('darwin', 'zip')
-            else:
-                raise CLIError('Azcopy ({}) does not exist.'.format(self.system))
-            try:
-                _urlretrieve(file_url, install_location)
-                os.chmod(install_location,
-                         os.stat(install_location).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
-            except IOError as err:
-                raise CLIError('Connection error while attempting to download azcopy ({})'.format(err))
-
         self.executable = install_location
         self.creds = creds
+        if not os.path.isfile(install_location) or self.check_version() != AZCOPY_VERSION:
+            self.install_azcopy(install_location)
+
+    def install_azcopy(self, install_location):
+        install_dir = os.path.dirname(install_location)
+        if not os.path.exists(install_dir):
+            os.makedirs(install_dir)
+        base_url = 'https://azcopyvnext.azureedge.net/release20191212/azcopy_{}_{}_{}.{}'
+
+        if self.system == 'Windows':
+            if platform.machine().endswith('64'):
+                file_url = base_url.format('windows', 'amd64', AZCOPY_VERSION, 'zip')
+            else:
+                file_url = base_url.format('windows', '386', AZCOPY_VERSION, 'zip')
+        elif self.system == 'Linux':
+            file_url = base_url.format('linux', 'amd64', AZCOPY_VERSION, 'tar.gz')
+        elif self.system == 'Darwin':
+            file_url = base_url.format('darwin', 'amd64', AZCOPY_VERSION, 'zip')
+        else:
+            raise CLIError('Azcopy ({}) does not exist.'.format(self.system))
+        try:
+            _urlretrieve(file_url, install_location)
+            os.chmod(install_location,
+                     os.stat(install_location).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)
+        except IOError as err:
+            raise CLIError('Connection error while attempting to download azcopy ({})'.format(err))
+
+    def check_version(self):
+        try:
+            import re
+            args = [self.executable] + ["--version"]
+            out_bytes = subprocess.check_output(args)
+            out_text = out_bytes.decode('utf-8')
+            version = re.findall(r"azcopy version (.+?)\n", out_text)[0]
+            return version
+        except subprocess.CalledProcessError:
+            return ""
 
     def run_command(self, args):
         args = [self.executable] + args

src/azure-cli/azure/cli/command_modules/storage/operations/azcopy.py

@@ -7,7 +7,7 @@
 from ..azcopy.util import AzCopy, client_auth_for_azcopy, login_auth_for_azcopy
 from azure.cli.command_modules.storage._client_factory import blob_data_service_factory, file_data_service_factory
 
-# pylint: disable=too-many-statements
+# pylint: disable=too-many-statements, too-many-locals
 
 
 def storage_copy(cmd, source=None,
@@ -27,7 +27,8 @@ def storage_copy(cmd, source=None,
                  destination_blob=None,
                  destination_share=None,
                  destination_file_path=None,
-                 destination_local_path=None):
+                 destination_local_path=None,
+                 exclude_pattern=None, include_pattern=None, exclude_path=None, include_path=None):
     def get_url_with_sas(source, account_name, container, blob, share, file_path, local_path):
         import re
         import os
@@ -94,28 +95,48 @@ def get_url_with_sas(source, account_name, container, blob, share, file_path, lo
         flags.append('--blob-type=' + blob_type)
     if preserve_s2s_access_tier is not None:
         flags.append('--s2s-preserve-access-tier=' + str(preserve_s2s_access_tier))
-
+    if include_pattern is not None:
+        flags.append('--include-pattern=' + include_pattern)
+    if exclude_pattern is not None:
+        flags.append('--exclude-pattern=' + exclude_pattern)
+    if include_path is not None:
+        flags.append('--include-path=' + include_path)
+    if exclude_pattern is not None:
+        flags.append('--exclude-path=' + exclude_path)
     azcopy.copy(full_source, full_destination, flags=flags)
 
 
-def storage_remove(cmd, client, service, target, exclude=None, include=None, recursive=None):
+def storage_remove(cmd, client, service, target, recursive=None, exclude_pattern=None, include_pattern=None,
+                   exclude_path=None, include_path=None):
     if service == 'file':
         azcopy = _azcopy_file_client(cmd, client)
     else:
         azcopy = _azcopy_blob_client(cmd, client)
     flags = []
     if recursive is not None:
         flags.append('--recursive')
-    if include is not None:
-        flags.append('--include=' + include)
-    if exclude is not None:
-        flags.append('--exclude=' + exclude)
+    if include_pattern is not None:
+        flags.append('--include-pattern=' + include_pattern)
+    if exclude_pattern is not None:
+        flags.append('--exclude-pattern=' + exclude_pattern)
+    if include_path is not None:
+        flags.append('--include-path=' + include_path)
+    if exclude_path is not None:
+        flags.append('--exclude-path=' + exclude_path)
     azcopy.remove(_add_url_sas(target, azcopy.creds.sas_token), flags=flags)
 
 
-def storage_blob_sync(cmd, client, source, destination):
+def storage_blob_sync(cmd, client, source, destination, exclude_pattern=None, include_pattern=None,
+                      exclude_path=None):
     azcopy = _azcopy_blob_client(cmd, client)
-    azcopy.sync(source, _add_url_sas(destination, azcopy.creds.sas_token), flags=['--delete-destination', 'true'])
+    flags = ['--delete-destination=true']
+    if include_pattern is not None:
+        flags.append('--include-pattern=' + include_pattern)
+    if exclude_pattern is not None:
+        flags.append('--exclude-pattern=' + exclude_pattern)
+    if exclude_path is not None:
+        flags.append('--exclude-path=' + exclude_path)
+    azcopy.sync(source, _add_url_sas(destination, azcopy.creds.sas_token), flags=flags)
 
 
 def storage_run_command(cmd, command_args):