Merge branch 'release-1.3.1'

* release-1.3.1: (25 commits)
  Bumping version to 1.3.1
  Add ELB changelog item
  Update CHANGELOG with latest changes
  Fix regression when parsing number types in shorthand
  Use public rather than private members
  When JSON model's no_paramfile attribute is given, check for its value
  Add 691 to changelog
  Update bundled installer to work with pip 1.5
  Add Route53 max items test case
  More useful error and added tests
  Fix pagination with string params
  Update CHANGELOG
  Don't encode the copy_source argument
  The fake session should no longer unquote the copy_source.
  Let botocore handle the quoting of the x-amz-copy-source header.
  Quote the filename in the x-amz-copy-source header for multipart COPY operations.  Fixes #614.
  Fix integ test failure on windows
  Update changelog with create-snapshots issue
  Update CHANGELOG with issue 681
  Remove region check in s3 high level commands
  ...

CHANGELOG.rst

@@ -5,6 +5,32 @@ CHANGELOG
 Next Release (TBD)
 ==================
 
+* bugfix:Parameters: Fix issue parsing with CLI
+  parameters of type ``long``
+  (`issue 693 <https://github.com/aws/aws-cli/pull/693/files>`__)
+* bugfix:Pagination: Fix issue where ``--max-items``
+  in pagination was always assumed to be an integer
+  (`issue 689 <https://github.com/aws/aws-cli/pull/689>`__)
+* feature:``aws elb``: Add support for AccessLog
+* bugfix:Bundled Installer: Allow creation of bundled
+  installer with ``pip 1.5``
+  (`issue 691 <https://github.com/aws/aws-cli/issues/691>`__)
+* bugfix:``aws s3``: Fix issue when copying objects using
+  ``aws s3 cp`` with key names containing ``+`` characters
+  (`issue #614 <https://github.com/aws/aws-cli/issues/614>`__)
+* bugfix:``ec2 create-snapshot``: Remove ``Tags`` key from
+  output response
+  (`issue 247 <https://github.com/boto/botocore/pull/247>`__)
+* bugfix:``aws s3``: ``aws s3`` commands should not be requiring regions
+  (`issue 681 <https://github.com/aws/aws-cli/issues/681>`__)
+* bugfix:``CLI Arguments``: Fix issue where unicode command line
+  arguments were not being handled correctly
+  (`issue 679 <https://github.com/aws/aws-cli/pull/679>`__)
+
+
+1.3.0
+=====
+
 * bugfix:``aws s3``: Fix issue where S3 downloads would hang
   in certain cases and could not be interrupted
   (`issue 650 <https://github.com/aws/aws-cli/issues/650>`__,

README.rst

@@ -248,7 +248,7 @@ security group.  In this case, we will add ingress access to port 22
 for all IP addresses::
 
     $ aws ec2 authorize-security-group-ingress --group-name MySecurityGroup \
-      --ip-permissions '{"FromPort":22,"ToPort":22,"IpProtocol":"tcp","IpRanges":[{"cidr_ip": "0.0.0.0/0"}]}'
+      --ip-permissions '{"FromPort":22,"ToPort":22,"IpProtocol":"tcp","IpRanges":[{"CidrIp": "0.0.0.0/0"}]}'
 
 --------------------------
 File-based Parameter Input
@@ -266,7 +266,7 @@ the file ip_perms.json::
     {"FromPort":22,
      "ToPort":22,
      "IpProtocol":"tcp",
-     "IpRanges":[{"cidr_ip":"0.0.0.0/0"}]}
+     "IpRanges":[{"CidrIp":"0.0.0.0/0"}]}
 
 Then, we could make the same call as above like this::
 

awscli/__init__.py

@@ -17,7 +17,7 @@
 """
 import os
 
-__version__ = '1.3.0'
+__version__ = '1.3.1'
 
 #
 # Get our data path to be added to botocore's search path

awscli/argparser.py

@@ -11,6 +11,8 @@
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
 import argparse
+import sys
+import six
 from difflib import get_close_matches
 
 
@@ -43,6 +45,19 @@ def _check_value(self, action, value):
                 msg.extend(extra)
             raise argparse.ArgumentError(action, '\n'.join(msg))
 
+    def parse_known_args(self, args, namespace=None):
+        parsed, remaining = super(CLIArgParser, self).parse_known_args(args, namespace)
+        terminal_encoding = getattr(sys.stdin, 'encoding', 'utf-8')
+        if terminal_encoding is None:
+            # In some cases, sys.stdin won't have an encoding set,
+            # (e.g if it's set to a StringIO).  In this case we just
+            # default to utf-8.
+            terminal_encoding = 'utf-8'
+        for arg, value in vars(parsed).items():
+            if isinstance(value, six.binary_type):
+                setattr(parsed, arg, value.decode(terminal_encoding))
+        return parsed, remaining
+
 
 class MainArgParser(CLIArgParser):
     Formatter = argparse.RawTextHelpFormatter

awscli/argprocess.py

@@ -53,7 +53,7 @@ def uri_param(param, value, **kwargs):
     # Some params have a 'no_paramfile' attribute in their JSON
     # models which means that we should not allow any uri based params
     # for this argument.
-    if hasattr(param, 'no_paramfile'):
+    if getattr(param, 'no_paramfile', False):
         return
     else:
         return _check_for_uri_param(param, value)
@@ -65,7 +65,7 @@ def _check_for_uri_param(param, value):
     try:
         return get_paramfile(value)
     except ResourceLoadingError as e:
-        raise ParamError(param, str(e))
+        raise ParamError(param, six.text_type(e))
 
 
 def detect_shape_structure(param):
@@ -162,8 +162,9 @@ def get_parse_method_for_param(self, param, value=None):
         if isinstance(value, list):
             check_val = value[0]
         else:
-            check_val = value.strip()
-        if isinstance(check_val, str) and check_val.startswith(('[', '{')):
+            check_val = value
+        if isinstance(check_val, six.string_types) and check_val.strip().startswith(
+                ('[', '{')):
             LOG.debug("Param %s looks like JSON, not considered for "
                       "param shorthand.", param.py_name)
             return
@@ -345,7 +346,7 @@ def _split_on_commas(self, value):
         try:
             return utils.split_on_commas(value)
         except ValueError as e:
-            raise ParamSyntaxError(str(e))
+            raise ParamSyntaxError(six.text_type(e))
 
 
 def unpack_cli_arg(parameter, value):
@@ -370,7 +371,7 @@ def unpack_cli_arg(parameter, value):
     elif parameter.type in COMPLEX_TYPES:
         return unpack_complex_cli_arg(parameter, value)
     else:
-        return str(value)
+        return six.text_type(value)
 
 
 def unpack_complex_cli_arg(parameter, value):
@@ -415,8 +416,8 @@ def unpack_scalar_cli_arg(parameter, value):
             raise ParamError(parameter, msg)
         return open(file_path, 'rb')
     elif parameter.type == 'boolean':
-        if isinstance(value, str) and value.lower() == 'false':
+        if isinstance(value, six.string_types) and value.lower() == 'false':
             return False
         return bool(value)
     else:
-        return str(value)
+        return six.text_type(value)

awscli/customizations/paginate.py

@@ -66,8 +66,22 @@ def unify_paging_params(argument_table, operation, event_name, **kwargs):
                                                     STARTING_TOKEN_HELP,
                                                     operation,
                                                     parse_type='string')
+    # Try to get the pagination parameter type
+    limit_param = None
+    if 'limit_key' in operation.pagination:
+        for param in operation.params:
+            if param.name == operation.pagination['limit_key']:
+                limit_param = param
+                break
+
+    type_ = limit_param and limit_param.type or 'integer'
+    if limit_param and limit_param.type not in PageArgument.type_map:
+        raise TypeError(('Unsupported pagination type {0} for operation {1}'
+                         ' and parameter {2}').format(type_, operation.name,
+                                                      limit_param.name))
+
     argument_table['max-items'] = PageArgument('max-items', MAX_ITEMS_HELP,
-                                               operation, parse_type='integer')
+                                               operation, parse_type=type_)
 
 
 def check_should_enable_pagination(input_tokens, parsed_args, parsed_globals, **kwargs):

awscli/customizations/s3/fileinfo.py

@@ -235,7 +235,7 @@ def copy(self):
         """
         Copies a object in s3 to another location in s3.
         """
-        copy_source = quote(self.src.encode('utf-8'), safe='/~')
+        copy_source = self.src
         bucket, key = find_bucket_key(self.dest)
         params = {'endpoint': self.endpoint, 'bucket': bucket,
                   'copy_source': copy_source, 'key': key}

awscli/customizations/s3/s3.py

@@ -753,32 +753,7 @@ def check_region(self, parsed_globals):
         If the region is specified on the command line it takes priority
         over specification via a configuration file or environment variable.
         """
-        configuration = self.session.get_config()
-        env = os.environ.copy()
-        region = None
-        if 'region' in configuration.keys():
-            region = configuration['region']
-        if 'AWS_DEFAULT_REGION' in env.keys():
-            region = env['AWS_DEFAULT_REGION']
-        parsed_region = None
-        if 'region' in parsed_globals:
-            parsed_region = getattr(parsed_globals, 'region')
-        if 'endpoint_url' in parsed_globals:
-            parsed_endpoint_url = getattr(parsed_globals, 'endpoint_url')
-        else:
-            parsed_endpoint_url = None
-        if not region and not parsed_region and parsed_endpoint_url is None:
-            raise Exception("A region must be specified --region or "
-                            "specifying the region\nin a configuration "
-                            "file or as an environment variable.\n"
-                            "Alternately, an endpoint can be specified "
-                            "with --endpoint-url")
-        if parsed_region:
-            self.parameters['region'] = parsed_region
-        elif region:
-            self.parameters['region'] = region
-        else:
-            self.parameters['region'] = None
+        self.parameters['region'] = parsed_globals.region
 
     def check_endpoint_url(self, parsed_globals):
         """
@@ -789,6 +764,7 @@ def check_endpoint_url(self, parsed_globals):
         else:
             self.parameters['endpoint_url'] = None
 
+
 # This is a dictionary useful for automatically adding the different commands,
 # the amount of arguments it takes, and the optional parameters that can appear
 # on the same line as the command.  It also contains descriptions and usage

doc/source/conf.py

@@ -52,7 +52,7 @@
 # The short X.Y version.
 version = '1.3'
 # The full version, including alpha/beta/rc tags.
-release = '1.3.0'
+release = '1.3.1'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

scripts/make-bundle

@@ -35,6 +35,7 @@ PACKAGE_VERSION = {
     'simplejson': '3.3.0',
     'argparse': '1.2.1',
 }
+INSTALL_ARGS = '--allow-all-external --no-use-wheel'
 
 
 class BadRCError(Exception):
@@ -76,8 +77,8 @@ def create_scratch_dir():
 def download_package_tarballs(dirname, packages):
     with cd(dirname):
         for package in packages:
-            run('pip install -d . %s==%s' % (
-                package, PACKAGE_VERSION[package]))
+            run('pip install -d . %s==%s %s' % (
+                package, PACKAGE_VERSION[package], INSTALL_ARGS))
 
 
 def download_cli_deps(scratch_dir):
@@ -97,7 +98,7 @@ def download_cli_deps(scratch_dir):
     awscli_dir = os.path.dirname(
         os.path.dirname(os.path.abspath(__file__)))
     with cd(scratch_dir):
-        run('%s install -d . -e %s' % (pip, awscli_dir))
+        run('%s install %s -d . -e %s' % (pip, INSTALL_ARGS, awscli_dir))
     # Remove the awscli package, we'll create our own sdist.
     _remove_cli_zip(scratch_dir)
 
@@ -141,7 +142,30 @@ def zip_dir(scratch_dir):
     return os.path.join(dirname, basename)
 
 
+def verify_preconditions():
+    # The pip version looks like:
+    # 'pip 1.4.1 from ....'
+    pip_version = run('pip --version').strip().split()[1]
+    # Virtualenv version just has the version string: '1.14.5\n'
+    virtualenv_version = run('virtualenv --version').strip()
+    _min_version_required('1.5.0', pip_version, 'pip')
+    _min_version_required('1.11.4', virtualenv_version, 'virtualenv')
+
+
+def _min_version_required(min_version, actual_version, name):
+    # precondition: min_version is major.minor.patch
+    #               actual_version is major.minor.patch
+    min_split = min_version.split('.')
+    actual_split = actual_version.split('.')
+    for minimum, actual in zip(min_split, actual_split):
+        if int(actual) < int(minimum):
+            raise ValueError("%s requires at least version %s, but "
+                             "version %s was found." % (name, min_version,
+                                                        actual_version))
+
+
 def main():
+    verify_preconditions()
     scratch_dir = create_scratch_dir()
     package_dir = os.path.join(scratch_dir, 'packages')
     print("Bundle dir at: %s" % scratch_dir)

setup.py

@@ -6,7 +6,7 @@
 import awscli
 
 
-requires = ['botocore>=0.34.0,<0.35.0',
+requires = ['botocore>=0.35.0,<0.36.0',
             'bcdoc>=0.12.0,<0.13.0',
             'six>=1.1.0',
             'colorama==0.2.5',

tests/integration/customizations/s3/test_plugin.py

@@ -527,6 +527,17 @@ def test_ls_bucket(self):
         p = aws('s3 ls')
         self.assert_no_errors(p)
 
+    def test_ls_with_no_env_vars(self):
+        # By default, the aws() function injects
+        # an AWS_DEFAULT_REGION into the env var of the
+        # process.  We're verifying that a region does *not*
+        # need to be set anywhere.  If we provide our
+        # own environ dict, then the aws() function won't
+        # inject a region.
+        env = os.environ.copy()
+        p = aws('s3 ls', env_vars=env)
+        self.assert_no_errors(p)
+
     def test_ls_bucket_with_s3_prefix(self):
         p = aws('s3 ls s3://')
         self.assert_no_errors(p)

tests/unit/customizations/s3/fake_session.py

@@ -13,10 +13,10 @@
 import hashlib
 from operator import itemgetter
 from botocore.vendored import requests
+from six import text_type
 from six import StringIO
 from io import BytesIO
 
-from botocore.compat import unquote
 from mock import MagicMock, Mock
 
 from awscli.customizations.s3.filegenerator import find_bucket_key
@@ -224,8 +224,7 @@ def copy_object(self, kwargs):
         key = kwargs['key']
         copy_source = kwargs['copy_source']
         src_bucket, src_key = find_bucket_key(copy_source)
-        src_key = unquote(src_key)
-        if hasattr(src_key, 'decode'):
+        if not isinstance(src_key, text_type) and hasattr(src_key, 'decode'):
             src_key = src_key.decode('utf-8')
         response_data = {}
         etag = ''

tests/unit/customizations/s3/test_s3.py

@@ -369,7 +369,7 @@ def test_check_path_type_pass(self):
 
         for cmd in cmds.keys():
             cmd_param = CommandParameters(self.session, cmd, {})
-            cmd_param.check_region([])
+            cmd_param.check_region(mock.Mock())
             correct_paths = cmds[cmd]
             for path_args in correct_paths:
                 cmd_param.check_path_type(combos[path_args])
@@ -397,7 +397,7 @@ def test_check_path_type_fail(self):
 
         for cmd in cmds.keys():
             cmd_param = CommandParameters(self.session, cmd, {})
-            cmd_param.check_region([])
+            cmd_param.check_region(mock.Mock())
             wrong_paths = cmds[cmd]
             for path_args in wrong_paths:
                 with self.assertRaises(TypeError):
@@ -423,7 +423,7 @@ def test_check_src_path_pass(self):
         for filename in files:
             parameters['dir_op'] = filename[1]
             cmd_parameter = CommandParameters(self.session, 'put', parameters)
-            cmd_parameter.check_region([])
+            cmd_parameter.check_region(mock.Mock())
             cmd_parameter.check_src_path(filename[0])
 
     def test_check_force(self):
@@ -434,23 +434,6 @@ def test_check_force(self):
         cmd_params.parameters['src'] = 's3://mybucket'
         cmd_params.check_force(None)
 
-    def test_region(self):
-        # This tests the ability to specify the region and throw an error
-        # if a region is never specified whether if it is an environment
-        # variable, config file, or parsed global.
-        cmd_params = CommandParameters(self.session, 'mb', {})
-        parser = argparse.ArgumentParser()
-        parser.add_argument('--region', nargs=1)
-        parser.add_argument('--test', action='store_true')
-        parsed_args = parser.parse_args(['--region', 'eu-west-1'])
-        cmd_params.check_region(parsed_args)
-        self.assertEqual(cmd_params.parameters['region'][0], 'eu-west-1')
-
-        cmd_params2 = CommandParameters(self.mock, 'mb', {})
-        parsed_args2 = parser.parse_args(['--test'])
-        with self.assertRaises(Exception):
-            cmd_params2.check_region(parsed_args2)
-
 
 class HelpDocTest(BaseAWSHelpOutputTest):
     def setUp(self):