Merge branch 'release-0.31.0'

* release-0.31.0: (22 commits)
  Bumping version to 0.31.0
  Remove debug logging message.
  Fix reference to no_auth.
  Allow for operations within a service to override the signature_version.  Fixes #206.  Supercedes #208
  Fix setting socket timeout in py3
  Add response parsing tests for S3 GetBucketLocation
  Expose output parameters matching root XML node, fix GetBucketLocation
  Use unittest2 on python2.6
  Detect incomplete reads (content length mismatch)
  Simplifying code and fixing test to use unicode constant.
  Fixing an issue that came up while fixing aws/aws-cli#593.
  Fixing an issue that came up while fixing aws/aws-cli#593.
  Fix elastictranscoder service
  Add default param to get_config_variable
  Add session config vars for metadata retry/timeouts
  Add support for per session config vars
  Rename get_variable to get_config_variable
  Rename env vars to session vars
  Move module vars into session class vars
  Update elasticache model to the latest version
  ...

botocore/__init__.py

@@ -23,7 +23,7 @@
 import re
 import logging
 
-__version__ = '0.30.0'
+__version__ = '0.31.0'
 
 
 class NullHandler(logging.Handler):

botocore/auth.py

@@ -60,8 +60,6 @@ class SigV2Auth(BaseSigner):
 
     def __init__(self, credentials):
         self.credentials = credentials
-        if self.credentials is None:
-            raise NoCredentialsError
 
     def calc_signature(self, request, params):
         logger.debug("Calculating signature using v2 auth.")
@@ -92,6 +90,8 @@ def add_auth(self, request):
         # Because of this we have to parse the query params
         # from the request body so we can update them with
         # the sigv2 auth params.
+        if self.credentials is None:
+            raise NoCredentialsError
         if request.data:
             # POST
             params = request.data
@@ -112,10 +112,10 @@ def add_auth(self, request):
 class SigV3Auth(BaseSigner):
     def __init__(self, credentials):
         self.credentials = credentials
-        if self.credentials is None:
-            raise NoCredentialsError
 
     def add_auth(self, request):
+        if self.credentials is None:
+            raise NoCredentialsError
         if 'Date' not in request.headers:
             request.headers['Date'] = formatdate(usegmt=True)
         if self.credentials.token:
@@ -138,8 +138,6 @@ class SigV4Auth(BaseSigner):
 
     def __init__(self, credentials, service_name, region_name):
         self.credentials = credentials
-        if self.credentials is None:
-            raise NoCredentialsError
         # We initialize these value here so the unit tests can have
         # valid values.  But these will get overriden in ``add_auth``
         # later for real requests.
@@ -171,17 +169,44 @@ def headers_to_sign(self, request):
 
     def canonical_query_string(self, request):
         cqs = ''
+        # The query string can come from two parts.  One is the
+        # params attribute of the request.  The other is from the request
+        # url (in which case we have to re-split the url into its components
+        # and parse out the query string component).
         if request.params:
-            params = request.params
-            l = []
-            for param in params:
-                value = str(params[param])
-                l.append('%s=%s' % (quote(param, safe='-_.~'),
-                                    quote(value, safe='-_.~')))
-            l = sorted(l)
-            cqs = '&'.join(l)
+            return self._canonical_query_string_params(request.params)
+        else:
+            return self._canonical_query_string_url(urlsplit(request.url))
+        return cqs
+
+    def _canonical_query_string_params(self, params):
+        l = []
+        for param in params:
+            value = str(params[param])
+            l.append('%s=%s' % (quote(param, safe='-_.~'),
+                                quote(value, safe='-_.~')))
+        l = sorted(l)
+        cqs = '&'.join(l)
         return cqs
 
+    def _canonical_query_string_url(self, parts):
+        buf = ''
+        if parts.query:
+            qsa = parts.query.split('&')
+            qsa = [a.split('=', 1) for a in qsa]
+            quoted_qsa = []
+            for q in qsa:
+                if len(q) == 2:
+                    quoted_qsa.append(
+                        '%s=%s' % (quote(q[0], safe='-_.~'),
+                                   quote(unquote(q[1]), safe='-_.~')))
+                elif len(q) == 1:
+                    quoted_qsa.append('%s=' % quote(q[0], safe='-_.~'))
+            if len(quoted_qsa) > 0:
+                quoted_qsa.sort(key=itemgetter(0))
+                buf += '&'.join(quoted_qsa)
+        return buf
+
     def canonical_headers(self, headers_to_sign):
         """
         Return the headers that need to be included in the StringToSign
@@ -274,6 +299,8 @@ def signature(self, string_to_sign):
         return self._sign(k_signing, string_to_sign, hex=True)
 
     def add_auth(self, request):
+        if self.credentials is None:
+            raise NoCredentialsError
         # Create a new timestamp for each signing event
         now = datetime.datetime.utcnow()
         self.timestamp = now.strftime('%Y%m%dT%H%M%SZ')
@@ -306,25 +333,6 @@ def _add_headers_before_signing(self, request):
 
 class S3SigV4Auth(SigV4Auth):
 
-    def canonical_query_string(self, request):
-        split = urlsplit(request.url)
-        buf = ''
-        if split.query:
-            qsa = split.query.split('&')
-            qsa = [a.split('=', 1) for a in qsa]
-            quoted_qsa = []
-            for q in qsa:
-                if len(q) == 2:
-                    quoted_qsa.append(
-                        '%s=%s' % (quote(q[0], safe='-_.~'),
-                                   quote(unquote(q[1]), safe='-_.~')))
-                elif len(q) == 1:
-                    quoted_qsa.append('%s=' % quote(q[0], safe='-_.~'))
-            if len(quoted_qsa) > 0:
-                quoted_qsa.sort(key=itemgetter(0))
-                buf += '&'.join(quoted_qsa)
-        return buf
-
     def _add_headers_before_signing(self, request):
         super(S3SigV4Auth, self)._add_headers_before_signing(request)
         request.headers['X-Amz-Content-SHA256'] = self.payload(request)
@@ -348,8 +356,6 @@ class HmacV1Auth(BaseSigner):
 
     def __init__(self, credentials, service_name=None, region_name=None):
         self.credentials = credentials
-        if self.credentials is None:
-            raise NoCredentialsError
         self.auth_path = None  # see comment in canonical_resource below
 
     def sign_string(self, string_to_sign):
@@ -442,6 +448,8 @@ def get_signature(self, method, split, headers, expires=None):
         return self.sign_string(string_to_sign)
 
     def add_auth(self, request):
+        if self.credentials is None:
+            raise NoCredentialsError
         logger.debug("Calculating signature using hmacv1 auth.")
         split = urlsplit(request.url)
         logger.debug('HTTP request method: %s', request.method)

botocore/base.py

@@ -169,7 +169,7 @@ def get_search_path(session):
             os.path.dirname(
                 os.path.abspath(__file__))), 'botocore', 'data')
     paths = [builtin_path]
-    search_path = session.get_variable('data_path')
+    search_path = session.get_config_variable('data_path')
     if search_path is not None:
         extra_paths = search_path.split(os.pathsep)
         for path in extra_paths:

botocore/compat.py

@@ -34,6 +34,15 @@ class HTTPHeaders(http_client.HTTPMessage):
     from urllib.parse import parse_qsl
     from io import IOBase as _IOBase
     file_type = _IOBase
+
+    def set_socket_timeout(http_response, timeout):
+        """Set the timeout of the socket from an HTTPResponse.
+
+        :param http_response: An instance of ``httplib.HTTPResponse``
+
+        """
+        http_response._fp.fp.raw._sock.settimeout(timeout)
+
 else:
     from urllib import quote
     from urllib import unquote
@@ -51,6 +60,14 @@ def __iter__(self):
             for field, value in self._headers:
                 yield field
 
+    def set_socket_timeout(http_response, timeout):
+        """Set the timeout of the socket from an HTTPResponse.
+
+        :param http_response: An instance of ``httplib.HTTPResponse``
+
+        """
+        http_response._fp.fp._sock.settimeout(timeout)
+
 try:
     from collections import OrderedDict
 except ImportError:

botocore/config.py

@@ -41,7 +41,7 @@ def get_config(session):
     """
     config = {}
     path = None
-    path = session.get_variable('config_file')
+    path = session.get_config_variable('config_file')
     if path is not None:
         path = os.path.expandvars(path)
         path = os.path.expanduser(path)

botocore/credentials.py

@@ -63,23 +63,23 @@ def __init__(self, access_key=None, secret_key=None, token=None):
 
 
 def retrieve_iam_role_credentials(url=METADATA_SECURITY_CREDENTIALS_URL,
-                                  timeout=None, num_retries=0):
+                                  timeout=None, num_attempts=1):
     if timeout is None:
         timeout = DEFAULT_METADATA_SERVICE_TIMEOUT
     d = {}
     try:
-        r = _get_request(url, timeout, num_retries)
+        r = _get_request(url, timeout, num_attempts)
         if r.content:
             fields = r.content.decode('utf-8').split('\n')
             for field in fields:
                 if field.endswith('/'):
                     d[field[0:-1]] = retrieve_iam_role_credentials(
-                        url + field, timeout, num_retries)
+                        url + field, timeout, num_attempts)
                 else:
                     val = _get_request(
                         url + field,
                         timeout=timeout,
-                        num_retries=num_retries).content.decode('utf-8')
+                        num_attempts=num_attempts).content.decode('utf-8')
                     if val[0] == '{':
                         val = json.loads(val)
                     d[field] = val
@@ -88,14 +88,14 @@ def retrieve_iam_role_credentials(url=METADATA_SECURITY_CREDENTIALS_URL,
                          "of %s for url: %s, content body: %s",
                          r.status_code, url, r.content)
     except _RetriesExceededError:
-        logger.debug("Max number of retries exceeded (%s) when "
+        logger.debug("Max number of attempts exceeded (%s) when "
                      "attempting to retrieve data from metadata service.",
-                     num_retries)
+                     num_attempts)
     return d
 
 
-def _get_request(url, timeout, num_retries):
-    for i in range(num_retries + 1):
+def _get_request(url, timeout, num_attempts):
+    for i in range(num_attempts):
         try:
             response = requests.get(url, timeout=timeout)
         except (requests.Timeout, requests.ConnectionError) as e:
@@ -107,9 +107,16 @@ def _get_request(url, timeout, num_retries):
     raise _RetriesExceededError()
 
 
-def search_iam_role(**kwargs):
+def search_iam_role(session, **kwargs):
     credentials = None
-    metadata = retrieve_iam_role_credentials()
+    timeout = session.get_config_variable('metadata_service_timeout')
+    num_attempts = session.get_config_variable('metadata_service_num_attempts')
+    retrieve_kwargs = {}
+    if timeout is not None:
+        retrieve_kwargs['timeout'] = float(timeout)
+    if num_attempts is not None:
+        retrieve_kwargs['num_attempts'] = int(num_attempts)
+    metadata = retrieve_iam_role_credentials(**retrieve_kwargs)
     if metadata:
         for role_name in metadata:
             credentials = Credentials(metadata[role_name]['AccessKeyId'],
@@ -126,9 +133,9 @@ def search_environment(**kwargs):
     """
     session = kwargs.get('session')
     credentials = None
-    access_key = session.get_variable('access_key', ('env',))
-    secret_key = session.get_variable('secret_key', ('env',))
-    token = session.get_variable('token', ('env',))
+    access_key = session.get_config_variable('access_key', ('env',))
+    secret_key = session.get_config_variable('secret_key', ('env',))
+    token = session.get_config_variable('token', ('env',))
     if access_key and secret_key:
         credentials = Credentials(access_key, secret_key, token)
         credentials.method = 'env'
@@ -165,9 +172,9 @@ def search_file(**kwargs):
     """
     credentials = None
     session = kwargs.get('session')
-    access_key = session.get_variable('access_key', methods=('config',))
-    secret_key = session.get_variable('secret_key', methods=('config',))
-    token = session.get_variable('token', ('config',))
+    access_key = session.get_config_variable('access_key', methods=('config',))
+    secret_key = session.get_config_variable('secret_key', methods=('config',))
+    token = session.get_config_variable('token', ('config',))
     if access_key and secret_key:
         credentials = Credentials(access_key, secret_key, token)
         credentials.method = 'config'

botocore/data/aws/elasticache.json

@@ -158,20 +158,17 @@
                     "NumCacheNodes": {
                         "shape_name": "IntegerOptional",
                         "type": "integer",
-                        "documentation": "\n        <p>The initial number of cache nodes that the cache cluster will have.</p>\n        <p>For a Memcached cluster, valid values are between 1 and 20. If you need to exceed this\n            limit, please fill out the ElastiCache Limit Increase Request form at <a href=\"http://aws.amazon.com/contact-us/elasticache-node-limit-request/\"></a> .</p>\n        <p>For Redis, only single-node cache clusters are supported at this time, so the value for\n            this parameter must be 1.</p>\n    ",
-                        "required": true
+                        "documentation": "\n        <p>The initial number of cache nodes that the cache cluster will have.</p>\n        <p>For a Memcached cluster, valid values are between 1 and 20. If you need to exceed this\n            limit, please fill out the ElastiCache Limit Increase Request form at <a href=\"http://aws.amazon.com/contact-us/elasticache-node-limit-request/\"></a> .</p>\n        <p>For Redis, only single-node cache clusters are supported at this time, so the value for\n            this parameter must be 1.</p>\n    "
                     },
                     "CacheNodeType": {
                         "shape_name": "String",
                         "type": "string",
-                        "documentation": "\n        <p>The compute and memory capacity of the nodes in the cache cluster.</p> \n<p>Valid values for Memcached:</p>\n    <p>\n        <code>cache.t1.micro</code> |  \n        <code>cache.m1.small</code> | \n        <code>cache.m1.medium</code> |\n        <code>cache.m1.large</code> | \n        <code>cache.m1.xlarge</code> | \n        <code>cache.m3.xlarge</code> | \n        <code>cache.m3.2xlarge</code> | \n        <code>cache.m2.xlarge</code> | \n        <code>cache.m2.2xlarge</code> |\n        <code>cache.m2.4xlarge</code> | \n        <code>cache.c1.xlarge</code>\n    </p>\n    <p>Valid values for Redis:</p>\n    <p>\n       <code>cache.t1.micro</code> | \n       <code>cache.m1.small</code> | \n       <code>cache.m1.medium</code> |\n       <code>cache.m1.large</code> | \n       <code>cache.m1.xlarge</code> |  \n       <code>cache.m2.xlarge</code> | \n       <code>cache.m2.2xlarge</code> | \n       <code>cache.m2.4xlarge</code> | \n       <code>cache.c1.xlarge</code>\n    </p>\n<p>For a complete listing of cache node types and specifications, see <a href=\"http://aws.amazon.com/elasticache/\"/>.</p>\n ",
-                        "required": true
+                        "documentation": "\n        <p>The compute and memory capacity of the nodes in the cache cluster.</p> \n<p>Valid values for Memcached:</p>\n    <p>\n        <code>cache.t1.micro</code> |  \n        <code>cache.m1.small</code> | \n        <code>cache.m1.medium</code> |\n        <code>cache.m1.large</code> | \n        <code>cache.m1.xlarge</code> | \n        <code>cache.m3.xlarge</code> | \n        <code>cache.m3.2xlarge</code> | \n        <code>cache.m2.xlarge</code> | \n        <code>cache.m2.2xlarge</code> |\n        <code>cache.m2.4xlarge</code> | \n        <code>cache.c1.xlarge</code>\n    </p>\n    <p>Valid values for Redis:</p>\n    <p>\n       <code>cache.t1.micro</code> | \n       <code>cache.m1.small</code> | \n       <code>cache.m1.medium</code> |\n       <code>cache.m1.large</code> | \n       <code>cache.m1.xlarge</code> |  \n       <code>cache.m2.xlarge</code> | \n       <code>cache.m2.2xlarge</code> | \n       <code>cache.m2.4xlarge</code> | \n       <code>cache.c1.xlarge</code>\n    </p>\n<p>For a complete listing of cache node types and specifications, see <a href=\"http://aws.amazon.com/elasticache/\"/>.</p>\n "
                     },
                     "Engine": {
                         "shape_name": "String",
                         "type": "string",
-                        "documentation": "\n        <p>The name of the cache engine to be used for this cache cluster.</p>\n        <p>Valid values for this parameter are:</p>\n            <p><code>memcached</code> | <code>redis</code></p>\n    ",
-                        "required": true
+                        "documentation": "\n        <p>The name of the cache engine to be used for this cache cluster.</p>\n        <p>Valid values for this parameter are:</p>\n            <p><code>memcached</code> | <code>redis</code></p>\n    "
                     },
                     "EngineVersion": {
                         "shape_name": "String",
@@ -1647,6 +1644,11 @@
                         "type": "string",
                         "documentation": "\n        <p>The identifier for the replication group to be deleted. This parameter is not case\n            sensitive.</p>\n    ",
                         "required": true
+                    },
+                    "RetainPrimaryCluster": {
+                        "shape_name": "BooleanOptional",
+                        "type": "boolean",
+                        "documentation": "\n        <p>If set to <i>true</i>, all of the read replicas will be deleted, but the primary \n        cache cluster will be retained.</p>\n    "
                     }
                 },
                 "documentation": "\n        <p>Represents the input of a <i>DeleteReplicationGroup</i> operation.</p>\n    "
@@ -1831,7 +1833,7 @@
                     "documentation": "\n        <p>Two or more incompatible parameters were specified.</p>\n    "
                 }
             ],
-            "documentation": "\n        <p>The <i>DeleteReplicationGroup</i> operation deletes an existing replication group.\n            <i>DeleteReplicationGroup</i> deletes the primary cache cluster and all of the read replicas in the replication\n            group. When you receive a successful response\n            from this operation, Amazon ElastiCache immediately begins deleting the entire replication group; you\n            cannot cancel or revert this operation.</p>\n    "
+            "documentation": "\n        <p>The <i>DeleteReplicationGroup</i> operation deletes an existing replication group.  \n        By default, this operation deletes the entire replication group, including the primary \n        cache cluster and all of the read replicas. You can optionally delete only the read \n        replicas, while retaining the primary cache cluster.</p>\n        <p>When you receive a successful response from this operation, Amazon ElastiCache immediately \n        begins deleting the selected resources; you cannot cancel or revert this operation.</p>\n    "
         },
         "DescribeCacheClusters": {
             "name": "DescribeCacheClusters",