Fix elastictranscoder service

This fixes two issues to get elastictranscoder working:

* account for the `rest-json` service type in `get_response()`
* account for query strings coming from the url path in sigv4 signer

I've also added integration tests that verify we can talk to
the Elastic Transcoder service as expected.

botocore/auth.py

@@ -171,17 +171,44 @@ def headers_to_sign(self, request):
 
     def canonical_query_string(self, request):
         cqs = ''
+        # The query string can come from two parts.  One is the
+        # params attribute of the request.  The other is from the request
+        # url (in which case we have to re-split the url into its components
+        # and parse out the query string component).
         if request.params:
-            params = request.params
-            l = []
-            for param in params:
-                value = str(params[param])
-                l.append('%s=%s' % (quote(param, safe='-_.~'),
-                                    quote(value, safe='-_.~')))
-            l = sorted(l)
-            cqs = '&'.join(l)
+            return self._canonical_query_string_params(request.params)
+        else:
+            return self._canonical_query_string_url(urlsplit(request.url))
         return cqs
 
+    def _canonical_query_string_params(self, params):
+        l = []
+        for param in params:
+            value = str(params[param])
+            l.append('%s=%s' % (quote(param, safe='-_.~'),
+                                quote(value, safe='-_.~')))
+        l = sorted(l)
+        cqs = '&'.join(l)
+        return cqs
+
+    def _canonical_query_string_url(self, parts):
+        buf = ''
+        if parts.query:
+            qsa = parts.query.split('&')
+            qsa = [a.split('=', 1) for a in qsa]
+            quoted_qsa = []
+            for q in qsa:
+                if len(q) == 2:
+                    quoted_qsa.append(
+                        '%s=%s' % (quote(q[0], safe='-_.~'),
+                                   quote(unquote(q[1]), safe='-_.~')))
+                elif len(q) == 1:
+                    quoted_qsa.append('%s=' % quote(q[0], safe='-_.~'))
+            if len(quoted_qsa) > 0:
+                quoted_qsa.sort(key=itemgetter(0))
+                buf += '&'.join(quoted_qsa)
+        return buf
+
     def canonical_headers(self, headers_to_sign):
         """
         Return the headers that need to be included in the StringToSign
@@ -306,25 +333,6 @@ def _add_headers_before_signing(self, request):
 
 class S3SigV4Auth(SigV4Auth):
 
-    def canonical_query_string(self, request):
-        split = urlsplit(request.url)
-        buf = ''
-        if split.query:
-            qsa = split.query.split('&')
-            qsa = [a.split('=', 1) for a in qsa]
-            quoted_qsa = []
-            for q in qsa:
-                if len(q) == 2:
-                    quoted_qsa.append(
-                        '%s=%s' % (quote(q[0], safe='-_.~'),
-                                   quote(unquote(q[1]), safe='-_.~')))
-                elif len(q) == 1:
-                    quoted_qsa.append('%s=' % quote(q[0], safe='-_.~'))
-            if len(quoted_qsa) > 0:
-                quoted_qsa.sort(key=itemgetter(0))
-                buf += '&'.join(quoted_qsa)
-        return buf
-
     def _add_headers_before_signing(self, request):
         super(S3SigV4Auth, self)._add_headers_before_signing(request)
         request.headers['X-Amz-Content-SHA256'] = self.payload(request)

botocore/response.py

@@ -405,7 +405,7 @@ def get_response(session, operation, http_response):
         "Response Headers:\n%s",
         '\n'.join("%s: %s" % (k, v) for k, v in http_response.headers.items()))
     logger.debug("Response Body:\n%s", body)
-    if operation.service.type == 'json':
+    if operation.service.type in ('json', 'rest-json'):
         json_response = JSONResponse(session, operation)
         if body:
             json_response.parse(body, encoding)

tests/integration/test_elastictranscoder.py

@@ -0,0 +1,105 @@
+# Copyright (c) 2014 Amazon.com, Inc. or its affiliates.  All Rights Reserved
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+from tests import unittest
+import functools
+import random
+
+import botocore.session
+
+DEFAULT_ROLE_POLICY = """\
+{"Statement": [
+    {
+        "Action": "sts:AssumeRole",
+        "Principal": {
+            "Service": "elastictranscoder.amazonaws.com"
+        },
+        "Effect": "Allow",
+        "Sid": "1"
+    }
+]}
+"""
+
+class TestElasticTranscoder(unittest.TestCase):
+    def setUp(self):
+        self.session = botocore.session.get_session()
+        self.service = self.session.get_service('elastictranscoder')
+        self.endpoint = self.service.get_endpoint('us-east-1')
+
+    def create_bucket(self):
+        s3 = self.session.get_service('s3')
+        bucket_name = 'ets-bucket-1-%s' % random.randint(1, 1000000)
+        create_bucket = s3.get_operation('CreateBucket')
+        delete_bucket = s3.get_operation('DeleteBucket')
+        endpoint = s3.get_endpoint('us-east-1')
+        response = create_bucket.call(endpoint, bucket=bucket_name)[0]
+        self.assertEqual(response.status_code, 200)
+        self.addCleanup(
+            functools.partial(delete_bucket.call, endpoint,
+                              bucket=bucket_name))
+        return bucket_name
+
+    def create_iam_role(self):
+        iam = self.session.get_service('iam')
+        endpoint = iam.get_endpoint('us-east-1')
+        create_role = iam.get_operation('CreateRole')
+        delete_role = iam.get_operation('DeleteRole')
+        role_name = 'ets-role-name-1-%s' % random.randint(1, 1000000)
+        response, parsed = create_role.call(endpoint, role_name=role_name,
+            assume_role_policy_document=DEFAULT_ROLE_POLICY)
+        self.assertEqual(response.status_code, 200)
+        arn = parsed['Role']['Arn']
+        self.addCleanup(
+            functools.partial(delete_role.call, endpoint, role_name=role_name))
+        return arn
+
+    def test_list_streams(self):
+        operation = self.service.get_operation('ListPipelines')
+        http, parsed = operation.call(self.endpoint)
+        self.assertEqual(http.status_code, 200)
+        self.assertIn('Pipelines', parsed)
+
+    def test_list_presets(self):
+        operation = self.service.get_operation('ListPresets')
+        http, parsed = operation.call(self.endpoint, ascending='true')
+        self.assertEqual(http.status_code, 200)
+        self.assertIn('Presets', parsed)
+
+    def test_create_pipeline(self):
+        # In order to create a pipeline, we need to create 2 s3 buckets
+        # and 1 iam role.
+        input_bucket = self.create_bucket()
+        output_bucket = self.create_bucket()
+        role = self.create_iam_role()
+        pipeline_name = 'botocore-test-create-%s' % (random.randint(1, 1000000))
+
+        operation = self.service.get_operation('CreatePipeline')
+        http, parsed = operation.call(
+            self.endpoint, input_bucket=input_bucket, output_bucket=output_bucket,
+            role=role, name=pipeline_name,
+            notifications={'Progressing': '', 'Completed': '',
+                           'Warning': '', 'Error': ''})
+        self.assertEqual(http.status_code, 201)
+        self.assertIn('Pipeline', parsed)
+
+
+if __name__ == '__main__':
+    unittest.main()

tests/unit/response_parsing/json/inputs/elastictranscoder-list-pipelines.json

@@ -0,0 +1 @@
+{"NextPageToken":null,"Pipelines":[{"Arn":"arn:aws:elastictranscoder:us-west-2:12345:pipeline/12345","ContentConfig":{"Bucket":"pipeline-12345","Permissions":[],"StorageClass":"Standard"},"Id":"12345","InputBucket":"12345","Name":"test-pipeline","Notifications":{"Completed":"","Error":"","Progressing":"","Warning":""},"OutputBucket":null,"Role":"arn:aws:iam::12345:role/Elastic_Transcoder_Default_Role","Status":"Active","ThumbnailConfig":{"Bucket":"12345","Permissions":[],"StorageClass":"Standard"}}]}

tests/unit/response_parsing/json/outputs/elastictranscoder-list-pipelines.json

@@ -0,0 +1,30 @@
+{
+    "NextPageToken": null,
+    "Pipelines": [
+        {
+            "ContentConfig": {
+                "Bucket": "pipeline-12345",
+                "StorageClass": "Standard",
+                "Permissions": []
+            },
+            "Status": "Active",
+            "Name": "test-pipeline",
+            "ThumbnailConfig": {
+                "Bucket": "12345",
+                "StorageClass": "Standard",
+                "Permissions": []
+            },
+            "Notifications": {
+                "Completed": "",
+                "Warning": "",
+                "Progressing": "",
+                "Error": ""
+            },
+            "Role": "arn:aws:iam::12345:role/Elastic_Transcoder_Default_Role",
+            "InputBucket": "12345",
+            "OutputBucket": null,
+            "Id": "12345",
+            "Arn": "arn:aws:elastictranscoder:us-west-2:12345:pipeline/12345"
+        }
+    ]
+}