Accept JSON on all client-facing endpoints

In addition to x-form-www-urlencoded, based on what content type
the client sends. This will allow us to migrate to JSON to be
consistent with the client/server API.

Also remove support for camelcase parameters like sendAttempt and
clientSecret as all clients moved over to using the underscored
versions long ago so there should be no clients sending camelcase
in the wild by now.

sydent/http/servlets/__init__.py

@@ -15,18 +15,50 @@
 # limitations under the License.
 
 import json
+import copy
 
 
-def require_args(request, rqArgs):
+def get_args(request, required_args):
+    """
+    Helper function to get arguments for an HTTP request
+    Currently takes args from the top level keys of a json object or
+    www-form-urlencoded for backwards compatability.
+    Returns a tuple (error, args) where if error is non-null,
+    the requesat is malformed. Otherwise, args contains the
+    parameters passed.
+    """
+    args = None
+    if (
+        request.requestHeaders.hasHeader('Content-Type') and
+        request.requestHeaders.getRawHeaders('Content-Type')[0] == 'application/json'
+    ):
+        try:
+            args = json.load(request.content)
+        except ValueError:
+            request.setResponseCode(400)
+            return {'errcode': 'M_BAD_JSON', 'error': 'Malformed JSON'}, None
+    else:
+        args = copy.copy(request.args)
+        # Twisted supplies everything as an array because it's valid to
+        # supply the same params multiple times with www-form-urlencoded
+        # params. This make it incompatible with the json object though,
+        # so we need to convert one of them. Since this is the
+        # backwards-compat option, we convert this one.
+        for k, v in args.items():
+            if isinstance(v, list) and len(v) == 1:
+                args[k] = v[0]
+
     missing = []
-    for a in rqArgs:
-        if a not in request.args:
+    for a in required_args:
+        if a not in args:
             missing.append(a)
 
     if len(missing) > 0:
         request.setResponseCode(400)
         msg = "Missing parameters: "+(",".join(missing))
-        return {'errcode': 'M_MISSING_PARAMS', 'error': msg}
+        return {'errcode': 'M_MISSING_PARAMS', 'error': msg}, None
+
+    return None, args
 
 def jsonwrap(f):
     def inner(*args, **kwargs):
@@ -39,4 +71,4 @@ def send_cors(request):
     request.setHeader("Access-Control-Allow-Methods",
                       "GET, POST, PUT, DELETE, OPTIONS")
     request.setHeader("Access-Control-Allow-Headers",
-                      "Origin, X-Requested-With, Content-Type, Accept")
+                      "Origin, X-Requested-With, Content-Type, Accept")

sydent/http/servlets/blindlysignstuffservlet.py

@@ -19,7 +19,7 @@
 import signedjson.key
 import signedjson.sign
 from sydent.db.invite_tokens import JoinTokenStore
-from sydent.http.servlets import require_args, jsonwrap, send_cors
+from sydent.http.servlets import get_args, jsonwrap, send_cors
 
 
 class BlindlySignStuffServlet(Resource):
@@ -31,13 +31,13 @@ def __init__(self, syd):
 
     def render_POST(self, request):
         send_cors(request)
-        err = require_args(request, ("private_key", "token", "mxid"))
+        err, args = get_args(request, ("private_key", "token", "mxid"))
         if err:
             return json.dumps(err)
 
-        private_key_base64 = request.args['private_key'][0]
-        token = request.args['token'][0]
-        mxid = request.args['mxid'][0]
+        private_key_base64 = args['private_key']
+        token = args['token']
+        mxid = args['mxid']
 
         sender = self.tokenStore.getSenderForToken(token)
         if sender is None:

sydent/http/servlets/emailservlet.py

@@ -20,7 +20,7 @@
 from sydent.validators.emailvalidator import SessionExpiredException
 from sydent.validators.emailvalidator import IncorrectClientSecretException
 
-from sydent.http.servlets import require_args, jsonwrap, send_cors
+from sydent.http.servlets import get_args, jsonwrap, send_cors
 
 
 class EmailRequestCodeServlet(Resource):
@@ -33,36 +33,20 @@ def __init__(self, syd):
     def render_POST(self, request):
         send_cors(request)
 
-        # error = require_args(request, ('email', 'client_secret', 'send_attempt'))
-        error = require_args(request, ('email',))
+        error, args = get_args(request, ('email', 'client_secret', 'send_attempt'))
         if error:
             request.setResponseCode(400)
             return error
 
-        # look for both camelcase and underscores for transition
-        if 'client_secret' in request.args:
-            clientSecret = request.args['client_secret'][0]
-        elif 'clientSecret' in request.args:
-            clientSecret = request.args['clientSecret'][0]
-        else:
-            request.setResponseCode(400)
-            return {'errcode': 'M_MISSING_PARAM', 'error':'No client_secret'}
-
-        if 'send_attempt' in request.args:
-            sendAttempt = request.args['send_attempt'][0]
-        elif 'sendAttempt' in request.args:
-            sendAttempt = request.args['sendAttempt'][0]
-        else:
-            request.setResponseCode(400)
-            return {'errcode': 'M_MISSING_PARAM', 'error':'No send_attempt'}
-
-        email = request.args['email'][0]
+        email = args['email']
+        clientSecret = args['client_secret']
+        sendAttempt = args['send_attempt']
 
         ipaddress = self.sydent.ip_from_request(request)
 
         nextLink = None
-        if 'next_link' in request.args:
-            nextLink = request.args['next_link'][0]
+        if 'next_link' in args:
+            nextLink = args['next_link']
 
         resp = None
 

sydent/http/servlets/getvalidated3pidservlet.py

@@ -18,7 +18,7 @@
 
 from twisted.web.resource import Resource
 
-from sydent.http.servlets import jsonwrap, require_args
+from sydent.http.servlets import jsonwrap, get_args
 from sydent.db.valsession import ThreePidValSessionStore
 from sydent.validators import SessionExpiredException, IncorrectClientSecretException, InvalidSessionIdException,\
     SessionNotValidatedException
@@ -31,21 +31,12 @@ def __init__(self, syd):
 
     @jsonwrap
     def render_GET(self, request):
-        # err = require_args(request, ('sid', 'client_secret'))
-        err = require_args(request, ('sid',))
+        err, args = require_args(request, ('sid', 'client_secret'))
         if err:
             return err
 
-        sid = request.args['sid'][0]
-        #clientSecret = request.args['client_secret'][0]
-
-        if 'client_secret' in request.args:
-            clientSecret = request.args['client_secret'][0]
-        elif 'clientSecret' in request.args:
-            clientSecret = request.args['clientSecret'][0]
-        else:
-            request.setResponseCode(400)
-            return {'errcode': 'M_MISSING_PARAM', 'error':'No client_secret'}
+        sid = args['sid']
+        clientSecret = args['client_secret']
 
         valSessionStore = ThreePidValSessionStore(self.sydent)
 
@@ -65,4 +56,4 @@ def render_GET(self, request):
             return {'errcode': 'M_SESSION_NOT_VALIDATED',
                     'error': "This validation session has not yet been completed"}
 
-        return { 'medium': s.medium, 'address': s.address, 'validated_at': s.mtime }
+        return { 'medium': s.medium, 'address': s.address, 'validated_at': s.mtime }

sydent/http/servlets/lookupservlet.py

@@ -20,7 +20,7 @@
 import json
 import signedjson.sign
 
-from sydent.http.servlets import require_args, jsonwrap, send_cors
+from sydent.http.servlets import get_args, jsonwrap, send_cors
 
 class LookupServlet(Resource):
     isLeaf = True
@@ -30,12 +30,12 @@ def __init__(self, syd):
 
     def render_GET(self, request):
         send_cors(request)
-        err = require_args(request, ('medium', 'address'))
+        err, args = get_args(request, ('medium', 'address'))
         if err:
             return err
 
-        medium = request.args['medium'][0]
-        address = request.args['address'][0]
+        medium = args['medium']
+        address = args['address']
 
         globalAssocStore = GlobalAssociationStore(self.sydent)
 
@@ -74,4 +74,4 @@ def render_GET(self, request):
     def render_OPTIONS(self, request):
         send_cors(request)
         request.setResponseCode(200)
-        return {}
+        return {}

sydent/http/servlets/msisdnservlet.py

@@ -21,7 +21,7 @@
 from sydent.validators.msisdnvalidator import SessionExpiredException
 from sydent.validators.msisdnvalidator import IncorrectClientSecretException
 
-from sydent.http.servlets import require_args, jsonwrap, send_cors
+from sydent.http.servlets import get_args, jsonwrap, send_cors
 
 
 logger = logging.getLogger(__name__)
@@ -37,15 +37,15 @@ def __init__(self, syd):
     def render_POST(self, request):
         send_cors(request)
 
-        error = require_args(request, ('phone_number', 'country', 'client_secret', 'send_attempt'))
+        error, args = get_args(request, ('phone_number', 'country', 'client_secret', 'send_attempt'))
         if error:
             request.setResponseCode(400)
             return error
 
-        raw_phone_number = request.args['phone_number'][0]
-        country = request.args['country'][0]
-        clientSecret = request.args['client_secret'][0]
-        sendAttempt = request.args['send_attempt'][0]
+        raw_phone_number = args['phone_number']
+        country = args['country']
+        clientSecret = args['client_secret']
+        sendAttempt = args['send_attempt']
 
         try:
             phone_number_object = phonenumbers.parse(raw_phone_number, country)
@@ -88,11 +88,17 @@ def __init__(self, syd):
         self.sydent = syd
 
     def render_GET(self, request):
-        resp = self.do_validate_request(request)
+        send_cors(request)
+
+        err, args = get_args(request, ('token', 'sid', 'client_secret'))
+        if err:
+            return err
+
+        resp = self.do_validate_request(args)
         if 'success' in resp and resp['success']:
             msg = "Verification successful! Please return to your Matrix client to continue."
-            if 'nextLink' in request.args:
-                next_link = request.args['nextLink'][0]
+            if 'next_link' in args:
+                next_link = args['next_link']
                 request.setResponseCode(302)
                 request.setHeader("Location", next_link)
         else:
@@ -105,18 +111,18 @@ def render_GET(self, request):
 
     @jsonwrap
     def render_POST(self, request):
-        return self.do_validate_request(request)
-
-    def do_validate_request(self, request):
         send_cors(request)
 
-        err = require_args(request, ('token', 'sid', 'client_secret'))
+        err, args = get_args(request, ('token', 'sid', 'client_secret'))
         if err:
             return err
 
-        sid = request.args['sid'][0]
-        tokenString = request.args['token'][0]
-        clientSecret = request.args['client_secret'][0]
+        return self.do_validate_request(args)
+
+    def do_validate_request(self, args):
+        sid = args['sid']
+        tokenString = args['token']
+        clientSecret = args['client_secret']
 
         try:
             resp = self.sydent.validators.msisdn.validateSessionWithToken(sid, clientSecret, tokenString)

sydent/http/servlets/pubkeyservlets.py

@@ -18,7 +18,7 @@
 
 import json
 from sydent.db.invite_tokens import JoinTokenStore
-from sydent.http.servlets import require_args
+from sydent.http.servlets import get_args
 
 
 class Ed25519Servlet(Resource):
@@ -40,14 +40,14 @@ def __init__(self, syd):
         self.sydent = syd
 
     def render_GET(self, request):
-        err = require_args(request, ("public_key",))
+        err, args = get_args(request, ("public_key",))
         if err:
             return json.dumps(err)
 
         pubKey = self.sydent.keyring.ed25519.verify_key
         pubKeyBase64 = encode_base64(pubKey.encode())
 
-        return json.dumps({'valid': request.args["public_key"][0] == pubKeyBase64})
+        return json.dumps({'valid': args["public_key"] == pubKeyBase64})
 
 
 class EphemeralPubkeyIsValidServlet(Resource):
@@ -57,10 +57,10 @@ def __init__(self, syd):
         self.joinTokenStore = JoinTokenStore(syd)
 
     def render_GET(self, request):
-        err = require_args(request, ("public_key",))
+        err, args = get_args(request, ("public_key",))
         if err:
             return json.dumps(err)
-        publicKey = request.args["public_key"][0]
+        publicKey = args["public_key"]
 
         return json.dumps({
             'valid': self.joinTokenStore.validateEphemeralPublicKey(publicKey),

sydent/http/servlets/replication.py

@@ -16,7 +16,7 @@
 
 import twisted.python.log
 from twisted.web.resource import Resource
-from sydent.http.servlets import require_args, jsonwrap
+from sydent.http.servlets import jsonwrap
 from sydent.threepid import threePidAssocFromDict
 from sydent.db.peers import PeerStore
 from sydent.db.threepid_associations import GlobalAssociationStore

sydent/http/servlets/store_invite_servlet.py

@@ -25,7 +25,7 @@
 from sydent.db.invite_tokens import JoinTokenStore
 from sydent.db.threepid_associations import GlobalAssociationStore
 
-from sydent.http.servlets import require_args, send_cors
+from sydent.http.servlets import get_args, send_cors
 from sydent.util.emailutils import sendEmail
 
 
@@ -38,10 +38,10 @@ def render_POST(self, request):
         err = require_args(request, ("medium", "address", "room_id", "sender",))
         if err:
             return json.dumps(err)
-        medium = request.args["medium"][0]
-        address = request.args["address"][0]
-        roomId = request.args["room_id"][0]
-        sender = request.args["sender"][0]
+        medium = args["medium"]
+        address = args["address"]
+        roomId = args["room_id"]
+        sender = args["sender"]
 
         globalAssocStore = GlobalAssociationStore(self.sydent)
         mxid = globalAssocStore.getMxid(medium, address)