[2/2] Allow homeservers to send registration emails | Accepting the v…

…erification (#5940)

Fixes: #5751
Fixes: #5928

#5835 allowed Synapse to send registration emails to the user. Now we need to accept them and have it succeed the `m.login.email.identity` registration flow step.

`account_threepid_handler` will also be switched from a `str` in the config file to a dictionary which contains entries for `msisdn` and `email`, each with their own `str`. This will let people use an external server to handle `msisdn` registration and password reset requests, while using Synapse for email-based things.

And the `password_servlet` hack that was introduced in https://github.com/matrix-org/synapse/pull/5377/files#diff-b8464485d36f6f87caee3f4d82524213R189 to distinguish a registration call from a password reset call will be removed.

changelog.d/5940.feature

@@ -0,0 +1 @@
+Add the ability to send registration emails from the homeserver rather than delegating to an identity server.

synapse/handlers/account_validity.py

@@ -38,6 +38,7 @@
 class AccountValidityHandler(object):
     def __init__(self, hs):
         self.hs = hs
+        self.config = hs.config
         self.store = self.hs.get_datastore()
         self.sendmail = self.hs.get_sendmail()
         self.clock = self.hs.get_clock()
@@ -62,9 +63,14 @@ def __init__(self, hs):
             self._raw_from = email.utils.parseaddr(self._from_string)[1]
 
             self._template_html, self._template_text = load_jinja2_templates(
-                config=self.hs.config,
-                template_html_name=self.hs.config.email_expiry_template_html,
-                template_text_name=self.hs.config.email_expiry_template_text,
+                self.config.email_template_dir,
+                [
+                    self.config.email_expiry_template_html,
+                    self.config.email_expiry_template_text,
+                ],
+                apply_format_ts_filter=True,
+                apply_mxc_to_http_filter=True,
+                public_baseurl=self.config.public_baseurl,
             )
 
             # Check the renewal emails to send and send them every 30min.

synapse/handlers/auth.py

@@ -159,7 +159,7 @@ def validate_user_via_ui_auth(self, requester, request_body, clientip):
         return params
 
     @defer.inlineCallbacks
-    def check_auth(self, flows, clientdict, clientip, password_servlet=False):
+    def check_auth(self, flows, clientdict, clientip):
         """
         Takes a dictionary sent by the client in the login / registration
         protocol and handles the User-Interactive Auth flow.
@@ -183,16 +183,6 @@ def check_auth(self, flows, clientdict, clientip, password_servlet=False):
 
             clientip (str): The IP address of the client.
 
-            password_servlet (bool): Whether the request originated from
-                PasswordRestServlet.
-                XXX: This is a temporary hack to distinguish between checking
-                for threepid validations locally (in the case of password
-                resets) and using the identity server (in the case of binding
-                a 3PID during registration). Once we start using the
-                homeserver for both tasks, this distinction will no longer be
-                necessary.
-
-
         Returns:
             defer.Deferred[dict, dict, str]: a deferred tuple of
                 (creds, params, session_id).
@@ -248,9 +238,7 @@ def check_auth(self, flows, clientdict, clientip, password_servlet=False):
         if "type" in authdict:
             login_type = authdict["type"]
             try:
-                result = yield self._check_auth_dict(
-                    authdict, clientip, password_servlet=password_servlet
-                )
+                result = yield self._check_auth_dict(authdict, clientip)
                 if result:
                     creds[login_type] = result
                     self._save_session(session)
@@ -357,7 +345,7 @@ def get_session_data(self, session_id, key, default=None):
         return sess.setdefault("serverdict", {}).get(key, default)
 
     @defer.inlineCallbacks
-    def _check_auth_dict(self, authdict, clientip, password_servlet=False):
+    def _check_auth_dict(self, authdict, clientip):
         """Attempt to validate the auth dict provided by a client
 
         Args:
@@ -375,11 +363,7 @@ def _check_auth_dict(self, authdict, clientip, password_servlet=False):
         login_type = authdict["type"]
         checker = self.checkers.get(login_type)
         if checker is not None:
-            # XXX: Temporary workaround for having Synapse handle password resets
-            # See AuthHandler.check_auth for further details
-            res = yield checker(
-                authdict, clientip=clientip, password_servlet=password_servlet
-            )
+            res = yield checker(authdict, clientip=clientip)
             return res
 
         # build a v1-login-style dict out of the authdict and fall back to the
@@ -450,7 +434,7 @@ def _check_terms_auth(self, authdict, **kwargs):
         return defer.succeed(True)
 
     @defer.inlineCallbacks
-    def _check_threepid(self, medium, authdict, password_servlet=False, **kwargs):
+    def _check_threepid(self, medium, authdict, **kwargs):
         if "threepid_creds" not in authdict:
             raise LoginError(400, "Missing threepid_creds", Codes.MISSING_PARAM)
 
@@ -459,10 +443,7 @@ def _check_threepid(self, medium, authdict, password_servlet=False, **kwargs):
         identity_handler = self.hs.get_handlers().identity_handler
 
         logger.info("Getting validated threepid. threepidcreds: %r", (threepid_creds,))
-        if (
-            not password_servlet
-            or self.hs.config.threepid_behaviour_email == ThreepidBehaviour.REMOTE
-        ):
+        if self.hs.config.threepid_behaviour_email == ThreepidBehaviour.REMOTE:
             threepid = yield identity_handler.threepid_from_creds(threepid_creds)
         elif self.hs.config.threepid_behaviour_email == ThreepidBehaviour.LOCAL:
             row = yield self.store.get_threepid_validation_session(

synapse/push/mailer.py

@@ -629,25 +629,50 @@ def format_ts_filter(value, format):
     return time.strftime(format, time.localtime(value / 1000))
 
 
-def load_jinja2_templates(config, template_html_name, template_text_name):
-    """Load the jinja2 email templates from disk
+def load_jinja2_templates(
+    template_dir,
+    template_filenames,
+    apply_format_ts_filter=False,
+    apply_mxc_to_http_filter=False,
+    public_baseurl=None,
+):
+    """Loads and returns one or more jinja2 templates and applies optional filters
+
+    Args:
+        template_dir (str): The directory where templates are stored
+        template_filenames (list[str]): A list of template filenames
+        apply_format_ts_filter (bool): Whether to apply a template filter that formats
+            timestamps
+        apply_mxc_to_http_filter (bool): Whether to apply a template filter that converts
+            mxc urls to http urls
+        public_baseurl (str|None): The public baseurl of the server. Required for
+            apply_mxc_to_http_filter to be enabled
 
     Returns:
-        (template_html, template_text)
+        A list of jinja2 templates corresponding to the given list of filenames,
+        with order preserved
     """
-    logger.info("loading email templates from '%s'", config.email_template_dir)
-    loader = jinja2.FileSystemLoader(config.email_template_dir)
+    logger.info(
+        "loading email templates %s from '%s'", template_filenames, template_dir
+    )
+    loader = jinja2.FileSystemLoader(template_dir)
     env = jinja2.Environment(loader=loader)
-    env.filters["format_ts"] = format_ts_filter
-    env.filters["mxc_to_http"] = _create_mxc_to_http_filter(config)
 
-    template_html = env.get_template(template_html_name)
-    template_text = env.get_template(template_text_name)
+    if apply_format_ts_filter:
+        env.filters["format_ts"] = format_ts_filter
+
+    if apply_mxc_to_http_filter and public_baseurl:
+        env.filters["mxc_to_http"] = _create_mxc_to_http_filter(public_baseurl)
+
+    templates = []
+    for template_filename in template_filenames:
+        template = env.get_template(template_filename)
+        templates.append(template)
 
-    return template_html, template_text
+    return templates
 
 
-def _create_mxc_to_http_filter(config):
+def _create_mxc_to_http_filter(public_baseurl):
     def mxc_to_http_filter(value, width, height, resize_method="crop"):
         if value[0:6] != "mxc://":
             return ""
@@ -660,7 +685,7 @@ def mxc_to_http_filter(value, width, height, resize_method="crop"):
 
         params = {"width": width, "height": height, "method": resize_method}
         return "%s_matrix/media/v1/thumbnail/%s?%s%s" % (
-            config.public_baseurl,
+            public_baseurl,
             serverAndMediaId,
             urllib.parse.urlencode(params),
             fragment or "",

synapse/push/pusher.py

@@ -35,19 +35,24 @@
 class PusherFactory(object):
     def __init__(self, hs):
         self.hs = hs
+        self.config = hs.config
 
         self.pusher_types = {"http": HttpPusher}
 
         logger.info("email enable notifs: %r", hs.config.email_enable_notifs)
         if hs.config.email_enable_notifs:
             self.mailers = {}  # app_name -> Mailer
 
-            templates = load_jinja2_templates(
-                config=hs.config,
-                template_html_name=hs.config.email_notif_template_html,
-                template_text_name=hs.config.email_notif_template_text,
+            self.notif_template_html, self.notif_template_text = load_jinja2_templates(
+                self.config.email_template_dir,
+                [
+                    self.config.email_notif_template_html,
+                    self.config.email_notif_template_text,
+                ],
+                apply_format_ts_filter=True,
+                apply_mxc_to_http_filter=True,
+                public_baseurl=self.config.public_baseurl,
             )
-            self.notif_template_html, self.notif_template_text = templates
 
             self.pusher_types["email"] = self._create_email_pusher
 
@@ -78,6 +83,6 @@ def _app_name_from_pusherdict(self, pusherdict):
         if "data" in pusherdict and "brand" in pusherdict["data"]:
             app_name = pusherdict["data"]["brand"]
         else:
-            app_name = self.hs.config.email_app_name
+            app_name = self.config.email_app_name
 
         return app_name

synapse/rest/client/v2_alpha/account.py

@@ -18,8 +18,6 @@
 
 from six.moves import http_client
 
-import jinja2
-
 from twisted.internet import defer
 
 from synapse.api.constants import LoginType
@@ -32,6 +30,7 @@
     parse_json_object_from_request,
     parse_string,
 )
+from synapse.push.mailer import Mailer, load_jinja2_templates
 from synapse.util.msisdn import phone_number_to_msisdn
 from synapse.util.threepids import check_3pid_allowed
 
@@ -51,18 +50,21 @@ def __init__(self, hs):
         self.identity_handler = hs.get_handlers().identity_handler
 
         if self.config.threepid_behaviour_email == ThreepidBehaviour.LOCAL:
-            from synapse.push.mailer import Mailer, load_jinja2_templates
-
-            templates = load_jinja2_templates(
-                config=hs.config,
-                template_html_name=hs.config.email_password_reset_template_html,
-                template_text_name=hs.config.email_password_reset_template_text,
+            template_html, template_text = load_jinja2_templates(
+                self.config.email_template_dir,
+                [
+                    self.config.email_password_reset_template_html,
+                    self.config.email_password_reset_template_text,
+                ],
+                apply_format_ts_filter=True,
+                apply_mxc_to_http_filter=True,
+                public_baseurl=self.config.public_baseurl,
             )
             self.mailer = Mailer(
                 hs=self.hs,
                 app_name=self.config.email_app_name,
-                template_html=templates[0],
-                template_text=templates[1],
+                template_html=template_html,
+                template_text=template_text,
             )
 
     @defer.inlineCallbacks
@@ -215,6 +217,7 @@ def __init__(self, hs):
 
     @defer.inlineCallbacks
     def on_GET(self, request, medium):
+        # We currently only handle threepid token submissions for email
         if medium != "email":
             raise SynapseError(
                 400, "This medium is currently not supported for password resets"
@@ -255,35 +258,20 @@ def on_GET(self, request, medium):
             html = self.config.email_password_reset_template_success_html
             request.setResponseCode(200)
         except ThreepidValidationError as e:
+            request.setResponseCode(e.code)
+
             # Show a failure page with a reason
-            html = self.load_jinja2_template(
+            html_template = load_jinja2_templates(
                 self.config.email_template_dir,
-                self.config.email_password_reset_template_failure_html,
-                template_vars={"failure_reason": e.msg},
+                [self.config.email_password_reset_template_failure_html],
             )
-            request.setResponseCode(e.code)
+
+            template_vars = {"failure_reason": e.msg}
+            html = html_template.render(**template_vars)
 
         request.write(html.encode("utf-8"))
         finish_request(request)
 
-    def load_jinja2_template(self, template_dir, template_filename, template_vars):
-        """Loads a jinja2 template with variables to insert
-
-        Args:
-            template_dir (str): The directory where templates are stored
-            template_filename (str): The name of the template in the template_dir
-            template_vars (Dict): Dictionary of keys in the template
-                alongside their values to insert
-
-        Returns:
-            str containing the contents of the rendered template
-        """
-        loader = jinja2.FileSystemLoader(template_dir)
-        env = jinja2.Environment(loader=loader)
-
-        template = env.get_template(template_filename)
-        return template.render(**template_vars)
-
     @defer.inlineCallbacks
     def on_POST(self, request, medium):
         if medium != "email":
@@ -340,7 +328,6 @@ def on_POST(self, request):
                 [[LoginType.EMAIL_IDENTITY], [LoginType.MSISDN]],
                 body,
                 self.hs.get_ip_from_request(request),
-                password_servlet=True,
             )
 
             if LoginType.EMAIL_IDENTITY in result: