Refactor webhooks, add modern webhooks, and fix issues with Bitbucket (…

…#2433)

* Refactor webhooks, add modern webhooks, and fix issues with Bitbucket

* Adds new API endpoint for webhook consumption
* Adds webhook signals for future use
* Cleans up old webhook implementations
* Fixes old Bitbucket webhook implementation
* Sets new webhooks to point to API endpoint
* Removes GitHub service creation by naming the webhook 'web' -- 'readthedocs' created
  an RTD service on GitHub
* Updates Gitlab fixture data with example from API
* Drops `request.POST['payload']` from all old webhooks, not sure what it was
* Made tests actually send JSON to webhook views

* Lint fixes on API integrations

* Fix incoming payload parsing, cleanup, and add logging

readthedocs/core/signals.py

@@ -1,6 +1,7 @@
 import logging
 from urlparse import urlparse
 
+from django.dispatch import Signal
 from corsheaders import signals
 
 from readthedocs.projects.models import Project, Domain
@@ -10,6 +11,11 @@
 WHITELIST_URLS = ['/api/v2/footer_html', '/api/v2/search', '/api/v2/docsearch']
 
 
+webhook_github = Signal(providing_args=['project', 'data', 'event'])
+webhook_gitlab = Signal(providing_args=['project', 'data', 'event'])
+webhook_bitbucket = Signal(providing_args=['project', 'data', 'event'])
+
+
 def decide_if_cors(sender, request, **kwargs):
     """
     Decide whether a request should be given CORS access.

readthedocs/core/views/hooks.py

@@ -1,4 +1,5 @@
 import json
+import re
 
 from django.http import HttpResponse, HttpResponseNotFound
 from django.shortcuts import redirect
@@ -58,7 +59,7 @@ def _build_version(project, slug, already_built=()):
         return None
 
 
-def _build_branches(project, branch_list):
+def build_branches(project, branch_list):
     """
     Build the branches for a specific project.
 
@@ -105,7 +106,7 @@ def _build_url(url, projects, branches):
     all_built = {}
     all_not_building = {}
     for project in projects:
-        (built, not_building) = _build_branches(project, branches)
+        (built, not_building) = build_branches(project, branches)
         if not built:
             # Call update_imported_docs to update tag/branch info
             update_imported_docs.delay(project.versions.get(slug=LATEST).pk)
@@ -136,91 +137,142 @@ def _build_url(url, projects, branches):
 
 @csrf_exempt
 def github_build(request):
-    """A post-commit hook for github."""
+    """GitHub webhook consumer
+
+    This will search for projects matching either a stripped down HTTP or SSH
+    URL. The search is error prone, use the API v2 webhook for new webhooks.
+
+    Old webhooks may not have specified the content type to POST with, and
+    therefore can use ``application/x-www-form-urlencoded`` to pass the JSON
+    payload. More information on the API docs here:
+    https://developer.github.com/webhooks/creating/#content-type
+    """
     if request.method == 'POST':
         try:
-            # GitHub RTD integration
-            obj = json.loads(request.POST['payload'])
-        except:
-            # Generic post-commit hook
-            obj = json.loads(request.body)
-        repo_url = obj['repository']['url']
-        hacked_repo_url = repo_url.replace('http://', '').replace('https://', '')
-        ssh_url = obj['repository']['ssh_url']
-        hacked_ssh_url = ssh_url.replace('git@', '').replace('.git', '')
-        try:
-            branch = obj['ref'].replace('refs/heads/', '')
-        except KeyError:
-            response = HttpResponse('ref argument required to build branches.')
-            response.status_code = 400
-            return response
-
+            if request.META['CONTENT_TYPE'] == 'application/x-www-form-urlencoded':
+                data = json.loads(request.POST.get('payload'))
+            else:
+                data = json.loads(request.body)
+            http_url = data['repository']['url']
+            http_search_url = http_url.replace('http://', '').replace('https://', '')
+            ssh_url = data['repository']['ssh_url']
+            ssh_search_url = ssh_url.replace('git@', '').replace('.git', '')
+            branches = [data['ref'].replace('refs/heads/', '')]
+        except (ValueError, TypeError, KeyError):
+            log.error('Invalid GitHub webhook payload', exc_info=True)
+            return HttpResponse('Invalid request', status=400)
         try:
-            repo_projects = get_project_from_url(hacked_repo_url)
+            repo_projects = get_project_from_url(http_search_url)
             if repo_projects:
-                log.info("(Incoming GitHub Build) %s [%s]" % (hacked_repo_url, branch))
-            ssh_projects = get_project_from_url(hacked_ssh_url)
+                log.info(
+                    'GitHub webhook search: url=%s branches=%s',
+                    http_search_url,
+                    branches
+                )
+            ssh_projects = get_project_from_url(ssh_search_url)
             if ssh_projects:
-                log.info("(Incoming GitHub Build) %s [%s]" % (hacked_ssh_url, branch))
+                log.info(
+                    'GitHub webhook search: url=%s branches=%s',
+                    ssh_search_url,
+                    branches
+                )
             projects = repo_projects | ssh_projects
-            return _build_url(hacked_repo_url, projects, [branch])
+            return _build_url(http_search_url, projects, branches)
         except NoProjectException:
-            log.error(
-                "(Incoming GitHub Build) Repo not found:  %s" % hacked_repo_url)
-            return HttpResponseNotFound('Repo not found: %s' % hacked_repo_url)
+            log.error('Project match not found: url=%s', http_search_url)
+            return HttpResponseNotFound('Project not found')
     else:
-        return HttpResponse("You must POST to this resource.")
+        return HttpResponse('Method not allowed, POST is required', status=405)
 
 
 @csrf_exempt
 def gitlab_build(request):
-    """A post-commit hook for GitLab."""
+    """GitLab webhook consumer
+
+    Search project repository URLs using the site URL from GitLab webhook payload.
+    This search is error-prone, use the API v2 webhook view for new webhooks.
+    """
     if request.method == 'POST':
         try:
-            # GitLab RTD integration
-            obj = json.loads(request.POST['payload'])
-        except:
-            # Generic post-commit hook
-            obj = json.loads(request.body)
-        url = obj['repository']['homepage']
-        ghetto_url = url.replace('http://', '').replace('https://', '')
-        branch = obj['ref'].replace('refs/heads/', '')
-        log.info("(Incoming GitLab Build) %s [%s]" % (ghetto_url, branch))
-        projects = get_project_from_url(ghetto_url)
+            data = json.loads(request.body)
+            url = data['project']['http_url']
+            search_url = re.sub(r'^https?://(.*?)(?:\.git|)$', '\\1', url)
+            branches = [data['ref'].replace('refs/heads/', '')]
+        except (ValueError, TypeError, KeyError):
+            log.error('Invalid GitLab webhook payload', exc_info=True)
+            return HttpResponse('Invalid request', status=400)
+        log.info(
+            'GitLab webhook search: url=%s branches=%s',
+            search_url,
+            branches
+        )
+        projects = get_project_from_url(search_url)
         if projects:
-            return _build_url(ghetto_url, projects, [branch])
+            return _build_url(search_url, projects, branches)
         else:
-            log.error(
-                "(Incoming GitLab Build) Repo not found:  %s" % ghetto_url)
-            return HttpResponseNotFound('Repo not found: %s' % ghetto_url)
+            log.error('Project match not found: url=%s', search_url)
+            return HttpResponseNotFound('Project match not found')
     else:
-        return HttpResponse("You must POST to this resource.")
+        return HttpResponse('Method not allowed, POST is required', status=405)
 
 
 @csrf_exempt
 def bitbucket_build(request):
+    """Consume webhooks from multiple versions of Bitbucket's API
+
+    New webhooks are set up with v2, but v1 webhooks will still point to this
+    endpoint. There are also "services" that point here and submit
+    ``application/x-www-form-urlencoded`` data.
+
+    API v1
+        https://confluence.atlassian.com/bitbucket/events-resources-296095220.html
+
+    API v2
+        https://confluence.atlassian.com/bitbucket/event-payloads-740262817.html#EventPayloads-Push
+
+    Services
+        https://confluence.atlassian.com/bitbucket/post-service-management-223216518.html
+    """
     if request.method == 'POST':
-        payload = request.POST.get('payload')
-        log.info("(Incoming Bitbucket Build) Raw: %s" % payload)
-        if not payload:
-            return HttpResponseNotFound('Invalid Request')
-        obj = json.loads(payload)
-        rep = obj['repository']
-        branches = [rec.get('branch', '') for rec in obj['commits']]
-        ghetto_url = "%s%s" % (
-            "bitbucket.org", rep['absolute_url'].rstrip('/'))
-        log.info("(Incoming Bitbucket Build) %s [%s]" % (
-            ghetto_url, ' '.join(branches)))
-        log.info("(Incoming Bitbucket Build) JSON: \n\n%s\n\n" % obj)
-        projects = get_project_from_url(ghetto_url)
+        try:
+            if request.META['CONTENT_TYPE'] == 'application/x-www-form-urlencoded':
+                data = json.loads(request.POST.get('payload'))
+            else:
+                data = json.loads(request.body)
+
+            version = 2 if request.META.get('HTTP_USER_AGENT') == 'Bitbucket-Webhooks/2.0' else 1
+            if version == 1:
+                branches = [commit.get('branch', '')
+                            for commit in data['commits']]
+                repository = data['repository']
+                search_url = 'bitbucket.org{0}'.format(
+                    repository['absolute_url'].rstrip('/')
+                )
+            elif version == 2:
+                changes = data['push']['changes']
+                branches = [change['new']['name']
+                            for change in changes]
+                search_url = 'bitbucket.org/{0}'.format(
+                    data['repository']['full_name']
+                )
+        except (TypeError, ValueError, KeyError):
+            log.error('Invalid Bitbucket webhook payload', exc_info=True)
+            return HttpResponse('Invalid request', status=400)
+
+        log.info(
+            'Bitbucket webhook search: url=%s branches=%s',
+            search_url,
+            branches
+        )
+        log.debug('Bitbucket webhook payload:\n\n%s\n\n', data)
+        projects = get_project_from_url(search_url)
         if projects:
-            return _build_url(ghetto_url, projects, branches)
+            return _build_url(search_url, projects, branches)
         else:
-            log.error(
-                "(Incoming Bitbucket Build) Repo not found:  %s" % ghetto_url)
-            return HttpResponseNotFound('Repo not found: %s' % ghetto_url)
+            log.error('Project match not found: url=%s', search_url)
+            return HttpResponseNotFound('Project match not found')
     else:
-        return HttpResponse("You must POST to this resource.")
+        return HttpResponse('Method not allowed, POST is required', status=405)
 
 
 @csrf_exempt

readthedocs/oauth/services/bitbucket.py

@@ -5,6 +5,7 @@
 import re
 
 from django.conf import settings
+from django.core.urlresolvers import reverse
 from requests.exceptions import RequestException
 from allauth.socialaccount.providers.bitbucket_oauth2.views import (
     BitbucketOAuth2Adapter)
@@ -185,7 +186,13 @@ def setup_webhook(self, project):
         owner, repo = build_utils.get_bitbucket_username_repo(url=project.repo)
         data = json.dumps({
             'description': 'Read the Docs ({domain})'.format(domain=settings.PRODUCTION_DOMAIN),
-            'url': 'https://{domain}/bitbucket'.format(domain=settings.PRODUCTION_DOMAIN),
+            'url': 'https://{domain}{path}'.format(
+                domain=settings.PRODUCTION_DOMAIN,
+                path=reverse(
+                    'api_webhook_bitbucket',
+                    kwargs={'project_slug': project.slug}
+                )
+            ),
             'active': True,
             'events': ['repo:push'],
         })

readthedocs/oauth/services/github.py

@@ -5,6 +5,7 @@
 import re
 
 from django.conf import settings
+from django.core.urlresolvers import reverse
 from requests.exceptions import RequestException
 from allauth.socialaccount.models import SocialToken
 from allauth.socialaccount.providers.github.views import GitHubOAuth2Adapter
@@ -169,9 +170,19 @@ def setup_webhook(self, project):
         session = self.get_session()
         owner, repo = build_utils.get_github_username_repo(url=project.repo)
         data = json.dumps({
-            'name': 'readthedocs',
+            'name': 'web',
             'active': True,
-            'config': {'url': 'https://{domain}/github'.format(domain=settings.PRODUCTION_DOMAIN)}
+            'config': {
+                'url': 'https://{domain}{path}'.format(
+                    domain=settings.PRODUCTION_DOMAIN,
+                    path=reverse(
+                        'api_webhook_github',
+                        kwargs={'project_slug': project.slug}
+                    )
+                ),
+                'content_type': 'json',
+            },
+            'events': ['push', 'pull_request'],
         })
         resp = None
         try:
@@ -213,5 +224,5 @@ def get_token_for_project(cls, project, force_local=False):
                     if tokens.exists():
                         token = tokens[0].token
         except Exception:
-            log.error('Failed to get token for user', exc_info=True)
+            log.error('Failed to get token for project', exc_info=True)
         return token

readthedocs/restapi/urls.py

@@ -2,16 +2,18 @@
 
 from rest_framework import routers
 
+from readthedocs.constants import pattern_opts
+from readthedocs.comments.views import CommentViewSet
+from readthedocs.restapi import views
+from readthedocs.restapi.views import (
+    core_views, footer_views, search_views, task_views, integrations
+)
+
 from .views.model_views import (BuildViewSet, BuildCommandViewSet,
                                 ProjectViewSet, NotificationViewSet,
                                 VersionViewSet, DomainViewSet,
                                 RemoteOrganizationViewSet,
                                 RemoteRepositoryViewSet)
-from readthedocs.comments.views import CommentViewSet
-from readthedocs.restapi import views
-from readthedocs.restapi.views import (
-    core_views, footer_views, search_views, task_views,
-)
 
 router = routers.DefaultRouter()
 router.register(r'build', BuildViewSet)
@@ -57,10 +59,22 @@
         name='api_sync_remote_repositories'),
 ]
 
+integration_urls = [
+    url(r'webhook/github/(?P<project_slug>{project_slug})/'.format(**pattern_opts),
+        integrations.GitHubWebhookView.as_view(),
+        name='api_webhook_github'),
+    url(r'webhook/gitlab/(?P<project_slug>{project_slug})/'.format(**pattern_opts),
+        integrations.GitLabWebhookView.as_view(),
+        name='api_webhook_gitlab'),
+    url(r'webhook/bitbucket/(?P<project_slug>{project_slug})/'.format(**pattern_opts),
+        integrations.BitbucketWebhookView.as_view(),
+        name='api_webhook_bitbucket'),
+]
 
 urlpatterns += function_urls
 urlpatterns += search_urls
 urlpatterns += task_urls
+urlpatterns += integration_urls
 
 try:
     from readthedocsext.search.docsearch import DocSearch