Initial stub of proxito

readthedocs/proxito/middleware.py

@@ -0,0 +1,128 @@
+"""
+Middleware for Proxito.
+
+This is used to take the request and map the host to the proper project slug.
+
+Additional processing is done to get the project from the URL in the ``views.py`` as well.
+"""
+import logging
+
+from django.conf import settings
+from django.core.exceptions import MiddlewareNotUsed
+from django.http import HttpResponseBadRequest
+from django.shortcuts import render
+from django.utils.deprecation import MiddlewareMixin
+from django.utils.translation import ugettext_lazy as _
+
+from readthedocs.projects.models import Domain
+
+
+log = logging.getLogger(__name__)  # noqa
+
+
+def map_host_to_project_slug(request):
+    """
+    Take the request and map the host to the proper project slug.
+
+    We check, in order:
+
+    * The ``HTTP_X_RTD_SLUG`` host header for explicit Project mapping
+        - This sets ``request.rtdheader`` True
+    * The ``PUBLIC_DOMAIN`` where we can use the subdomain as the project name
+        - This sets ``request.subdomain`` True
+    * The hostname without port information, which maps to ``Domain`` objects
+        - This sets ``request.cname`` True
+    """
+
+    host = request.get_host().lower().split(':')[0]
+    public_domain = settings.PUBLIC_DOMAIN.lower().split(':')[0]
+    host_parts = host.split('.')
+    public_domain_parts = public_domain.split('.')
+
+    # Explicit Project slug being passed in
+    if 'HTTP_X_RTD_SLUG' in request.META:
+        project_slug = request.META['HTTP_X_RTD_SLUG'].lower()
+        request.rtdheader = True
+
+    elif public_domain in host:
+        # Serve from the PUBLIC_DOMAIN, ensuring it looks like `foo.PUBLIC_DOMAIN`
+        if public_domain_parts == host_parts[1:]:
+            project_slug = host_parts[0]
+            request.subdomain = True
+            log.debug('Proxito Public Domain: host=%s', host)
+        else:
+            # TODO: This can catch some possibly valid domains (docs.readthedocs.io.com) for example
+            # But these feel like they might be phishing, etc. so let's block them for now.
+            project_slug = None
+            log.warning('Weird variation on our hostname: %s', host)
+            return HttpResponseBadRequest(_('Invalid hostname'))
+
+    # Serve CNAMEs
+    else:
+        domain_qs = Domain.objects.filter(domain=host
+                                          ).prefetch_related('project')
+        if domain_qs.exists():
+            project_slug = domain_qs.first().project.slug
+            request.cname = True
+            log.debug('Proxito CNAME: %s', host)
+        else:
+            # Some person is CNAMEing to us without configuring a domain - 404.
+            project_slug = None
+            log.debug('CNAME 404: %s', host)
+            return render(
+                request, 'core/dns-404.html', context={'host': host}, status=404
+            )
+    log.debug('Proxito Project: %s', project_slug)
+    return project_slug
+
+
+class ProxitoMiddleware(MiddlewareMixin):
+
+    """The actual middleware we'll be using in prod."""
+
+    def process_request(self, request):  # noqa
+        if any([not settings.USE_SUBDOMAIN, 'localhost' in request.get_host(),
+                'testserver' in request.get_host()]):
+            log.debug('Not processing Proxito middleware')
+            return None
+
+        ret = map_host_to_project_slug(request)
+
+        # Handle returning a response
+        if hasattr(ret, 'status_code'):
+            return ret
+
+        # Otherwise set the slug on the request
+        request.host_project_slug = request.slug = ret
+
+        return None
+
+
+class NewStyleProxitoMiddleware:
+
+    """The new style middleware, I can't figure out how to test it."""
+
+    def __init__(self, get_response):
+        self.get_response = get_response
+        if not settings.USE_SUBDOMAIN:
+            raise MiddlewareNotUsed('USE_SUBDOMAIN setting is not on')
+
+    def __call__(self, request):
+        # For local dev to hit the main site
+        if 'localhost' in request.get_host() or 'testserver' in request.get_host():
+            return self.get_response(request)
+
+        # Code to be executed for each request before
+        # the view (and later middleware) are called.
+
+        host_project = map_host_to_project_slug(request)
+        request.host_project_slug = host_project
+        request.slug = host_project
+        # request.urlconf = 'readthedocs.proxito.urls'
+
+        response = self.get_response(request)
+
+        # Code to be executed for each request/response after
+        # the view is called.
+
+        return response

readthedocs/proxito/tests/base.py

@@ -0,0 +1,49 @@
+# Copied from .org
+
+
+import django_dynamic_fixture as fixture
+from django.contrib.auth.models import User
+from django.test import TestCase
+from django.test.utils import override_settings
+
+from readthedocs.projects.models import Project
+
+
+@override_settings(
+    PUBLIC_DOMAIN='dev.readthedocs.io',
+    ROOT_URLCONF='readthedocs.proxito.urls',
+    MIDDLEWARE=['readthedocs.proxito.middleware.ProxitoMiddleware'],
+    USE_SUBDOMAIN=True,
+)
+class BaseDocServing(TestCase):
+
+    def setUp(self):
+        self.eric = fixture.get(User, username='eric')
+        self.eric.set_password('eric')
+        self.eric.save()
+        self.private = fixture.get(
+            Project, slug='private', privacy_level='private',
+            version_privacy_level='private', users=[self.eric],
+            main_language_project=None,
+        )
+        self.subproject = fixture.get(
+            Project,
+            slug='subproject',
+            users=[self.eric],
+            main_language_project=None,
+        )
+        self.private.add_subproject(self.subproject)
+        self.translation = fixture.get(
+            Project,
+            language='es',
+            slug='translation',
+            users=[self.eric],
+            main_language_project=self.private,
+        )
+        self.subproject_translation = fixture.get(
+            Project,
+            language='es',
+            slug='subproject-translation',
+            users=[self.eric],
+            main_language_project=self.subproject,
+        )

readthedocs/proxito/tests/test_full.py

@@ -0,0 +1,131 @@
+# Copied from .org
+
+import os
+
+import mock
+from django.conf import settings
+from django.http import HttpResponse
+from django.test.utils import override_settings
+
+from .base import BaseDocServing
+
+
+@override_settings(PYTHON_MEDIA=False)
+class TestFullDocServing(BaseDocServing):
+    # Test the full range of possible doc URL's
+
+    def test_subproject_serving(self):
+        url = '/projects/subproject/en/latest/awesome.html'
+        host = 'private.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/subproject/latest/awesome.html',
+        )
+
+    def test_subproject_single_version(self):
+        self.subproject.single_version = True
+        self.subproject.save()
+        url = '/projects/subproject/awesome.html'
+        host = 'private.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/subproject/latest/awesome.html',
+        )
+
+    def test_subproject_translation_serving(self):
+        url = '/projects/subproject/es/latest/awesome.html'
+        host = 'private.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/subproject-translation/latest/awesome.html',
+        )
+
+    def test_translation_serving(self):
+        url = '/es/latest/awesome.html'
+        host = 'private.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/translation/latest/awesome.html',
+        )
+
+    def test_normal_serving(self):
+        url = '/en/latest/awesome.html'
+        host = 'private.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/private/latest/awesome.html',
+        )
+
+    def test_single_version_serving(self):
+        self.private.single_version = True
+        self.private.save()
+        url = '/awesome.html'
+        host = 'private.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/private/latest/awesome.html',
+        )
+
+    def test_single_version_serving_looks_like_normal(self):
+        self.private.single_version = True
+        self.private.save()
+        url = '/en/stable/awesome.html'
+        host = 'private.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/private/latest/en/stable/awesome.html',
+        )
+
+        # Invalid tests
+
+    def test_invalid_url_for_project_with_versions(self):
+        url = '/cn/latest/awesome.html'
+        host = 'private.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(resp.status_code, 404)
+
+    def test_invalid_translation(self):
+        url = '/cs/latest/awesome.html'
+        host = 'private.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(resp.status_code, 404)
+
+    def test_invalid_subproject(self):
+        url = '/projects/doesnt-exist/foo.html'
+        host = 'private.dev.readthedocs.io'
+        resp = self.client.get(url, HTTP_HOST=host)
+        self.assertEqual(resp.status_code, 404)
+
+
+class TestDocServingBackends(BaseDocServing):
+    # Test that nginx and python backends both work
+
+    @override_settings(PYTHON_MEDIA=True)
+    def test_python_media_serving(self):
+        with mock.patch(
+                'readthedocs.proxito.views.serve', return_value=HttpResponse()) as serve_mock:
+            url = '/en/latest/awesome.html'
+            host = 'private.dev.readthedocs.io'
+            self.client.get(url, HTTP_HOST=host)
+            serve_mock.assert_called_with(
+                mock.ANY,
+                'html/private/latest/awesome.html',
+                os.path.join(settings.SITE_ROOT, 'media'),
+            )
+
+    @override_settings(PYTHON_MEDIA=False)
+    def test_nginx_media_serving(self):
+        resp = self.client.get('/en/latest/awesome.html', HTTP_HOST='private.dev.readthedocs.io')
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(
+            resp['x-accel-redirect'], '/proxito/html/private/latest/awesome.html',
+        )
+
+    @override_settings(PYTHON_MEDIA=False)
+    def test_private_nginx_serving_unicode_filename(self):
+        resp = self.client.get('/en/latest/úñíčódé.html', HTTP_HOST='private.dev.readthedocs.io')
+        self.assertEqual(resp.status_code, 200)
+        self.assertEqual(
+            resp['x-accel-redirect'],
+            '/proxito/html/private/latest/%C3%BA%C3%B1%C3%AD%C4%8D%C3%B3d%C3%A9.html',
+        )