-
-
Notifications
You must be signed in to change notification settings - Fork 3.6k
/
serve.py
441 lines (358 loc) · 14.6 KB
/
serve.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
"""
Doc serving from Python.
In production there are two modes,
* Serving from public symlinks in nginx (readthedocs.org & readthedocs.com)
* Serving from private symlinks in Python (readthedocs.com only)
In development, we have two modes:
* Serving from public symlinks in Python
* Serving from private symlinks in Python
This means we should only serve from public symlinks in dev,
and generally default to serving from private symlinks in Python only.
Privacy
-------
These views will take into account the version privacy level.
Settings
--------
PYTHON_MEDIA (False) - Set this to True to serve docs & media from Python
SERVE_DOCS (['private']) - The list of ['private', 'public'] docs to serve.
"""
import itertools
import logging
import mimetypes
import os
from functools import wraps
from urllib.parse import urlparse
from django.conf import settings
from django.http import Http404, HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render
from django.utils.encoding import iri_to_uri
from django.views.decorators.cache import cache_page
from django.views.static import serve
from readthedocs.builds.models import Version
from readthedocs.core.permissions import AdminPermission
from readthedocs.core.resolver import resolve, resolve_path
from readthedocs.core.symlink import PrivateSymlink, PublicSymlink
from readthedocs.projects import constants
from readthedocs.projects.models import Project, ProjectRelationship
from readthedocs.projects.templatetags.projects_tags import sort_version_aware
log = logging.getLogger(__name__)
def map_subproject_slug(view_func):
"""
A decorator that maps a ``subproject_slug`` URL param into a Project.
:raises: Http404 if the Project doesn't exist
.. warning:: Does not take into account any kind of privacy settings.
"""
@wraps(view_func)
def inner_view( # noqa
request, subproject=None, subproject_slug=None, *args, **kwargs
):
if subproject is None and subproject_slug:
# Try to fetch by subproject alias first, otherwise we might end up
# redirected to an unrelated project.
try:
# Depends on a project passed into kwargs
rel = ProjectRelationship.objects.get(
parent=kwargs['project'],
alias=subproject_slug,
)
subproject = rel.child
except (ProjectRelationship.DoesNotExist, KeyError):
subproject = get_object_or_404(Project, slug=subproject_slug)
return view_func(request, subproject=subproject, *args, **kwargs)
return inner_view
def map_project_slug(view_func):
"""
A decorator that maps a ``project_slug`` URL param into a Project.
:raises: Http404 if the Project doesn't exist
.. warning:: Does not take into account any kind of privacy settings.
"""
@wraps(view_func)
def inner_view( # noqa
request, project=None, project_slug=None, *args, **kwargs
):
if project is None:
if not project_slug:
project_slug = request.slug
try:
project = Project.objects.get(slug=project_slug)
except Project.DoesNotExist:
raise Http404('Project does not exist.')
return view_func(request, project=project, *args, **kwargs)
return inner_view
@map_project_slug
@map_subproject_slug
def redirect_project_slug(request, project, subproject): # pylint: disable=unused-argument
"""Handle / -> /en/latest/ directs on subdomains."""
urlparse_result = urlparse(request.get_full_path())
return HttpResponseRedirect(
resolve(
subproject or project,
query_params=urlparse_result.query,
)
)
@map_project_slug
@map_subproject_slug
def redirect_page_with_filename(request, project, subproject, filename): # pylint: disable=unused-argument # noqa
"""Redirect /page/file.html to /en/latest/file.html."""
urlparse_result = urlparse(request.get_full_path())
return HttpResponseRedirect(
resolve(
subproject or project,
filename=filename,
query_params=urlparse_result.query,
)
)
def _serve_401(request, project):
res = render(request, '401.html')
res.status_code = 401
log.debug('Unauthorized access to %s documentation', project.slug)
return res
def _serve_file(request, filename, basepath):
"""
Serve media file via Django or NGINX based on ``PYTHON_MEDIA``.
When using ``PYTHON_MEDIA=True`` (or when ``DEBUG=True``) the file is served
by ``django.views.static.serve`` function.
On the other hand, when ``PYTHON_MEDIA=False`` the file is served by using
``X-Accel-Redirect`` header for NGINX to take care of it and serve the file.
:param request: Django HTTP request
:param filename: path to the filename to be served relative to ``basepath``
:param basepath: base path to prepend to the filename
:returns: Django HTTP response object
:raises: ``Http404`` on ``UnicodeEncodeError``
"""
# Serve the file from the proper location
if settings.DEBUG or settings.PYTHON_MEDIA:
# Serve from Python
return serve(request, filename, basepath)
# Serve from Nginx
content_type, encoding = mimetypes.guess_type(
os.path.join(basepath, filename),
)
content_type = content_type or 'application/octet-stream'
response = HttpResponse(content_type=content_type)
if encoding:
response['Content-Encoding'] = encoding
try:
iri_path = os.path.join(
basepath[len(settings.SITE_ROOT):],
filename,
)
# NGINX does not support non-ASCII characters in the header, so we
# convert the IRI path to URI so it's compatible with what NGINX expects
# as the header value.
# https://github.com/benoitc/gunicorn/issues/1448
# https://docs.djangoproject.com/en/1.11/ref/unicode/#uri-and-iri-handling
x_accel_redirect = iri_to_uri(iri_path)
response['X-Accel-Redirect'] = x_accel_redirect
except UnicodeEncodeError:
raise Http404
return response
@map_project_slug
@map_subproject_slug
def serve_docs(
request,
project,
subproject,
lang_slug=None,
version_slug=None,
filename='',
):
"""Map existing proj, lang, version, filename views to the file format."""
if not version_slug:
version_slug = project.get_default_version()
try:
version = project.versions.public(request.user).get(slug=version_slug)
except Version.DoesNotExist:
# Properly raise a 404 if the version doesn't exist (or is inactive) and
# a 401 if it does
if project.versions.filter(slug=version_slug, active=True).exists():
return _serve_401(request, project)
raise Http404('Version does not exist.')
filename = resolve_path(
subproject or project, # Resolve the subproject if it exists
version_slug=version_slug,
language=lang_slug,
filename=filename,
subdomain=True, # subdomain will make it a "full" path without a URL prefix
)
if (version.privacy_level == constants.PRIVATE and
not AdminPermission.is_member(user=request.user, obj=project)):
return _serve_401(request, project)
return _serve_symlink_docs(
request,
filename=filename,
project=project,
privacy_level=version.privacy_level,
)
@map_project_slug
def _serve_symlink_docs(request, project, privacy_level, filename=''):
"""Serve a file by symlink, or a 404 if not found."""
# Handle indexes
if filename == '' or filename[-1] == '/':
filename += 'index.html'
# This breaks path joining, by ignoring the root when given an "absolute" path
if filename[0] == '/':
filename = filename[1:]
log.info('Serving %s for %s', filename, project)
files_tried = []
if (settings.DEBUG or constants.PUBLIC in settings.SERVE_DOCS) and privacy_level != constants.PRIVATE: # yapf: disable # noqa
public_symlink = PublicSymlink(project)
basepath = public_symlink.project_root
if os.path.exists(os.path.join(basepath, filename)):
return _serve_file(request, filename, basepath)
files_tried.append(os.path.join(basepath, filename))
if (settings.DEBUG or constants.PRIVATE in settings.SERVE_DOCS) and privacy_level == constants.PRIVATE: # yapf: disable # noqa
# Handle private
private_symlink = PrivateSymlink(project)
basepath = private_symlink.project_root
if os.path.exists(os.path.join(basepath, filename)):
return _serve_file(request, filename, basepath)
files_tried.append(os.path.join(basepath, filename))
raise Http404(
'File not found. Tried these files: {}'.format(','.join(files_tried)),
)
@map_project_slug
def robots_txt(request, project):
"""
Serve custom user's defined ``/robots.txt``.
If the user added a ``robots.txt`` in the "default version" of the project,
we serve it directly.
"""
# Use the ``robots.txt`` file from the default version configured
version_slug = project.get_default_version()
version = project.versions.get(slug=version_slug)
no_serve_robots_txt = any([
# If project is private or,
project.privacy_level == constants.PRIVATE,
# default version is private or,
version.privacy_level == constants.PRIVATE,
# default version is not active or,
not version.active,
# default version is not built
not version.built,
])
if no_serve_robots_txt:
# ... we do return a 404
raise Http404()
filename = resolve_path(
project,
version_slug=version_slug,
filename='robots.txt',
subdomain=True, # subdomain will make it a "full" path without a URL prefix
)
# This breaks path joining, by ignoring the root when given an "absolute" path
if filename[0] == '/':
filename = filename[1:]
basepath = PublicSymlink(project).project_root
fullpath = os.path.join(basepath, filename)
if os.path.exists(fullpath):
return HttpResponse(open(fullpath).read(), content_type='text/plain')
sitemap_url = '{scheme}://{domain}/sitemap.xml'.format(
scheme='https',
domain=project.subdomain(),
)
return HttpResponse(
'User-agent: *\nAllow: /\nSitemap: {}\n'.format(sitemap_url),
content_type='text/plain',
)
@map_project_slug
@cache_page(60 * 60 * 24 * 3) # 3 days
def sitemap_xml(request, project):
"""
Generate and serve a ``sitemap.xml`` for a particular ``project``.
The sitemap is generated from all the ``active`` and public versions of
``project``. These versions are sorted by using semantic versioning
prepending ``latest`` and ``stable`` (if they are enabled) at the beginning.
Following this order, the versions are assigned priorities and change
frequency. Starting from 1 and decreasing by 0.1 for priorities and starting
from daily, weekly to monthly for change frequency.
If the project is private, the view raises ``Http404``. On the other hand,
if the project is public but a version is private, this one is not included
in the sitemap.
:param request: Django request object
:param project: Project instance to generate the sitemap
:returns: response with the ``sitemap.xml`` template rendered
:rtype: django.http.HttpResponse
"""
def priorities_generator():
"""
Generator returning ``priority`` needed by sitemap.xml.
It generates values from 1 to 0.1 by decreasing in 0.1 on each
iteration. After 0.1 is reached, it will keep returning 0.1.
"""
priorities = [1, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2]
yield from itertools.chain(priorities, itertools.repeat(0.1))
def hreflang_formatter(lang):
"""
sitemap hreflang should follow correct format.
Use hyphen instead of underscore in language and country value.
ref: https://en.wikipedia.org/wiki/Hreflang#Common_Mistakes
"""
if '_' in lang:
return lang.replace("_", "-")
return lang
def changefreqs_generator():
"""
Generator returning ``changefreq`` needed by sitemap.xml.
It returns ``daily`` on first iteration, then ``weekly`` and then it
will return always ``monthly``.
We are using ``monthly`` as last value because ``never`` is too
aggressive. If the tag is removed and a branch is created with the same
name, we will want bots to revisit this.
"""
changefreqs = ['daily', 'weekly']
yield from itertools.chain(changefreqs, itertools.repeat('monthly'))
if project.privacy_level == constants.PRIVATE:
raise Http404
sorted_versions = sort_version_aware(
Version.objects.public(
project=project,
only_active=True,
),
)
versions = []
for version, priority, changefreq in zip(
sorted_versions,
priorities_generator(),
changefreqs_generator(),
):
element = {
'loc': version.get_subdomain_url(),
'priority': priority,
'changefreq': changefreq,
'languages': [],
}
# Version can be enabled, but not ``built`` yet. We want to show the
# link without a ``lastmod`` attribute
last_build = version.builds.order_by('-date').first()
if last_build:
element['lastmod'] = last_build.date.isoformat()
if project.translations.exists():
for translation in project.translations.all():
translation_versions = translation.versions.public(
).values_list('slug', flat=True)
if version.slug in translation_versions:
href = project.get_docs_url(
version_slug=version.slug,
lang_slug=translation.language,
private=False,
)
element['languages'].append({
'hreflang': hreflang_formatter(translation.language),
'href': href,
})
# Add itself also as protocol requires
element['languages'].append({
'hreflang': project.language,
'href': element['loc'],
})
versions.append(element)
context = {
'versions': versions,
}
return render(
request,
'sitemap.xml',
context,
content_type='application/xml',
)