From d90abf23cb578256f9d65c1b8a5fc2f32d619b97 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 26 Feb 2019 15:39:24 +0100 Subject: [PATCH 01/96] Initial structure for APIv3 --- readthedocs/settings/base.py | 6 + readthedocs/urls.py | 1 + readthedocs/v3/__init__.py | 0 readthedocs/v3/admin.py | 3 + readthedocs/v3/apps.py | 5 + readthedocs/v3/migrations/__init__.py | 0 readthedocs/v3/models.py | 3 + readthedocs/v3/serializers.py | 269 ++++++++++++++++++++++++++ readthedocs/v3/tests.py | 3 + readthedocs/v3/urls.py | 10 + readthedocs/v3/views.py | 36 ++++ 11 files changed, 336 insertions(+) create mode 100644 readthedocs/v3/__init__.py create mode 100644 readthedocs/v3/admin.py create mode 100644 readthedocs/v3/apps.py create mode 100644 readthedocs/v3/migrations/__init__.py create mode 100644 readthedocs/v3/models.py create mode 100644 readthedocs/v3/serializers.py create mode 100644 readthedocs/v3/tests.py create mode 100644 readthedocs/v3/urls.py create mode 100644 readthedocs/v3/views.py diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index d8a0f04e7e3..397a48fce97 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -107,6 +107,8 @@ def INSTALLED_APPS(self): # noqa 'guardian', 'django_gravatar', 'rest_framework', + 'rest_framework.authtoken', + 'rest_framework_serializer_extensions', 'corsheaders', 'textclassifier', 'annoying', @@ -125,6 +127,10 @@ def INSTALLED_APPS(self): # noqa 'readthedocs.redirects', 'readthedocs.rtd_tests', 'readthedocs.restapi', + + # TODO: refactor this module to be ``api.v3`` + 'readthedocs.v3', + 'readthedocs.gold', 'readthedocs.payments', 'readthedocs.notifications', diff --git a/readthedocs/urls.py b/readthedocs/urls.py index 8dc2836a760..164b4038cc8 100644 --- a/readthedocs/urls.py +++ b/readthedocs/urls.py @@ -66,6 +66,7 @@ r'^api-auth/', include('rest_framework.urls', namespace='rest_framework') ), + url(r'^api/v3/', include('readthedocs.v3.urls')), ] i18n_urls = [ diff --git a/readthedocs/v3/__init__.py b/readthedocs/v3/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/readthedocs/v3/admin.py b/readthedocs/v3/admin.py new file mode 100644 index 00000000000..8c38f3f3dad --- /dev/null +++ b/readthedocs/v3/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/readthedocs/v3/apps.py b/readthedocs/v3/apps.py new file mode 100644 index 00000000000..ce747d0030f --- /dev/null +++ b/readthedocs/v3/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class V3Config(AppConfig): + name = 'v3' diff --git a/readthedocs/v3/migrations/__init__.py b/readthedocs/v3/migrations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/readthedocs/v3/models.py b/readthedocs/v3/models.py new file mode 100644 index 00000000000..71a83623907 --- /dev/null +++ b/readthedocs/v3/models.py @@ -0,0 +1,3 @@ +from django.db import models + +# Create your models here. diff --git a/readthedocs/v3/serializers.py b/readthedocs/v3/serializers.py new file mode 100644 index 00000000000..78eb6ea8d88 --- /dev/null +++ b/readthedocs/v3/serializers.py @@ -0,0 +1,269 @@ +import datetime + +from django.core.urlresolvers import reverse +from django.contrib.auth.models import User + +from rest_framework import serializers +from readthedocs.projects.constants import LANGUAGES, PROGRAMMING_LANGUAGES +from readthedocs.projects.models import Project +from readthedocs.builds.models import Build, Version +from readthedocs.projects.version_handling import determine_stable_version +from readthedocs.builds.constants import STABLE + +from rest_framework_serializer_extensions.serializers import SerializerExtensionsMixin + + +class UserSerializer(serializers.ModelSerializer): + + # TODO: return ``null`` when ``last_name`` or ``first_name`` are `''``. I'm + # thinking on writing a decorator or similar that dynamically creates the + # methods based on a field with a list + + class Meta: + model = User + fields = [ + 'username', + 'date_joined', + 'last_login', + 'first_name', + 'last_name', + ] + + +class BuildSerializer(serializers.ModelSerializer): + + created = serializers.DateTimeField(source='date') + finished = serializers.SerializerMethodField() + duration = serializers.IntegerField(source='length') + + class Meta: + model = Build + expandable_fields = dict( + version='readthedocs.v3.serializers.VersionSerializer', + project='readthedocs.v3.serializers.ProjectSerializer', + # config=BuildConfigSerializer, + ) + fields = [ + 'id', + 'version', + 'project', + 'created', + 'finished', + 'duration', + 'state', + 'success', + 'error', + 'commit', + 'builder', + 'cold_storage', + 'config', + # 'links', + ] + + def get_finished(self, obj): + if obj.date and obj.length: + return obj.date + datetime.timedelta(seconds=obj.length) + + +class PrivacyLevelSerializer(serializers.Serializer): + code = serializers.CharField(source='privacy_level') + name = serializers.SerializerMethodField() + + def get_name(self, obj): + return obj.privacy_level.title() + + +class VersionURLsSerializer(serializers.Serializer): + documentation = serializers.SerializerMethodField() + vcs = serializers.SerializerMethodField() + + def get_documentation(self, obj): + return obj.project.get_docs_url( + version_slug=obj.slug, + ) + + def get_vcs(self, obj): + # TODO: make this method to work properly + if obj.project.repo_type == 'git': + return obj.project.repo + f'/tree/{obj.slug}' + + +class VersionSerializer(serializers.ModelSerializer): + + privacy_level = PrivacyLevelSerializer(source='*') + ref = serializers.SerializerMethodField() + last_build = serializers.SerializerMethodField() + + # FIXME: generate URLs with HTTPS schema + downloads = serializers.DictField(source='get_downloads') + + urls = VersionURLsSerializer(source='*') + + class Meta: + model = Version + # expandable_fields = dict( + # last_build=serializers.SerializerMethodField, + # ) + fields = [ + 'id', + 'slug', + 'verbose_name', + 'identifier', + 'ref', + 'built', + 'active', + 'uploaded', + 'privacy_level', + 'type', + + # TODO: make this field expandable also. Nested expandable fields + # are not working when using ``SerializerMethodField`` at the moment + 'last_build', + + 'downloads', + 'urls', + # 'links', + ] + + def get_ref(self, obj): + if obj.slug == STABLE: + stable = determine_stable_version(obj.project.versions.all()) + if stable: + return stable.slug + + def get_last_build(self, obj): + build = obj.builds.order_by('-date').first() + return BuildSerializer(build).data + + +class LanguageSerializer(serializers.Serializer): + + code = serializers.SerializerMethodField() + name = serializers.SerializerMethodField() + + def get_code(self, language): + return language + + def get_name(self, language): + for code, name in LANGUAGES: + if code == language: + return name + return 'Unknown' + + +class ProgrammingLanguageSerializer(serializers.Serializer): + + code = serializers.SerializerMethodField() + name = serializers.SerializerMethodField() + + def get_code(self, programming_language): + return programming_language + + def get_name(self, programming_language): + for code, name in PROGRAMMING_LANGUAGES: + if code == programming_language: + return name + return 'Unknown' + + +class ProjectURLsSerializer(serializers.Serializer): + documentation = serializers.CharField(source='get_docs_url') + project = serializers.SerializerMethodField() + + def get_project(self, obj): + # Overridden only to return ``None`` when the description is ``''`` + return obj.project_url or None + + +class RepositorySerializer(serializers.Serializer): + + url = serializers.CharField(source='repo') + type = serializers.CharField(source='repo_type') + + +class ProjectLinksSerializer(serializers.Serializer): + + _self = serializers.SerializerMethodField() + # users = serializers.URLField(source='get_link_users') + # versions = serializers.URLField(source='get_link_versions') + # users = serializers.URLField(source='get_link_users') + # builds = serializers.URLField(source='get_link_builds') + # subprojects = serializers.URLField(source='get_link_subprojects') + # translations = serializers.URLField(source='get_link_translations') + + def get__self(self, obj): + # TODO: maybe we want to make them full URLs instead of absolute + return reverse('projects-detail', kwargs={'project_slug': obj.slug}) + + +class ProjectSerializer(SerializerExtensionsMixin, serializers.ModelSerializer): + + language = LanguageSerializer() + programming_language = ProgrammingLanguageSerializer() + repository = RepositorySerializer(source='*') + privacy_level = PrivacyLevelSerializer(source='*') + urls = ProjectURLsSerializer(source='*') + # active_versions = serializers.SerializerMethodField() + subproject_of = serializers.SerializerMethodField() + translation_of = serializers.SerializerMethodField() + default_branch = serializers.CharField(source='get_default_branch') + tags = serializers.StringRelatedField(many=True) + + description = serializers.SerializerMethodField() + + links = ProjectLinksSerializer(source='*') + + # TODO: adapt these fields with the proper names in the db and then remove + # them from here + created = serializers.DateTimeField(source='pub_date') + modified = serializers.DateTimeField(source='modified_date') + + class Meta: + model = Project + expandable_fields = dict( + users=dict( + serializer=UserSerializer, + many=True, + ), + active_versions=dict( + serializer=serializers.SerializerMethodField, + ), + ) + fields = [ + 'id', + 'name', + 'slug', + 'description', + 'created', + 'modified', + 'language', + 'programming_language', + 'repository', + 'default_version', + 'default_branch', + 'privacy_level', + 'subproject_of', + 'translation_of', + 'urls', + 'tags', + 'links', + ] + + def get_description(self, obj): + # Overridden only to return ``None`` when the description is ``''`` + return obj.description or None + + def get_active_versions(self, obj): + return VersionSerializer(obj.versions.filter(active=True), many=True).data + + def get_translation_of(self, obj): + try: + return obj.main_language_project.slug + except: + return None + + def get_subproject_of(self, obj): + try: + return obj.superprojects.first().slug + except: + return None diff --git a/readthedocs/v3/tests.py b/readthedocs/v3/tests.py new file mode 100644 index 00000000000..7ce503c2dd9 --- /dev/null +++ b/readthedocs/v3/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/readthedocs/v3/urls.py b/readthedocs/v3/urls.py new file mode 100644 index 00000000000..56bb2f7e3f9 --- /dev/null +++ b/readthedocs/v3/urls.py @@ -0,0 +1,10 @@ +from rest_framework import routers + +from .views import ( + ProjectsViewSet, +) + +router = routers.DefaultRouter() +router.register(r'projects', ProjectsViewSet, basename='projects') + +urlpatterns = router.urls diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py new file mode 100644 index 00000000000..e71bcd42bbc --- /dev/null +++ b/readthedocs/v3/views.py @@ -0,0 +1,36 @@ +import django_filters.rest_framework +from rest_framework.authentication import SessionAuthentication, TokenAuthentication +from rest_framework.permissions import IsAuthenticated +from rest_framework.renderers import JSONRenderer +from rest_framework.throttling import UserRateThrottle, AnonRateThrottle +# from rest_framework import generics +from rest_framework.viewsets import ModelViewSet +from rest_framework_serializer_extensions.views import SerializerExtensionsAPIViewMixin + +from readthedocs.projects.models import Project + +from .serializers import ProjectSerializer + + +class APIv3Settings: + + authentication_classes = (SessionAuthentication, TokenAuthentication) + permission_classes = (IsAuthenticated,) + renderer_classes = (JSONRenderer,) + throttle_classes = (UserRateThrottle, AnonRateThrottle) + filter_backends = (django_filters.rest_framework.DjangoFilterBackend,) + + +class ProjectsViewSet(SerializerExtensionsAPIViewMixin, APIv3Settings, ModelViewSet): + + model = Project + lookup_field = 'slug' + lookup_url_kwarg = 'project_slug' + serializer_class = ProjectSerializer + filterset_fields = ( + 'privacy_level', + ) + + def get_queryset(self): + user = self.request.user + return user.projects.all() From 4e33c1a695f96be0b14f43500256fec1354c4f46 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 28 Feb 2019 13:40:09 +0100 Subject: [PATCH 02/96] Improve expandable fields using drf-flex-fields --- readthedocs/builds/models.py | 4 ++ readthedocs/settings/base.py | 1 - readthedocs/v3/serializers.py | 88 +++++++++++++++++++++-------------- readthedocs/v3/views.py | 16 +++++-- requirements/pip.txt | 2 + 5 files changed, 70 insertions(+), 41 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index acc0f8102d3..c09bce62c62 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -127,6 +127,10 @@ def __str__(self): ), ) + @property + def last_build(self): + return self.builds.order_by('-date').first() + @property def config(self): """ diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 397a48fce97..5dbbe9da931 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -108,7 +108,6 @@ def INSTALLED_APPS(self): # noqa 'django_gravatar', 'rest_framework', 'rest_framework.authtoken', - 'rest_framework_serializer_extensions', 'corsheaders', 'textclassifier', 'annoying', diff --git a/readthedocs/v3/serializers.py b/readthedocs/v3/serializers.py index 78eb6ea8d88..02fa763d20c 100644 --- a/readthedocs/v3/serializers.py +++ b/readthedocs/v3/serializers.py @@ -10,10 +10,11 @@ from readthedocs.projects.version_handling import determine_stable_version from readthedocs.builds.constants import STABLE -from rest_framework_serializer_extensions.serializers import SerializerExtensionsMixin +from rest_flex_fields import FlexFieldsModelSerializer +from rest_flex_fields.serializers import FlexFieldsSerializerMixin -class UserSerializer(serializers.ModelSerializer): +class UserSerializer(FlexFieldsModelSerializer): # TODO: return ``null`` when ``last_name`` or ``first_name`` are `''``. I'm # thinking on writing a decorator or similar that dynamically creates the @@ -30,19 +31,30 @@ class Meta: ] -class BuildSerializer(serializers.ModelSerializer): +class BuildConfigSerializer(FlexFieldsSerializerMixin, serializers.Serializer): + + def to_representation(self, obj): + # For now, we want to return the ``config`` object as it is without + # manipulating it. + return obj + + +class BuildSerializer(FlexFieldsModelSerializer): created = serializers.DateTimeField(source='date') finished = serializers.SerializerMethodField() duration = serializers.IntegerField(source='length') + expandable_fields = dict( + config=( + BuildConfigSerializer, dict( + source='config', + ), + ), + ) + class Meta: model = Build - expandable_fields = dict( - version='readthedocs.v3.serializers.VersionSerializer', - project='readthedocs.v3.serializers.ProjectSerializer', - # config=BuildConfigSerializer, - ) fields = [ 'id', 'version', @@ -56,7 +68,7 @@ class Meta: 'commit', 'builder', 'cold_storage', - 'config', + # 'config', # 'links', ] @@ -88,22 +100,26 @@ def get_vcs(self, obj): return obj.project.repo + f'/tree/{obj.slug}' -class VersionSerializer(serializers.ModelSerializer): +class VersionSerializer(FlexFieldsModelSerializer): privacy_level = PrivacyLevelSerializer(source='*') ref = serializers.SerializerMethodField() - last_build = serializers.SerializerMethodField() # FIXME: generate URLs with HTTPS schema downloads = serializers.DictField(source='get_downloads') urls = VersionURLsSerializer(source='*') + expandable_fields = dict( + last_build=( + BuildSerializer, dict( + source='last_build', + ), + ), + ) + class Meta: model = Version - # expandable_fields = dict( - # last_build=serializers.SerializerMethodField, - # ) fields = [ 'id', 'slug', @@ -116,10 +132,6 @@ class Meta: 'privacy_level', 'type', - # TODO: make this field expandable also. Nested expandable fields - # are not working when using ``SerializerMethodField`` at the moment - 'last_build', - 'downloads', 'urls', # 'links', @@ -131,10 +143,6 @@ def get_ref(self, obj): if stable: return stable.slug - def get_last_build(self, obj): - build = obj.builds.order_by('-date').first() - return BuildSerializer(build).data - class LanguageSerializer(serializers.Serializer): @@ -196,14 +204,13 @@ def get__self(self, obj): return reverse('projects-detail', kwargs={'project_slug': obj.slug}) -class ProjectSerializer(SerializerExtensionsMixin, serializers.ModelSerializer): +class ProjectSerializer(FlexFieldsModelSerializer): language = LanguageSerializer() programming_language = ProgrammingLanguageSerializer() repository = RepositorySerializer(source='*') privacy_level = PrivacyLevelSerializer(source='*') urls = ProjectURLsSerializer(source='*') - # active_versions = serializers.SerializerMethodField() subproject_of = serializers.SerializerMethodField() translation_of = serializers.SerializerMethodField() default_branch = serializers.CharField(source='get_default_branch') @@ -218,17 +225,25 @@ class ProjectSerializer(SerializerExtensionsMixin, serializers.ModelSerializer): created = serializers.DateTimeField(source='pub_date') modified = serializers.DateTimeField(source='modified_date') - class Meta: - model = Project - expandable_fields = dict( - users=dict( - serializer=UserSerializer, + expandable_fields = dict( + users=( + UserSerializer, dict( + source='users', many=True, ), - active_versions=dict( - serializer=serializers.SerializerMethodField, + ), + active_versions=( + VersionSerializer, dict( + # NOTE: this has to be a Model method, can't be a + # ``SerializerMethodField`` as far as I know + source='active_versions', + many=True, ), - ) + ), + ) + + class Meta: + model = Project fields = [ 'id', 'name', @@ -246,6 +261,12 @@ class Meta: 'translation_of', 'urls', 'tags', + + # NOTE: ``expandable_fields`` must not be included here. Otherwise, + # they will be tried to be rendered and fail + # 'users', + # 'active_versions', + 'links', ] @@ -253,9 +274,6 @@ def get_description(self, obj): # Overridden only to return ``None`` when the description is ``''`` return obj.description or None - def get_active_versions(self, obj): - return VersionSerializer(obj.versions.filter(active=True), many=True).data - def get_translation_of(self, obj): try: return obj.main_language_project.slug diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index e71bcd42bbc..b3e610aec46 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -3,11 +3,10 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.renderers import JSONRenderer from rest_framework.throttling import UserRateThrottle, AnonRateThrottle -# from rest_framework import generics -from rest_framework.viewsets import ModelViewSet -from rest_framework_serializer_extensions.views import SerializerExtensionsAPIViewMixin +from rest_flex_fields import FlexFieldsModelViewSet from readthedocs.projects.models import Project +from readthedocs.restapi.permissions import IsOwner from .serializers import ProjectSerializer @@ -15,21 +14,28 @@ class APIv3Settings: authentication_classes = (SessionAuthentication, TokenAuthentication) - permission_classes = (IsAuthenticated,) + permission_classes = (IsAuthenticated, IsOwner) renderer_classes = (JSONRenderer,) throttle_classes = (UserRateThrottle, AnonRateThrottle) filter_backends = (django_filters.rest_framework.DjangoFilterBackend,) -class ProjectsViewSet(SerializerExtensionsAPIViewMixin, APIv3Settings, ModelViewSet): +class ProjectsViewSet(APIv3Settings, FlexFieldsModelViewSet): model = Project lookup_field = 'slug' lookup_url_kwarg = 'project_slug' serializer_class = ProjectSerializer filterset_fields = ( + 'slug', 'privacy_level', ) + permit_list_expands = [ + 'users', + 'active_versions', + 'active_versions.last_build', + 'active_versions.last_build.config', + ] def get_queryset(self): user = self.request.user diff --git a/requirements/pip.txt b/requirements/pip.txt index a5af6d65a38..7d16dedaebe 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -17,6 +17,8 @@ Sphinx==1.8.5 # pyup: <2.0.0 # Filtering for the REST API django-filter==2.1.0 +drf-flex-fields==0.3.5 + django-vanilla-views==1.0.6 jsonfield==2.0.2 From 2494cc9d2afabb65d7549395d2f26a6a54ce9aed Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 28 Feb 2019 15:33:13 +0100 Subject: [PATCH 03/96] Basic throttle rates setting --- readthedocs/settings/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 5dbbe9da931..9b7f7aed1eb 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -480,6 +480,10 @@ def USE_PROMOS(self): # noqa REST_FRAMEWORK = { 'DEFAULT_FILTER_BACKENDS': ('django_filters.rest_framework.DjangoFilterBackend',), 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', # NOQA + 'DEFAULT_THROTTLE_RATES': { + 'anon': '5/minute', + 'user': '10/minute', + }, 'PAGE_SIZE': 10, } SILENCED_SYSTEM_CHECKS = ['fields.W342', 'guardian.W001'] From 6930782947f604630142b106cb079e627fcff499 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 28 Feb 2019 15:39:30 +0100 Subject: [PATCH 04/96] Use a class filter to allow expansion --- readthedocs/v3/views.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index b3e610aec46..53bc89edd22 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -1,4 +1,4 @@ -import django_filters.rest_framework +import django_filters.rest_framework as filters from rest_framework.authentication import SessionAuthentication, TokenAuthentication from rest_framework.permissions import IsAuthenticated from rest_framework.renderers import JSONRenderer @@ -8,6 +8,7 @@ from readthedocs.projects.models import Project from readthedocs.restapi.permissions import IsOwner +from .filters import ProjectFilter from .serializers import ProjectSerializer @@ -17,7 +18,7 @@ class APIv3Settings: permission_classes = (IsAuthenticated, IsOwner) renderer_classes = (JSONRenderer,) throttle_classes = (UserRateThrottle, AnonRateThrottle) - filter_backends = (django_filters.rest_framework.DjangoFilterBackend,) + filter_backends = (filters.DjangoFilterBackend,) class ProjectsViewSet(APIv3Settings, FlexFieldsModelViewSet): @@ -26,10 +27,7 @@ class ProjectsViewSet(APIv3Settings, FlexFieldsModelViewSet): lookup_field = 'slug' lookup_url_kwarg = 'project_slug' serializer_class = ProjectSerializer - filterset_fields = ( - 'slug', - 'privacy_level', - ) + filterset_class = ProjectFilter permit_list_expands = [ 'users', 'active_versions', From d0919d09427e9fa93fab60d4ad1107ad654e6526 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 28 Feb 2019 16:41:43 +0100 Subject: [PATCH 05/96] Use methods from models instead of custom from serializer --- readthedocs/builds/models.py | 28 ++++++++++++++++++++++++++++ readthedocs/v3/serializers.py | 16 +++------------- 2 files changed, 31 insertions(+), 13 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index c09bce62c62..3bea7868de0 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -26,6 +26,7 @@ PRIVATE, ) from readthedocs.projects.models import APIProject, Project +from readthedocs.projects.version_handling import determine_stable_version from .constants import ( BRANCH, @@ -127,6 +128,33 @@ def __str__(self): ), ) + @property + def ref(self): + if self.slug == STABLE: + stable = determine_stable_version(self.project.versions.all()) + if stable: + return stable.slug + + @property + def vcs_url(self): + url = '' + if self.slug == STABLE: + slug_url = self.ref + elif self.slug == LATEST: + slug_url = self.project.default_branch or self.project.vcs_repo().fallback_branch + else: + slug_url = self.slug + + if self.project.repo_type in ('git', 'gitlab'): + url = f'/tree/{slug_url}/' + + if self.project.repo_type == 'bitbucket': + slug_url = self.identifier + url = f'/src/{slug_url}' + + # TODO: improve this replacing + return self.project.repo.replace('git://', 'https://').replace('.git', '') + url + @property def last_build(self): return self.builds.order_by('-date').first() diff --git a/readthedocs/v3/serializers.py b/readthedocs/v3/serializers.py index 02fa763d20c..dcdffc32b65 100644 --- a/readthedocs/v3/serializers.py +++ b/readthedocs/v3/serializers.py @@ -7,8 +7,6 @@ from readthedocs.projects.constants import LANGUAGES, PROGRAMMING_LANGUAGES from readthedocs.projects.models import Project from readthedocs.builds.models import Build, Version -from readthedocs.projects.version_handling import determine_stable_version -from readthedocs.builds.constants import STABLE from rest_flex_fields import FlexFieldsModelSerializer from rest_flex_fields.serializers import FlexFieldsSerializerMixin @@ -87,27 +85,19 @@ def get_name(self, obj): class VersionURLsSerializer(serializers.Serializer): documentation = serializers.SerializerMethodField() - vcs = serializers.SerializerMethodField() + vcs = serializers.URLField(source='vcs_url') def get_documentation(self, obj): return obj.project.get_docs_url( version_slug=obj.slug, ) - def get_vcs(self, obj): - # TODO: make this method to work properly - if obj.project.repo_type == 'git': - return obj.project.repo + f'/tree/{obj.slug}' - class VersionSerializer(FlexFieldsModelSerializer): privacy_level = PrivacyLevelSerializer(source='*') - ref = serializers.SerializerMethodField() - - # FIXME: generate URLs with HTTPS schema - downloads = serializers.DictField(source='get_downloads') - + ref = serializers.CharField() + downloads = serializers.SerializerMethodField() urls = VersionURLsSerializer(source='*') expandable_fields = dict( From 7150fb2dd24be5aebd2d6e039f3b709d62cf0182 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 28 Feb 2019 16:42:32 +0100 Subject: [PATCH 06/96] Improve serializers --- readthedocs/v3/serializers.py | 41 ++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/readthedocs/v3/serializers.py b/readthedocs/v3/serializers.py index dcdffc32b65..c6977d49db1 100644 --- a/readthedocs/v3/serializers.py +++ b/readthedocs/v3/serializers.py @@ -1,5 +1,7 @@ import datetime +import urllib +from django.conf import settings from django.core.urlresolvers import reverse from django.contrib.auth.models import User @@ -66,7 +68,6 @@ class Meta: 'commit', 'builder', 'cold_storage', - # 'config', # 'links', ] @@ -121,17 +122,20 @@ class Meta: 'uploaded', 'privacy_level', 'type', - 'downloads', 'urls', # 'links', ] - def get_ref(self, obj): - if obj.slug == STABLE: - stable = determine_stable_version(obj.project.versions.all()) - if stable: - return stable.slug + def get_downloads(self, obj): + downloads = obj.get_downloads() + data = {} + + for k, v in downloads.items(): + if k in ('htmlzip', 'pdf', 'epub'): + data[k] = ('http:' if settings.DEBUG else 'https:') + v + + return data class LanguageSerializer(serializers.Serializer): @@ -182,16 +186,19 @@ class RepositorySerializer(serializers.Serializer): class ProjectLinksSerializer(serializers.Serializer): _self = serializers.SerializerMethodField() - # users = serializers.URLField(source='get_link_users') - # versions = serializers.URLField(source='get_link_versions') - # users = serializers.URLField(source='get_link_users') - # builds = serializers.URLField(source='get_link_builds') - # subprojects = serializers.URLField(source='get_link_subprojects') - # translations = serializers.URLField(source='get_link_translations') + + # TODO: add these once the endpoints get implemented + # users = serializers.SerializerMethodField() + # versions = serializers.SerializerMethodField() + # builds = serializers.SerializerMethodField() + # subprojects = serializers.SerializerMethodField() + # translations = serializers.SerializerMethodField() def get__self(self, obj): - # TODO: maybe we want to make them full URLs instead of absolute - return reverse('projects-detail', kwargs={'project_slug': obj.slug}) + scheme = 'http' if settings.DEBUG else 'https' + domain = settings.PRODUCTION_DOMAIN + path = reverse('projects-detail', kwargs={'project_slug': obj.slug}) + return urllib.parse.urlunparse((scheme, domain, path, '', '', '')) class ProjectSerializer(FlexFieldsModelSerializer): @@ -267,11 +274,11 @@ def get_description(self, obj): def get_translation_of(self, obj): try: return obj.main_language_project.slug - except: + except Exception: return None def get_subproject_of(self, obj): try: return obj.superprojects.first().slug - except: + except Exception: return None From b1f3d81dd9ac384528054f3344355839caa8cad9 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Sun, 3 Mar 2019 14:08:01 +0100 Subject: [PATCH 07/96] Push filters file --- readthedocs/v3/filters.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 readthedocs/v3/filters.py diff --git a/readthedocs/v3/filters.py b/readthedocs/v3/filters.py new file mode 100644 index 00000000000..50f5f64b2fe --- /dev/null +++ b/readthedocs/v3/filters.py @@ -0,0 +1,17 @@ +import django_filters.rest_framework as filters +from readthedocs.projects.models import Project + + +class ProjectFilter(filters.FilterSet): + name__contains = filters.CharFilter(field_name='name', lookup_expr='contains') + slug__contains = filters.CharFilter(field_name='slug', lookup_expr='contains') + + class Meta: + model = Project + fields = [ + 'name', + 'name__contains', + 'slug', + 'slug__contains', + 'privacy_level', + ] From 73f49df136c2c71d348e62834b95385e5169f8e1 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 5 Mar 2019 19:51:48 +0100 Subject: [PATCH 08/96] Nested (under /projects/) versions endpoint Listing http /api/v3/projects//versions/ Filtering http /api/v3/projects//versions/?active=true Expand extra fields http /api/v3/projects//versions/?expand=last_build Detail http /api/v3/projects//versions// Editing http PATCH /api/v3/projects//versions// active=true --- readthedocs/v3/filters.py | 22 +++++++++++++ readthedocs/v3/serializers.py | 10 ++++++ readthedocs/v3/urls.py | 13 ++++++-- readthedocs/v3/views.py | 60 +++++++++++++++++++++++++++++++---- requirements/pip.txt | 1 + 5 files changed, 96 insertions(+), 10 deletions(-) diff --git a/readthedocs/v3/filters.py b/readthedocs/v3/filters.py index 50f5f64b2fe..5c18cdcd6bb 100644 --- a/readthedocs/v3/filters.py +++ b/readthedocs/v3/filters.py @@ -1,4 +1,5 @@ import django_filters.rest_framework as filters +from readthedocs.builds.models import Version from readthedocs.projects.models import Project @@ -15,3 +16,24 @@ class Meta: 'slug__contains', 'privacy_level', ] + + +class VersionFilter(filters.FilterSet): + verbose_name__contains = filters.CharFilter( + field_name='versbose_name', + lookup_expr='contains', + ) + slug__contains = filters.CharFilter(field_name='slug', lookup_expr='contains') + + class Meta: + model = Version + fields = [ + 'verbose_name', + 'verbose_name__contains', + 'slug', + 'slug__contains', + 'privacy_level', + 'active', + 'built', + 'uploaded', + ] diff --git a/readthedocs/v3/serializers.py b/readthedocs/v3/serializers.py index c6977d49db1..ec21dc2547b 100644 --- a/readthedocs/v3/serializers.py +++ b/readthedocs/v3/serializers.py @@ -138,6 +138,16 @@ def get_downloads(self, obj): return data +class VersionUpdateSerializer(serializers.ModelSerializer): + + class Meta: + model = Version + fields = [ + 'active', + 'privacy_level', + ] + + class LanguageSerializer(serializers.Serializer): code = serializers.SerializerMethodField() diff --git a/readthedocs/v3/urls.py b/readthedocs/v3/urls.py index 56bb2f7e3f9..b71e8607789 100644 --- a/readthedocs/v3/urls.py +++ b/readthedocs/v3/urls.py @@ -1,10 +1,17 @@ -from rest_framework import routers +from rest_framework_extensions.routers import ExtendedSimpleRouter from .views import ( ProjectsViewSet, + VersionsViewSet, ) -router = routers.DefaultRouter() -router.register(r'projects', ProjectsViewSet, basename='projects') +router = ExtendedSimpleRouter() +router.register(r'projects', ProjectsViewSet, basename='projects') \ + .register( + r'versions', + VersionsViewSet, + base_name='projects-versions', + parents_query_lookups=['project__slug'], + ) urlpatterns = router.urls diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index 53bc89edd22..3bdffac6f70 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -1,33 +1,36 @@ import django_filters.rest_framework as filters from rest_framework.authentication import SessionAuthentication, TokenAuthentication -from rest_framework.permissions import IsAuthenticated +from rest_framework.permissions import IsAdminUser from rest_framework.renderers import JSONRenderer +from rest_framework.response import Response from rest_framework.throttling import UserRateThrottle, AnonRateThrottle +from rest_framework_extensions.mixins import NestedViewSetMixin from rest_flex_fields import FlexFieldsModelViewSet +from readthedocs.builds.models import Version from readthedocs.projects.models import Project -from readthedocs.restapi.permissions import IsOwner -from .filters import ProjectFilter -from .serializers import ProjectSerializer +from .filters import ProjectFilter, VersionFilter +from .serializers import ProjectSerializer, VersionSerializer, VersionUpdateSerializer class APIv3Settings: authentication_classes = (SessionAuthentication, TokenAuthentication) - permission_classes = (IsAuthenticated, IsOwner) + permission_classes = (IsAdminUser,) renderer_classes = (JSONRenderer,) throttle_classes = (UserRateThrottle, AnonRateThrottle) filter_backends = (filters.DjangoFilterBackend,) -class ProjectsViewSet(APIv3Settings, FlexFieldsModelViewSet): +class ProjectsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsModelViewSet): model = Project lookup_field = 'slug' lookup_url_kwarg = 'project_slug' serializer_class = ProjectSerializer filterset_class = ProjectFilter + queryset = Project.objects.all() permit_list_expands = [ 'users', 'active_versions', @@ -35,6 +38,49 @@ class ProjectsViewSet(APIv3Settings, FlexFieldsModelViewSet): 'active_versions.last_build.config', ] + # NOTE: accessing a existent project when we don't have permissions to + # access it, returns 404 instead of 403. + def get_queryset(self): + queryset = super().get_queryset() + return queryset.filter(users=self.request.user) + + +class VersionsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsModelViewSet): + + model = Version + lookup_field = 'slug' + lookup_url_kwarg = 'version_slug' + serializer_class = VersionSerializer + filterset_class = VersionFilter + queryset = Version.objects.all() + permit_list_expands = [ + 'last_build', + 'last_build.config', + ] + + # NOTE: ``NestedViewSetMixin`` is really good, but if the ``project.slug`` + # does not exist it does not return 404, but 200 instead: + # /api/v3/projects/nonexistent/versions/ + + def get_queryset(self): + # ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` + queryset = super().get_queryset() + + # we force to filter only by the versions the user has access to user = self.request.user - return user.projects.all() + queryset = queryset.filter(project__users=user) + return queryset + + def partial_update(self, request, pk=None, **kwargs): + version = self.get_object() + serializer = VersionUpdateSerializer( + version, + data=request.data, + partial=True, + ) + if serializer.is_valid(): + serializer.save() + return Response(status=204) + + return Response(data=serializer.errors, status=400) diff --git a/requirements/pip.txt b/requirements/pip.txt index 7d16dedaebe..419653caf7a 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -18,6 +18,7 @@ Sphinx==1.8.5 # pyup: <2.0.0 django-filter==2.1.0 drf-flex-fields==0.3.5 +drf-extensions==0.4.0 django-vanilla-views==1.0.6 jsonfield==2.0.2 From 2c3eecd4a1dfea2b7042e20596996eaafca674cc Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 5 Mar 2019 20:36:16 +0100 Subject: [PATCH 09/96] Build endpoints Listing (by project) http /api/v3/projects/pip/builds/ http /api/v3/projects/pip/builds/?expand=config Listing (by version in project) http /api/v3/projects/pip/versions/19.0.3/builds/ Detail http /api/v3/projects/pip/builds/1500/ Trigger (default version) http POST /api/v3/projects/test-builds/builds/ Trigger (specific version) http POST /api/v3/projects/pip/versions/19.0.3/builds/ --- readthedocs/v3/serializers.py | 2 ++ readthedocs/v3/urls.py | 33 +++++++++++++++---- readthedocs/v3/views.py | 60 +++++++++++++++++++++++++++++++++-- 3 files changed, 86 insertions(+), 9 deletions(-) diff --git a/readthedocs/v3/serializers.py b/readthedocs/v3/serializers.py index ec21dc2547b..eba1e2e0417 100644 --- a/readthedocs/v3/serializers.py +++ b/readthedocs/v3/serializers.py @@ -41,6 +41,8 @@ def to_representation(self, obj): class BuildSerializer(FlexFieldsModelSerializer): + project = serializers.SlugRelatedField(slug_field='slug', read_only=True) + version = serializers.SlugRelatedField(slug_field='slug', read_only=True) created = serializers.DateTimeField(source='date') finished = serializers.SerializerMethodField() duration = serializers.IntegerField(source='length') diff --git a/readthedocs/v3/urls.py b/readthedocs/v3/urls.py index b71e8607789..1722ca2dd88 100644 --- a/readthedocs/v3/urls.py +++ b/readthedocs/v3/urls.py @@ -1,17 +1,36 @@ from rest_framework_extensions.routers import ExtendedSimpleRouter from .views import ( + BuildsViewSet, ProjectsViewSet, VersionsViewSet, ) router = ExtendedSimpleRouter() -router.register(r'projects', ProjectsViewSet, basename='projects') \ - .register( - r'versions', - VersionsViewSet, - base_name='projects-versions', - parents_query_lookups=['project__slug'], - ) +projects = router.register(r'projects', ProjectsViewSet, basename='projects') +versions = projects.register( + r'versions', + VersionsViewSet, + base_name='projects-versions', + parents_query_lookups=['project__slug'], +) + +# allows /api/v3/projects/pip/versions/v3.6.2/builds/ +versions.register( + r'builds', + BuildsViewSet, + base_name='projects-versions-builds', + parents_query_lookups=[ + 'project__slug', + 'version__slug', + ], +) + +projects.register( + r'builds', + BuildsViewSet, + base_name='projects-builds', + parents_query_lookups=['project__slug'], +) urlpatterns = router.urls diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index 3bdffac6f70..e96628d6f20 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -1,3 +1,4 @@ +from django.shortcuts import get_object_or_404 import django_filters.rest_framework as filters from rest_framework.authentication import SessionAuthentication, TokenAuthentication from rest_framework.permissions import IsAdminUser @@ -7,11 +8,12 @@ from rest_framework_extensions.mixins import NestedViewSetMixin from rest_flex_fields import FlexFieldsModelViewSet -from readthedocs.builds.models import Version +from readthedocs.core.utils import trigger_build +from readthedocs.builds.models import Version, Build from readthedocs.projects.models import Project from .filters import ProjectFilter, VersionFilter -from .serializers import ProjectSerializer, VersionSerializer, VersionUpdateSerializer +from .serializers import ProjectSerializer, VersionSerializer, VersionUpdateSerializer, BuildSerializer class APIv3Settings: @@ -84,3 +86,57 @@ def partial_update(self, request, pk=None, **kwargs): return Response(status=204) return Response(data=serializer.errors, status=400) + + + +class BuildsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsModelViewSet): + model = Build + lookup_field = 'pk' + lookup_url_kwarg = 'build_pk' + serializer_class = BuildSerializer + # filterset_class = VersionFilter + queryset = Build.objects.all() + permit_list_expands = [ + 'config', + ] + + def get_queryset(self): + # ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` + queryset = super().get_queryset() + + # we force to filter only by the versions the user has access to + user = self.request.user + queryset = queryset.filter(project__users=user) + return queryset + + def create(self, request, **kwargs): + parent_lookup_project__slug = kwargs.get('parent_lookup_project__slug') + parent_lookup_version__slug = kwargs.get('parent_lookup_version__slug') + + version = None + project = get_object_or_404( + Project, + slug=parent_lookup_project__slug, + users=request.user, + ) + + if parent_lookup_version__slug: + version = get_object_or_404( + project.versions.all(), + slug=parent_lookup_version__slug, + ) + + _, build = trigger_build(project, version=version) + data = { + 'build': BuildSerializer(build).data, + 'project': ProjectSerializer(project).data, + 'version': VersionSerializer(build.version).data, + } + + if build: + data.update({'triggered': True}) + status = 202 + else: + data.update({'triggered': False}) + status = 400 + return Response(data=data, status=status) From 4577fe5f1fe3d86b56eada459ba9e33afafec5ed Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 5 Mar 2019 20:57:44 +0100 Subject: [PATCH 10/96] Build filters Filter by commit http /api/v3/projects/pip/builds/?commit=0d755d38a894324ab495e12307ec3efd5927df88 Filter by commit (and version) http /api/v3/projects/pip/versions/19.0.1/builds/?commit=0d755d38a894324ab495e12307ec3efd5927df88 Filter by running http /api/v3/projects/pip/builds/?running=true Filter by running (and version) http /api/v3/projects/pip/versions/19.0.1/builds/?running=true --- readthedocs/v3/filters.py | 20 +++++++++++++++++++- readthedocs/v3/views.py | 4 ++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/readthedocs/v3/filters.py b/readthedocs/v3/filters.py index 5c18cdcd6bb..07adeb14904 100644 --- a/readthedocs/v3/filters.py +++ b/readthedocs/v3/filters.py @@ -1,5 +1,6 @@ import django_filters.rest_framework as filters -from readthedocs.builds.models import Version +from readthedocs.builds.constants import BUILD_STATE_FINISHED +from readthedocs.builds.models import Build, Version from readthedocs.projects.models import Project @@ -37,3 +38,20 @@ class Meta: 'built', 'uploaded', ] + + +class BuildFilter(filters.FilterSet): + running = filters.BooleanFilter(method='get_running') + + class Meta: + model = Build + fields = [ + 'commit', + 'running', + ] + + def get_running(self, queryset, name, value): + if value: + return queryset.exclude(state=BUILD_STATE_FINISHED) + else: + return queryset.filter(state=BUILD_STATE_FINISHED) diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index e96628d6f20..213a2ebc43b 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -12,7 +12,7 @@ from readthedocs.builds.models import Version, Build from readthedocs.projects.models import Project -from .filters import ProjectFilter, VersionFilter +from .filters import ProjectFilter, VersionFilter, BuildFilter from .serializers import ProjectSerializer, VersionSerializer, VersionUpdateSerializer, BuildSerializer @@ -94,7 +94,7 @@ class BuildsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsModelViewSet): lookup_field = 'pk' lookup_url_kwarg = 'build_pk' serializer_class = BuildSerializer - # filterset_class = VersionFilter + filterset_class = BuildFilter queryset = Build.objects.all() permit_list_expands = [ 'config', From a0435898fcfe5e271aa495fb62a256e501b0e412 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 5 Mar 2019 21:16:46 +0100 Subject: [PATCH 11/96] Endpoints for related projects Translations http /api/v3/projects/pip/translations/ Subprojects http /api/v3/projects/pip/subprojects/ Superprojects http /api/v3/projects/pip/superprojects/ --- readthedocs/v3/views.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index 213a2ebc43b..226ef6d8d4b 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -1,6 +1,7 @@ from django.shortcuts import get_object_or_404 import django_filters.rest_framework as filters from rest_framework.authentication import SessionAuthentication, TokenAuthentication +from rest_framework.decorators import action from rest_framework.permissions import IsAdminUser from rest_framework.renderers import JSONRenderer from rest_framework.response import Response @@ -11,7 +12,6 @@ from readthedocs.core.utils import trigger_build from readthedocs.builds.models import Version, Build from readthedocs.projects.models import Project - from .filters import ProjectFilter, VersionFilter, BuildFilter from .serializers import ProjectSerializer, VersionSerializer, VersionUpdateSerializer, BuildSerializer @@ -47,6 +47,30 @@ def get_queryset(self): queryset = super().get_queryset() return queryset.filter(users=self.request.user) + @action(detail=True, methods=['get']) + def translations(self, request, project_slug): + project = self.get_object() + return self._related_projects(project.translations.all()) + + @action(detail=True, methods=['get']) + def superprojects(self, request, project_slug): + project = self.get_object() + return self._related_projects(project.superprojects.all()) + + @action(detail=True, methods=['get']) + def subprojects(self, request, project_slug): + project = self.get_object() + return self._related_projects(project.subprojects.all()) + + def _related_projects(self, queryset): + page = self.paginate_queryset(queryset) + if page is not None: + serializer = self.get_serializer(page, many=True) + return self.get_paginated_response(serializer.data) + + serializer = self.get_serializer(queryset, many=True) + return Response(serializer.data) + class VersionsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsModelViewSet): From 193665429d31f448dcd13437a7b2bdf0cd5d246c Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 5 Mar 2019 21:54:18 +0100 Subject: [PATCH 12/96] Add "links" field for Project, Version and Build --- readthedocs/v3/serializers.py | 144 +++++++++++++++++++++++++++++++--- 1 file changed, 134 insertions(+), 10 deletions(-) diff --git a/readthedocs/v3/serializers.py b/readthedocs/v3/serializers.py index eba1e2e0417..12a84c509a6 100644 --- a/readthedocs/v3/serializers.py +++ b/readthedocs/v3/serializers.py @@ -31,6 +31,49 @@ class Meta: ] +class BaseLinksSerializer(serializers.Serializer): + + def _absolute_url(self, path): + scheme = 'http' if settings.DEBUG else 'https' + domain = settings.PRODUCTION_DOMAIN + return urllib.parse.urlunparse((scheme, domain, path, '', '', '')) + + +class BuildLinksSerializer(BaseLinksSerializer): + _self = serializers.SerializerMethodField() + version = serializers.SerializerMethodField() + project = serializers.SerializerMethodField() + + def get__self(self, obj): + path = reverse( + 'projects-builds-detail', + kwargs={ + 'parent_lookup_project__slug': obj.project.slug, + 'build_pk': obj.pk, + }, + ) + return self._absolute_url(path) + + def get_version(self, obj): + path = reverse( + 'projects-versions-detail', + kwargs={ + 'parent_lookup_project__slug': obj.project.slug, + 'version_slug': obj.version.slug, + }, + ) + return self._absolute_url(path) + + def get_project(self, obj): + path = reverse( + 'projects-detail', + kwargs={ + 'project_slug': obj.project.slug, + }, + ) + return self._absolute_url(path) + + class BuildConfigSerializer(FlexFieldsSerializerMixin, serializers.Serializer): def to_representation(self, obj): @@ -46,6 +89,7 @@ class BuildSerializer(FlexFieldsModelSerializer): created = serializers.DateTimeField(source='date') finished = serializers.SerializerMethodField() duration = serializers.IntegerField(source='length') + links = BuildLinksSerializer(source='*') expandable_fields = dict( config=( @@ -70,7 +114,7 @@ class Meta: 'commit', 'builder', 'cold_storage', - # 'links', + 'links', ] def get_finished(self, obj): @@ -86,6 +130,41 @@ def get_name(self, obj): return obj.privacy_level.title() +class VersionLinksSerializer(BaseLinksSerializer): + _self = serializers.SerializerMethodField() + builds = serializers.SerializerMethodField() + project = serializers.SerializerMethodField() + + def get__self(self, obj): + path = reverse( + 'projects-versions-detail', + kwargs={ + 'parent_lookup_project__slug': obj.project.slug, + 'version_slug': obj.slug, + }, + ) + return self._absolute_url(path) + + def get_builds(self, obj): + path = reverse( + 'projects-versions-builds-list', + kwargs={ + 'parent_lookup_project__slug': obj.project.slug, + 'parent_lookup_version__slug': obj.slug, + }, + ) + return self._absolute_url(path) + + def get_project(self, obj): + path = reverse( + 'projects-detail', + kwargs={ + 'project_slug': obj.project.slug, + }, + ) + return self._absolute_url(path) + + class VersionURLsSerializer(serializers.Serializer): documentation = serializers.SerializerMethodField() vcs = serializers.URLField(source='vcs_url') @@ -102,6 +181,7 @@ class VersionSerializer(FlexFieldsModelSerializer): ref = serializers.CharField() downloads = serializers.SerializerMethodField() urls = VersionURLsSerializer(source='*') + links = VersionLinksSerializer(source='*') expandable_fields = dict( last_build=( @@ -126,7 +206,7 @@ class Meta: 'type', 'downloads', 'urls', - # 'links', + 'links', ] def get_downloads(self, obj): @@ -195,22 +275,66 @@ class RepositorySerializer(serializers.Serializer): type = serializers.CharField(source='repo_type') -class ProjectLinksSerializer(serializers.Serializer): +class ProjectLinksSerializer(BaseLinksSerializer): _self = serializers.SerializerMethodField() # TODO: add these once the endpoints get implemented # users = serializers.SerializerMethodField() - # versions = serializers.SerializerMethodField() - # builds = serializers.SerializerMethodField() - # subprojects = serializers.SerializerMethodField() - # translations = serializers.SerializerMethodField() + versions = serializers.SerializerMethodField() + builds = serializers.SerializerMethodField() + subprojects = serializers.SerializerMethodField() + superprojects = serializers.SerializerMethodField() + translations = serializers.SerializerMethodField() def get__self(self, obj): - scheme = 'http' if settings.DEBUG else 'https' - domain = settings.PRODUCTION_DOMAIN path = reverse('projects-detail', kwargs={'project_slug': obj.slug}) - return urllib.parse.urlunparse((scheme, domain, path, '', '', '')) + return self._absolute_url(path) + + def get_versions(self, obj): + path = reverse( + 'projects-versions-list', + kwargs={ + 'parent_lookup_project__slug': obj.slug, + }, + ) + return self._absolute_url(path) + + def get_builds(self, obj): + path = reverse( + 'projects-builds-list', + kwargs={ + 'parent_lookup_project__slug': obj.slug, + }, + ) + return self._absolute_url(path) + + def get_subprojects(self, obj): + path = reverse( + 'projects-subprojects', + kwargs={ + 'project_slug': obj.slug, + }, + ) + return self._absolute_url(path) + + def get_superprojects(self, obj): + path = reverse( + 'projects-superprojects', + kwargs={ + 'project_slug': obj.slug, + }, + ) + return self._absolute_url(path) + + def get_translations(self, obj): + path = reverse( + 'projects-translations', + kwargs={ + 'project_slug': obj.slug, + }, + ) + return self._absolute_url(path) class ProjectSerializer(FlexFieldsModelSerializer): From 7968af2b8b9cd8e90b242bb8c4249d5adef558e7 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 5 Mar 2019 22:06:28 +0100 Subject: [PATCH 13/96] Style --- readthedocs/v3/urls.py | 15 ++++++++++++++- readthedocs/v3/views.py | 1 - 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/readthedocs/v3/urls.py b/readthedocs/v3/urls.py index 1722ca2dd88..d987f4763d6 100644 --- a/readthedocs/v3/urls.py +++ b/readthedocs/v3/urls.py @@ -7,7 +7,17 @@ ) router = ExtendedSimpleRouter() -projects = router.register(r'projects', ProjectsViewSet, basename='projects') + +# allows /api/v3/projects/ +# allows /api/v3/projects/pip/ +projects = router.register( + r'projects', + ProjectsViewSet, + basename='projects', +) + +# allows /api/v3/projects/pip/versions/ +# allows /api/v3/projects/pip/versions/latest/ versions = projects.register( r'versions', VersionsViewSet, @@ -16,6 +26,7 @@ ) # allows /api/v3/projects/pip/versions/v3.6.2/builds/ +# allows /api/v3/projects/pip/versions/v3.6.2/builds/1053/ versions.register( r'builds', BuildsViewSet, @@ -26,6 +37,8 @@ ], ) +# allows /api/v3/projects/pip/builds/ +# allows /api/v3/projects/pip/builds/1053/ projects.register( r'builds', BuildsViewSet, diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index 226ef6d8d4b..15f454a6228 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -112,7 +112,6 @@ def partial_update(self, request, pk=None, **kwargs): return Response(data=serializer.errors, status=400) - class BuildsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsModelViewSet): model = Build lookup_field = 'pk' From cedb137d4bcf42b57fcffa783e6d61acd66b358b Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 6 Mar 2019 09:04:11 +0100 Subject: [PATCH 14/96] Allow . (dots) on version_slug endpoint URLs --- readthedocs/v3/views.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index 15f454a6228..2a010059880 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -77,6 +77,10 @@ class VersionsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsModelViewSet) model = Version lookup_field = 'slug' lookup_url_kwarg = 'version_slug' + + # Allow ``.`` (dots) on version slug + lookup_value_regex = r'[^/]+' + serializer_class = VersionSerializer filterset_class = VersionFilter queryset = Version.objects.all() From 577f4aec43a3993df1b2b7ddd822b1c450ce398c Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 7 Mar 2019 15:56:16 +0100 Subject: [PATCH 15/96] Make the API really browsable - use DefaultRouter to get this feature - inherit view only from methods that are allowed (using Mixin classes) - add docstring for `projects` endpoint which is displayed on the browsable response --- readthedocs/settings/base.py | 1 + readthedocs/v3/urls.py | 4 +-- readthedocs/v3/views.py | 57 ++++++++++++++++++++++++++++++++---- 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 9b7f7aed1eb..76bd21bef97 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -116,6 +116,7 @@ def INSTALLED_APPS(self): # noqa 'messages_extends', 'django_filters', 'django_elasticsearch_dsl', + 'django_filters', # our apps 'readthedocs.projects', diff --git a/readthedocs/v3/urls.py b/readthedocs/v3/urls.py index d987f4763d6..0e2d4496069 100644 --- a/readthedocs/v3/urls.py +++ b/readthedocs/v3/urls.py @@ -1,4 +1,4 @@ -from rest_framework_extensions.routers import ExtendedSimpleRouter +from .routers import DefaultRouterWithNesting from .views import ( BuildsViewSet, @@ -6,7 +6,7 @@ VersionsViewSet, ) -router = ExtendedSimpleRouter() +router = DefaultRouterWithNesting() # allows /api/v3/projects/ # allows /api/v3/projects/pip/ diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index 2a010059880..32f7226f336 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -3,11 +3,13 @@ from rest_framework.authentication import SessionAuthentication, TokenAuthentication from rest_framework.decorators import action from rest_framework.permissions import IsAdminUser -from rest_framework.renderers import JSONRenderer +from rest_framework.renderers import JSONRenderer, BrowsableAPIRenderer from rest_framework.response import Response from rest_framework.throttling import UserRateThrottle, AnonRateThrottle +from rest_framework.mixins import ListModelMixin, RetrieveModelMixin, UpdateModelMixin, CreateModelMixin +from rest_framework.viewsets import GenericViewSet from rest_framework_extensions.mixins import NestedViewSetMixin -from rest_flex_fields import FlexFieldsModelViewSet +from rest_flex_fields.views import FlexFieldsMixin from readthedocs.core.utils import trigger_build from readthedocs.builds.models import Version, Build @@ -20,12 +22,55 @@ class APIv3Settings: authentication_classes = (SessionAuthentication, TokenAuthentication) permission_classes = (IsAdminUser,) - renderer_classes = (JSONRenderer,) + renderer_classes = (JSONRenderer, BrowsableAPIRenderer) throttle_classes = (UserRateThrottle, AnonRateThrottle) filter_backends = (filters.DjangoFilterBackend,) -class ProjectsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsModelViewSet): +class ProjectsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): + + """ + Endpoints related to ``Project`` objects. + + * Listing objects. + * Detailed object. + + Retrieving only needed data using ``?fields=`` URL attribute is allowed. + + ### Filters + + Allowed via URL attributes: + + * slug + * slug__contains + * name + * name__contains + + ### Expandable fields + + There are some fields that are not returned by default because they are + expensive to calculate. Although, they are available for those cases where + they are needed. + + Allowed via ``?expand=`` URL attribue: + + * users + * active_versions + * active_versions.last_build + * active_versions.last_build.confg + + + ### Examples: + + * List my projects: ``/api/v3/projects/`` + * Filter list: ``/api/v3/projects/?name__contains=test`` + * Retrieve only needed data: ``/api/v3/projects/?fields=slug,created`` + * Retrieve specific project: ``/api/v3/projects/pip/`` + * Expand required fields: ``/api/v3/projects/pip/?expand=active_versions`` + * Translations of a projects: ``/api/v3/projects/pip/translations/`` + * Subprojects of a projects: ``/api/v3/projects/pip/subprojects/`` + * Superprojects of a projects: ``/api/v3/projects/pip/superprojects/`` + """ model = Project lookup_field = 'slug' @@ -72,7 +117,7 @@ def _related_projects(self, queryset): return Response(serializer.data) -class VersionsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsModelViewSet): +class VersionsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, UpdateModelMixin, GenericViewSet): model = Version lookup_field = 'slug' @@ -116,7 +161,7 @@ def partial_update(self, request, pk=None, **kwargs): return Response(data=serializer.errors, status=400) -class BuildsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsModelViewSet): +class BuildsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, CreateModelMixin, GenericViewSet): model = Build lookup_field = 'pk' lookup_url_kwarg = 'build_pk' From f5502eaa88be746dbb430dc8aae8637fa4456698 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 7 Mar 2019 15:57:56 +0100 Subject: [PATCH 16/96] Make build.state render as code/name --- readthedocs/v3/serializers.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/readthedocs/v3/serializers.py b/readthedocs/v3/serializers.py index 12a84c509a6..90863b20466 100644 --- a/readthedocs/v3/serializers.py +++ b/readthedocs/v3/serializers.py @@ -82,6 +82,14 @@ def to_representation(self, obj): return obj +class BuildStateSerializer(serializers.Serializer): + code = serializers.CharField(source='state') + name = serializers.SerializerMethodField() + + def get_name(self, obj): + return obj.state.title() + + class BuildSerializer(FlexFieldsModelSerializer): project = serializers.SlugRelatedField(slug_field='slug', read_only=True) @@ -89,6 +97,7 @@ class BuildSerializer(FlexFieldsModelSerializer): created = serializers.DateTimeField(source='date') finished = serializers.SerializerMethodField() duration = serializers.IntegerField(source='length') + state = BuildStateSerializer(source='*') links = BuildLinksSerializer(source='*') expandable_fields = dict( From 47888dd7dc219d3b2c28747465ecd62bdb9294ad Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 7 Mar 2019 16:40:23 +0100 Subject: [PATCH 17/96] Generate valid links for current logged in user in documentation --- readthedocs/v3/views.py | 35 ++++++++++++++++++++++++++++++----- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index 32f7226f336..e5f9eee632e 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -1,4 +1,5 @@ from django.shortcuts import get_object_or_404 +from django.utils.safestring import mark_safe import django_filters.rest_framework as filters from rest_framework.authentication import SessionAuthentication, TokenAuthentication from rest_framework.decorators import action @@ -14,6 +15,7 @@ from readthedocs.core.utils import trigger_build from readthedocs.builds.models import Version, Build from readthedocs.projects.models import Project +from rest_framework.metadata import SimpleMetadata from .filters import ProjectFilter, VersionFilter, BuildFilter from .serializers import ProjectSerializer, VersionSerializer, VersionUpdateSerializer, BuildSerializer @@ -25,6 +27,7 @@ class APIv3Settings: renderer_classes = (JSONRenderer, BrowsableAPIRenderer) throttle_classes = (UserRateThrottle, AnonRateThrottle) filter_backends = (filters.DjangoFilterBackend,) + metadata_class = SimpleMetadata class ProjectsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): @@ -65,11 +68,11 @@ class ProjectsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListMo * List my projects: ``/api/v3/projects/`` * Filter list: ``/api/v3/projects/?name__contains=test`` * Retrieve only needed data: ``/api/v3/projects/?fields=slug,created`` - * Retrieve specific project: ``/api/v3/projects/pip/`` - * Expand required fields: ``/api/v3/projects/pip/?expand=active_versions`` - * Translations of a projects: ``/api/v3/projects/pip/translations/`` - * Subprojects of a projects: ``/api/v3/projects/pip/subprojects/`` - * Superprojects of a projects: ``/api/v3/projects/pip/superprojects/`` + * Retrieve specific project: ``/api/v3/projects/{project_slug}/`` + * Expand required fields: ``/api/v3/projects/{project_slug}/?expand=active_versions`` + * Translations of a projects: ``/api/v3/projects/{project_slug}/translations/`` + * Subprojects of a projects: ``/api/v3/projects/{project_slug}/subprojects/`` + * Superprojects of a projects: ``/api/v3/projects/{project_slug}/superprojects/`` """ model = Project @@ -88,6 +91,24 @@ class ProjectsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListMo # NOTE: accessing a existent project when we don't have permissions to # access it, returns 404 instead of 403. + def get_view_description(self, *args, **kwargs): + """ + Make valid links for the user's documentation browseable API. + + If the user has already one project, we pick the first and make all the + links for that project. Otherwise, we default to the placeholder. + """ + description = super().get_view_description(*args, **kwargs) + project = self.request.user.projects.first() + + # TODO: make the links clickable when ``kwargs.html=True`` + + if project: + return mark_safe(description.format( + project_slug=project.slug, + )) + return description + def get_queryset(self): queryset = super().get_queryset() return queryset.filter(users=self.request.user) @@ -172,6 +193,10 @@ class BuildsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListMode 'config', ] + # TODO: browsable API shows the BuildSerializer for POST method, but it + # should be empty. This can be achieved by using a custom ``metadata_class`` + # and overriding the ``actions`` field + def get_queryset(self): # ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` queryset = super().get_queryset() From 9da9edea7324b7897a2c614bc74740ba01d35527 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 7 Mar 2019 17:04:08 +0100 Subject: [PATCH 18/96] Omit form rendering on POST for builds endpoint --- readthedocs/v3/renderer.py | 21 +++++++++++++++++++++ readthedocs/v3/views.py | 6 ++---- 2 files changed, 23 insertions(+), 4 deletions(-) create mode 100644 readthedocs/v3/renderer.py diff --git a/readthedocs/v3/renderer.py b/readthedocs/v3/renderer.py new file mode 100644 index 00000000000..ae0af3a6505 --- /dev/null +++ b/readthedocs/v3/renderer.py @@ -0,0 +1,21 @@ +from rest_framework.renderers import BrowsableAPIRenderer + + +class BuildsBrowsableAPIRenderer(BrowsableAPIRenderer): + """ + APIRenderer that does not render a raw/html form for POST. + + Builds endpoint accept the creation of a new Build object, but does not + accept any data on body. So, we omit rendering the raw and html forms when + browsing it. + """ + + def get_raw_data_form(self, data, view, method, request): + if method == 'POST': + return None + return super().get_raw_data_form(data, view, method, request) + + def get_rendered_html_form(self, data, view, method, request): + if method == 'POST': + return None + return super().get_rendered_html_form(data, view, method, request) diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index e5f9eee632e..a5fa5bda358 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -17,6 +17,7 @@ from readthedocs.projects.models import Project from rest_framework.metadata import SimpleMetadata from .filters import ProjectFilter, VersionFilter, BuildFilter +from .renderer import BuildsBrowsableAPIRenderer from .serializers import ProjectSerializer, VersionSerializer, VersionUpdateSerializer, BuildSerializer @@ -192,10 +193,7 @@ class BuildsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListMode permit_list_expands = [ 'config', ] - - # TODO: browsable API shows the BuildSerializer for POST method, but it - # should be empty. This can be achieved by using a custom ``metadata_class`` - # and overriding the ``actions`` field + renderer_classes = (JSONRenderer, BuildsBrowsableAPIRenderer) def get_queryset(self): # ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` From 693090254c5bf57e267f1c62440b88612653f0ff Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 7 Mar 2019 17:04:30 +0100 Subject: [PATCH 19/96] Default router with nesting support --- readthedocs/v3/routers.py | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 readthedocs/v3/routers.py diff --git a/readthedocs/v3/routers.py b/readthedocs/v3/routers.py new file mode 100644 index 00000000000..47c9e1038fd --- /dev/null +++ b/readthedocs/v3/routers.py @@ -0,0 +1,6 @@ +from rest_framework.routers import DefaultRouter +from rest_framework_extensions.routers import NestedRouterMixin + + +class DefaultRouterWithNesting(NestedRouterMixin, DefaultRouter): + pass From a528102c8af05b49ea807f8596bb69bb621d8be6 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 7 Mar 2019 18:44:00 +0100 Subject: [PATCH 20/96] Use custom AlphabeticalSortedJSONRenderer --- readthedocs/v3/renderer.py | 56 +++++++++++++++++++++++++++++++++++++- readthedocs/v3/views.py | 8 ++++-- 2 files changed, 60 insertions(+), 4 deletions(-) diff --git a/readthedocs/v3/renderer.py b/readthedocs/v3/renderer.py index ae0af3a6505..6f0b976ebfb 100644 --- a/readthedocs/v3/renderer.py +++ b/readthedocs/v3/renderer.py @@ -1,4 +1,8 @@ -from rest_framework.renderers import BrowsableAPIRenderer +import json +from rest_framework.compat import ( + INDENT_SEPARATORS, LONG_SEPARATORS, SHORT_SEPARATORS, +) +from rest_framework.renderers import BrowsableAPIRenderer, JSONRenderer class BuildsBrowsableAPIRenderer(BrowsableAPIRenderer): @@ -19,3 +23,53 @@ def get_rendered_html_form(self, data, view, method, request): if method == 'POST': return None return super().get_rendered_html_form(data, view, method, request) + + +class AlphaneticalSortedJSONRenderer(JSONRenderer): + + """ + Renderer that sort they keys from the JSON alphabetically. + + See https://github.com/encode/django-rest-framework/pull/4202 + """ + + def render(self, data, accepted_media_type=None, renderer_context=None): + """ + Copied from ``rest_framework.renders.JSONRenderer``. + + Changes: + + - sort_keys=True on json.dumps + - use str instead of six.text_types + + https://github.com/encode/django-rest-framework/blob/master/rest_framework/renderers.py#L89 + """ + if data is None: + return bytes() + + renderer_context = renderer_context or {} + indent = self.get_indent(accepted_media_type, renderer_context) + + if indent is None: + separators = SHORT_SEPARATORS if self.compact else LONG_SEPARATORS + else: + separators = INDENT_SEPARATORS + + ret = json.dumps( + data, + cls=self.encoder_class, + indent=indent, + ensure_ascii=self.ensure_ascii, + allow_nan=not self.strict, + separators=separators, + sort_keys=True, + ) + + if isinstance(ret, str): + # We always fully escape \u2028 and \u2029 to ensure we output JSON + # that is a strict javascript subset. If bytes were returned + # by json.dumps() then we don't have these characters in any case. + # See: http://timelessrepo.com/json-isnt-a-javascript-subset + ret = ret.replace('\u2028', '\\u2028').replace('\u2029', '\\u2029') + return bytes(ret.encode('utf-8')) + return ret diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index a5fa5bda358..80fa76fd327 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -17,7 +17,7 @@ from readthedocs.projects.models import Project from rest_framework.metadata import SimpleMetadata from .filters import ProjectFilter, VersionFilter, BuildFilter -from .renderer import BuildsBrowsableAPIRenderer +from .renderer import BuildsBrowsableAPIRenderer, AlphaneticalSortedJSONRenderer from .serializers import ProjectSerializer, VersionSerializer, VersionUpdateSerializer, BuildSerializer @@ -25,7 +25,7 @@ class APIv3Settings: authentication_classes = (SessionAuthentication, TokenAuthentication) permission_classes = (IsAdminUser,) - renderer_classes = (JSONRenderer, BrowsableAPIRenderer) + renderer_classes = (AlphaneticalSortedJSONRenderer, BrowsableAPIRenderer) throttle_classes = (UserRateThrottle, AnonRateThrottle) filter_backends = (filters.DjangoFilterBackend,) metadata_class = SimpleMetadata @@ -33,6 +33,8 @@ class APIv3Settings: class ProjectsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): + # Markdown docstring is automatically rendered by BrowsableAPIRenderer. + """ Endpoints related to ``Project`` objects. @@ -193,7 +195,7 @@ class BuildsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListMode permit_list_expands = [ 'config', ] - renderer_classes = (JSONRenderer, BuildsBrowsableAPIRenderer) + renderer_classes = (AlphaneticalSortedJSONRenderer, BuildsBrowsableAPIRenderer) def get_queryset(self): # ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` From e827cad8a5d4a2beec7a8a674793b1cb4042b110 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 7 Mar 2019 19:19:04 +0100 Subject: [PATCH 21/96] Use JWT as default Token authentication --- readthedocs/settings/base.py | 12 +++++++++++- readthedocs/v3/urls.py | 17 +++++++++++++++-- readthedocs/v3/views.py | 5 +++-- requirements/pip.txt | 1 + 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 76bd21bef97..a1fdf8f7518 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -6,6 +6,7 @@ import getpass import os +import datetime from celery.schedules import crontab @@ -107,7 +108,7 @@ def INSTALLED_APPS(self): # noqa 'guardian', 'django_gravatar', 'rest_framework', - 'rest_framework.authtoken', + # 'rest_framework.authtoken', 'corsheaders', 'textclassifier', 'annoying', @@ -487,6 +488,15 @@ def USE_PROMOS(self): # noqa }, 'PAGE_SIZE': 10, } + + SIMPLE_JWT = { + 'ACCESS_TOKEN_LIFETIME': datetime.timedelta(minutes=5), + 'REFRESH_TOKEN_LIFETIME': datetime.timedelta(days=1), + 'ROTATE_REFRESH_TOKENS': False, + 'BLACKLIST_AFTER_ROTATION': True, + 'AUTH_HEADER_TYPES': ('JWT',), + } + SILENCED_SYSTEM_CHECKS = ['fields.W342', 'guardian.W001'] # Logging diff --git a/readthedocs/v3/urls.py b/readthedocs/v3/urls.py index 0e2d4496069..1900aa18458 100644 --- a/readthedocs/v3/urls.py +++ b/readthedocs/v3/urls.py @@ -1,5 +1,12 @@ -from .routers import DefaultRouterWithNesting +from django.conf.urls import url + +from rest_framework_simplejwt.views import ( + TokenObtainPairView, + TokenRefreshView, + TokenVerifyView, +) +from .routers import DefaultRouterWithNesting from .views import ( BuildsViewSet, ProjectsViewSet, @@ -46,4 +53,10 @@ parents_query_lookups=['project__slug'], ) -urlpatterns = router.urls +urlpatterns = [ + url(r'^token/$', TokenObtainPairView.as_view(), name='token_obtain_pair'), + url(r'^token/refresh/$', TokenRefreshView.as_view(), name='token_refresh'), + url(r'^token/verify/$', TokenVerifyView.as_view(), name='token_verify'), +] + +urlpatterns += router.urls diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index 80fa76fd327..f8ced9c7287 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -4,7 +4,7 @@ from rest_framework.authentication import SessionAuthentication, TokenAuthentication from rest_framework.decorators import action from rest_framework.permissions import IsAdminUser -from rest_framework.renderers import JSONRenderer, BrowsableAPIRenderer +from rest_framework.renderers import BrowsableAPIRenderer from rest_framework.response import Response from rest_framework.throttling import UserRateThrottle, AnonRateThrottle from rest_framework.mixins import ListModelMixin, RetrieveModelMixin, UpdateModelMixin, CreateModelMixin @@ -16,6 +16,7 @@ from readthedocs.builds.models import Version, Build from readthedocs.projects.models import Project from rest_framework.metadata import SimpleMetadata +from rest_framework_simplejwt.authentication import JWTAuthentication from .filters import ProjectFilter, VersionFilter, BuildFilter from .renderer import BuildsBrowsableAPIRenderer, AlphaneticalSortedJSONRenderer from .serializers import ProjectSerializer, VersionSerializer, VersionUpdateSerializer, BuildSerializer @@ -23,7 +24,7 @@ class APIv3Settings: - authentication_classes = (SessionAuthentication, TokenAuthentication) + authentication_classes = (SessionAuthentication, JWTAuthentication) permission_classes = (IsAdminUser,) renderer_classes = (AlphaneticalSortedJSONRenderer, BrowsableAPIRenderer) throttle_classes = (UserRateThrottle, AnonRateThrottle) diff --git a/requirements/pip.txt b/requirements/pip.txt index 419653caf7a..59020336a8d 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -19,6 +19,7 @@ django-filter==2.1.0 drf-flex-fields==0.3.5 drf-extensions==0.4.0 +djangorestframework-simplejwt==4.0.0 django-vanilla-views==1.0.6 jsonfield==2.0.2 From 36cf2d2e41bf907d37d5e081ec864bcb215321fc Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 7 Mar 2019 19:30:50 +0100 Subject: [PATCH 22/96] Register `users` endpoint --- readthedocs/v3/serializers.py | 12 ++++++++++-- readthedocs/v3/urls.py | 8 ++++++++ readthedocs/v3/views.py | 22 ++++++++++++++++++++-- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/readthedocs/v3/serializers.py b/readthedocs/v3/serializers.py index 90863b20466..46a752baf59 100644 --- a/readthedocs/v3/serializers.py +++ b/readthedocs/v3/serializers.py @@ -288,8 +288,7 @@ class ProjectLinksSerializer(BaseLinksSerializer): _self = serializers.SerializerMethodField() - # TODO: add these once the endpoints get implemented - # users = serializers.SerializerMethodField() + users = serializers.SerializerMethodField() versions = serializers.SerializerMethodField() builds = serializers.SerializerMethodField() subprojects = serializers.SerializerMethodField() @@ -300,6 +299,15 @@ def get__self(self, obj): path = reverse('projects-detail', kwargs={'project_slug': obj.slug}) return self._absolute_url(path) + def get_users(self, obj): + path = reverse( + 'projects-users-list', + kwargs={ + 'parent_lookup_projects__slug': obj.slug, + }, + ) + return self._absolute_url(path) + def get_versions(self, obj): path = reverse( 'projects-versions-list', diff --git a/readthedocs/v3/urls.py b/readthedocs/v3/urls.py index 1900aa18458..77fa7ae353f 100644 --- a/readthedocs/v3/urls.py +++ b/readthedocs/v3/urls.py @@ -11,6 +11,7 @@ BuildsViewSet, ProjectsViewSet, VersionsViewSet, + UsersViewSet, ) router = DefaultRouterWithNesting() @@ -53,6 +54,13 @@ parents_query_lookups=['project__slug'], ) +projects.register( + r'users', + UsersViewSet, + base_name='projects-users', + parents_query_lookups=['projects__slug'], +) + urlpatterns = [ url(r'^token/$', TokenObtainPairView.as_view(), name='token_obtain_pair'), url(r'^token/refresh/$', TokenRefreshView.as_view(), name='token_refresh'), diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index f8ced9c7287..2065a9fd176 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -1,7 +1,8 @@ +from django.contrib.auth.models import User from django.shortcuts import get_object_or_404 from django.utils.safestring import mark_safe import django_filters.rest_framework as filters -from rest_framework.authentication import SessionAuthentication, TokenAuthentication +from rest_framework.authentication import SessionAuthentication from rest_framework.decorators import action from rest_framework.permissions import IsAdminUser from rest_framework.renderers import BrowsableAPIRenderer @@ -19,7 +20,7 @@ from rest_framework_simplejwt.authentication import JWTAuthentication from .filters import ProjectFilter, VersionFilter, BuildFilter from .renderer import BuildsBrowsableAPIRenderer, AlphaneticalSortedJSONRenderer -from .serializers import ProjectSerializer, VersionSerializer, VersionUpdateSerializer, BuildSerializer +from .serializers import ProjectSerializer, VersionSerializer, VersionUpdateSerializer, BuildSerializer, UserSerializer class APIv3Settings: @@ -238,3 +239,20 @@ def create(self, request, **kwargs): data.update({'triggered': False}) status = 400 return Response(data=data, status=status) + + +class UsersViewSet(APIv3Settings, NestedViewSetMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): + model = User + lookup_field = 'username' + lookup_url_kwarg = 'user_username' + serializer_class = UserSerializer + queryset = User.objects.all() + + def get_queryset(self): + # ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` + queryset = super().get_queryset() + + # we force to filter only by the projects the user has access to + user = self.request.user + queryset = queryset.filter(projects__in=user.projects.all()).distinct() + return queryset From bbeb9b780908cf1322722669f1c68259345fe261 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 7 Mar 2019 19:41:58 +0100 Subject: [PATCH 23/96] Add documentation to the root view of BrowsableAPI --- readthedocs/v3/routers.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/readthedocs/v3/routers.py b/readthedocs/v3/routers.py index 47c9e1038fd..0bde9f1f020 100644 --- a/readthedocs/v3/routers.py +++ b/readthedocs/v3/routers.py @@ -1,6 +1,21 @@ -from rest_framework.routers import DefaultRouter +from rest_framework.routers import DefaultRouter, APIRootView from rest_framework_extensions.routers import NestedRouterMixin +class DocsAPIRootView(APIRootView): + + # Overridden only to add documentation for BrowsableAPIRenderer. + + """ + Read the Docs APIv3 root endpoint. + + Full documentation at [https://docs.readthedocs.io/en/latest/api/v3.html](https://docs.readthedocs.io/en/latest/api/v3.html). + """ + + def get_view_name(self): + return 'Read the Docs APIv3' + + class DefaultRouterWithNesting(NestedRouterMixin, DefaultRouter): - pass + APIRootView = DocsAPIRootView + root_view_name = 'api-v3-root' From d5aed8a62d80a60c6b91949d889ce3fced2b011c Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 7 Mar 2019 19:58:44 +0100 Subject: [PATCH 24/96] Follow the pattern for URLs --- readthedocs/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/urls.py b/readthedocs/urls.py index 164b4038cc8..27b29e0b1a0 100644 --- a/readthedocs/urls.py +++ b/readthedocs/urls.py @@ -63,7 +63,7 @@ # Keep the `doc_search` at root level, so the test does not fail for other API url(r'^api/v2/docsearch/$', PageSearchAPIView.as_view(), name='doc_search'), url( - r'^api-auth/', + r'^api/auth/', include('rest_framework.urls', namespace='rest_framework') ), url(r'^api/v3/', include('readthedocs.v3.urls')), From b7055c552bb1693e4347f043147ca172a2e7a0f5 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 7 Mar 2019 20:27:03 +0100 Subject: [PATCH 25/96] Return proper serializers (Version, Build) depending on the action --- readthedocs/v3/renderer.py | 22 +--------------------- readthedocs/v3/serializers.py | 7 +++++++ readthedocs/v3/views.py | 26 ++++++++++++++++++++------ 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/readthedocs/v3/renderer.py b/readthedocs/v3/renderer.py index 6f0b976ebfb..fad520f9c61 100644 --- a/readthedocs/v3/renderer.py +++ b/readthedocs/v3/renderer.py @@ -2,27 +2,7 @@ from rest_framework.compat import ( INDENT_SEPARATORS, LONG_SEPARATORS, SHORT_SEPARATORS, ) -from rest_framework.renderers import BrowsableAPIRenderer, JSONRenderer - - -class BuildsBrowsableAPIRenderer(BrowsableAPIRenderer): - """ - APIRenderer that does not render a raw/html form for POST. - - Builds endpoint accept the creation of a new Build object, but does not - accept any data on body. So, we omit rendering the raw and html forms when - browsing it. - """ - - def get_raw_data_form(self, data, view, method, request): - if method == 'POST': - return None - return super().get_raw_data_form(data, view, method, request) - - def get_rendered_html_form(self, data, view, method, request): - if method == 'POST': - return None - return super().get_rendered_html_form(data, view, method, request) +from rest_framework.renderers import JSONRenderer class AlphaneticalSortedJSONRenderer(JSONRenderer): diff --git a/readthedocs/v3/serializers.py b/readthedocs/v3/serializers.py index 46a752baf59..2813153535a 100644 --- a/readthedocs/v3/serializers.py +++ b/readthedocs/v3/serializers.py @@ -39,6 +39,13 @@ def _absolute_url(self, path): return urllib.parse.urlunparse((scheme, domain, path, '', '', '')) +class BuildTriggerSerializer(serializers.ModelSerializer): + + class Meta: + model = Build + fields = [] + + class BuildLinksSerializer(BaseLinksSerializer): _self = serializers.SerializerMethodField() version = serializers.SerializerMethodField() diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index 2065a9fd176..fb78887cfa8 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -19,8 +19,8 @@ from rest_framework.metadata import SimpleMetadata from rest_framework_simplejwt.authentication import JWTAuthentication from .filters import ProjectFilter, VersionFilter, BuildFilter -from .renderer import BuildsBrowsableAPIRenderer, AlphaneticalSortedJSONRenderer -from .serializers import ProjectSerializer, VersionSerializer, VersionUpdateSerializer, BuildSerializer, UserSerializer +from .renderer import AlphaneticalSortedJSONRenderer +from .serializers import ProjectSerializer, VersionSerializer, VersionUpdateSerializer, BuildTriggerSerializer, BuildSerializer, UserSerializer class APIv3Settings: @@ -152,7 +152,6 @@ class VersionsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListMo # Allow ``.`` (dots) on version slug lookup_value_regex = r'[^/]+' - serializer_class = VersionSerializer filterset_class = VersionFilter queryset = Version.objects.all() permit_list_expands = [ @@ -164,6 +163,14 @@ class VersionsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListMo # does not exist it does not return 404, but 200 instead: # /api/v3/projects/nonexistent/versions/ + def get_serializer_class(self): + """ + Return correct serializer depending on the action (GET or PUT/PATCH/POST). + """ + if self.action in ('list', 'retrieve'): + return VersionSerializer + return VersionUpdateSerializer + def get_queryset(self): # ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` queryset = super().get_queryset() @@ -175,7 +182,8 @@ def get_queryset(self): def partial_update(self, request, pk=None, **kwargs): version = self.get_object() - serializer = VersionUpdateSerializer( + serializer_class = self.get_serializer_class() + serializer = serializer_class( version, data=request.data, partial=True, @@ -191,13 +199,11 @@ class BuildsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListMode model = Build lookup_field = 'pk' lookup_url_kwarg = 'build_pk' - serializer_class = BuildSerializer filterset_class = BuildFilter queryset = Build.objects.all() permit_list_expands = [ 'config', ] - renderer_classes = (AlphaneticalSortedJSONRenderer, BuildsBrowsableAPIRenderer) def get_queryset(self): # ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` @@ -208,6 +214,11 @@ def get_queryset(self): queryset = queryset.filter(project__users=user) return queryset + def get_serializer_class(self): + if self.action in ('list', 'retrieve'): + return BuildSerializer + return BuildTriggerSerializer + def create(self, request, **kwargs): parent_lookup_project__slug = kwargs.get('parent_lookup_project__slug') parent_lookup_version__slug = kwargs.get('parent_lookup_version__slug') @@ -226,6 +237,9 @@ def create(self, request, **kwargs): ) _, build = trigger_build(project, version=version) + + # TODO: refactor this to be a serializer + # BuildTriggeredSerializer(build, project, version).data data = { 'build': BuildSerializer(build).data, 'project': ProjectSerializer(project).data, From 0f2a7fc741ee4cc5041f0186af3e65b2f4330170 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 7 Mar 2019 21:31:58 +0100 Subject: [PATCH 26/96] Comment about why BrowsableAPI does not work on no GET/POST methods --- readthedocs/v3/views.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index fb78887cfa8..520658afc44 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -181,6 +181,10 @@ def get_queryset(self): return queryset def partial_update(self, request, pk=None, **kwargs): + # NOTE: ``Authorization: `` is mandatory to use this method from + # Browsable API since SessionAuthentication can't be used because we set + # ``httpOnly`` on our cookies and the ``PUT/PATCH`` method are triggered + # via Javascript version = self.get_object() serializer_class = self.get_serializer_class() serializer = serializer_class( From 6a620c80875987fa0305e674a67d559dbd8d8788 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 7 Mar 2019 22:11:56 +0100 Subject: [PATCH 27/96] Make PATCH/PUT for version return properly 204 and No Content --- readthedocs/v3/views.py | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index 520658afc44..42da818f6e3 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -180,23 +180,19 @@ def get_queryset(self): queryset = queryset.filter(project__users=user) return queryset - def partial_update(self, request, pk=None, **kwargs): + def update(self, request, *args, **kwargs): + """ + Make PUT/PATCH behaves in the same way. + + Force to return 204 is the update was good. + """ + # NOTE: ``Authorization: `` is mandatory to use this method from # Browsable API since SessionAuthentication can't be used because we set # ``httpOnly`` on our cookies and the ``PUT/PATCH`` method are triggered # via Javascript - version = self.get_object() - serializer_class = self.get_serializer_class() - serializer = serializer_class( - version, - data=request.data, - partial=True, - ) - if serializer.is_valid(): - serializer.save() - return Response(status=204) - - return Response(data=serializer.errors, status=400) + response = super().update(request, *args, **kwargs) + return Response(status=204) class BuildsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, CreateModelMixin, GenericViewSet): From e970e9610e67e38f54afbfa04070f198d407e565 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Sun, 10 Mar 2019 16:17:09 +0100 Subject: [PATCH 28/96] Make superproject endpoint to return a single project if exists --- readthedocs/v3/serializers.py | 2 +- readthedocs/v3/views.py | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/readthedocs/v3/serializers.py b/readthedocs/v3/serializers.py index 2813153535a..7b587da02d3 100644 --- a/readthedocs/v3/serializers.py +++ b/readthedocs/v3/serializers.py @@ -344,7 +344,7 @@ def get_subprojects(self, obj): def get_superprojects(self, obj): path = reverse( - 'projects-superprojects', + 'projects-superproject', kwargs={ 'project_slug': obj.slug, }, diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index 42da818f6e3..c4eb678bb77 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -75,9 +75,9 @@ class ProjectsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListMo * Retrieve only needed data: ``/api/v3/projects/?fields=slug,created`` * Retrieve specific project: ``/api/v3/projects/{project_slug}/`` * Expand required fields: ``/api/v3/projects/{project_slug}/?expand=active_versions`` - * Translations of a projects: ``/api/v3/projects/{project_slug}/translations/`` - * Subprojects of a projects: ``/api/v3/projects/{project_slug}/subprojects/`` - * Superprojects of a projects: ``/api/v3/projects/{project_slug}/superprojects/`` + * Translations of a project: ``/api/v3/projects/{project_slug}/translations/`` + * Subprojects of a project: ``/api/v3/projects/{project_slug}/subprojects/`` + * Superproject of a project: ``/api/v3/projects/{project_slug}/superproject/`` """ model = Project @@ -124,14 +124,19 @@ def translations(self, request, project_slug): return self._related_projects(project.translations.all()) @action(detail=True, methods=['get']) - def superprojects(self, request, project_slug): + def superproject(self, request, project_slug): project = self.get_object() - return self._related_projects(project.superprojects.all()) + superproject = getattr(project, 'main_project', None) + data = None + if superproject: + data = self.get_serializer(superproject).data + return Response(data) + return Response(status=404) @action(detail=True, methods=['get']) def subprojects(self, request, project_slug): project = self.get_object() - return self._related_projects(project.subprojects.all()) + return self._related_projects(project.superprojects.all()) def _related_projects(self, queryset): page = self.paginate_queryset(queryset) From dfa5b2e788adfaafef3e3b5310878c35a5e4d1aa Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Sun, 10 Mar 2019 17:43:46 +0100 Subject: [PATCH 29/96] Browsable API branding --- readthedocs/templates/rest_framework/api.html | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 readthedocs/templates/rest_framework/api.html diff --git a/readthedocs/templates/rest_framework/api.html b/readthedocs/templates/rest_framework/api.html new file mode 100644 index 00000000000..953fe8e6f1c --- /dev/null +++ b/readthedocs/templates/rest_framework/api.html @@ -0,0 +1,10 @@ +{% extends "rest_framework/base.html" %} +{% load static %} + +{% block branding %} + + Read the Docs + +{% endblock %} + +{% block userlinks %}{% endblock %} From f4475a18a435fdc21dc24668211d39af21034bda Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Sun, 10 Mar 2019 17:44:12 +0100 Subject: [PATCH 30/96] Proper query for dynamic description --- readthedocs/v3/views.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index c4eb678bb77..2f15d2c20fa 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -104,14 +104,15 @@ def get_view_description(self, *args, **kwargs): links for that project. Otherwise, we default to the placeholder. """ description = super().get_view_description(*args, **kwargs) - project = self.request.user.projects.first() - # TODO: make the links clickable when ``kwargs.html=True`` - - if project: - return mark_safe(description.format( - project_slug=project.slug, - )) + project = None + if self.request and self.request.user.is_authenticated(): + project = self.request.user.projects.first() + if project: + # TODO: make the links clickable when ``kwargs.html=True`` + return mark_safe(description.format( + project_slug=project.slug, + )) return description def get_queryset(self): From cf905c57bfe9fcf5918c4f9a13a400c03fa41776 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Sun, 10 Mar 2019 17:48:44 +0100 Subject: [PATCH 31/96] Fix superproject serializer field --- readthedocs/v3/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/v3/serializers.py b/readthedocs/v3/serializers.py index 7b587da02d3..1c9a22b4a4e 100644 --- a/readthedocs/v3/serializers.py +++ b/readthedocs/v3/serializers.py @@ -299,7 +299,7 @@ class ProjectLinksSerializer(BaseLinksSerializer): versions = serializers.SerializerMethodField() builds = serializers.SerializerMethodField() subprojects = serializers.SerializerMethodField() - superprojects = serializers.SerializerMethodField() + superproject = serializers.SerializerMethodField() translations = serializers.SerializerMethodField() def get__self(self, obj): @@ -342,7 +342,7 @@ def get_subprojects(self, obj): ) return self._absolute_url(path) - def get_superprojects(self, obj): + def get_superproject(self, obj): path = reverse( 'projects-superproject', kwargs={ From c363eeb0fd1851644b6448297c40849d85ef2c62 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Sun, 10 Mar 2019 18:01:27 +0100 Subject: [PATCH 32/96] Go back to simple Token authorization --- readthedocs/settings/base.py | 10 +--------- readthedocs/v3/urls.py | 6 ------ readthedocs/v3/views.py | 5 ++--- 3 files changed, 3 insertions(+), 18 deletions(-) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index a1fdf8f7518..d1a3c19dc20 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -108,7 +108,7 @@ def INSTALLED_APPS(self): # noqa 'guardian', 'django_gravatar', 'rest_framework', - # 'rest_framework.authtoken', + 'rest_framework.authtoken', 'corsheaders', 'textclassifier', 'annoying', @@ -489,14 +489,6 @@ def USE_PROMOS(self): # noqa 'PAGE_SIZE': 10, } - SIMPLE_JWT = { - 'ACCESS_TOKEN_LIFETIME': datetime.timedelta(minutes=5), - 'REFRESH_TOKEN_LIFETIME': datetime.timedelta(days=1), - 'ROTATE_REFRESH_TOKENS': False, - 'BLACKLIST_AFTER_ROTATION': True, - 'AUTH_HEADER_TYPES': ('JWT',), - } - SILENCED_SYSTEM_CHECKS = ['fields.W342', 'guardian.W001'] # Logging diff --git a/readthedocs/v3/urls.py b/readthedocs/v3/urls.py index 77fa7ae353f..a45de673243 100644 --- a/readthedocs/v3/urls.py +++ b/readthedocs/v3/urls.py @@ -1,11 +1,5 @@ from django.conf.urls import url -from rest_framework_simplejwt.views import ( - TokenObtainPairView, - TokenRefreshView, - TokenVerifyView, -) - from .routers import DefaultRouterWithNesting from .views import ( BuildsViewSet, diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index 2f15d2c20fa..c8fd0e684c2 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -2,7 +2,7 @@ from django.shortcuts import get_object_or_404 from django.utils.safestring import mark_safe import django_filters.rest_framework as filters -from rest_framework.authentication import SessionAuthentication +from rest_framework.authentication import SessionAuthentication, TokenAuthentication from rest_framework.decorators import action from rest_framework.permissions import IsAdminUser from rest_framework.renderers import BrowsableAPIRenderer @@ -17,7 +17,6 @@ from readthedocs.builds.models import Version, Build from readthedocs.projects.models import Project from rest_framework.metadata import SimpleMetadata -from rest_framework_simplejwt.authentication import JWTAuthentication from .filters import ProjectFilter, VersionFilter, BuildFilter from .renderer import AlphaneticalSortedJSONRenderer from .serializers import ProjectSerializer, VersionSerializer, VersionUpdateSerializer, BuildTriggerSerializer, BuildSerializer, UserSerializer @@ -25,7 +24,7 @@ class APIv3Settings: - authentication_classes = (SessionAuthentication, JWTAuthentication) + authentication_classes = (SessionAuthentication, TokenAuthentication) permission_classes = (IsAdminUser,) renderer_classes = (AlphaneticalSortedJSONRenderer, BrowsableAPIRenderer) throttle_classes = (UserRateThrottle, AnonRateThrottle) From 547ead4b965d4e6eb0a6746f71eb12741d20a08c Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Sun, 10 Mar 2019 18:01:57 +0100 Subject: [PATCH 33/96] Use internal documentation framework --- readthedocs/v3/urls.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/readthedocs/v3/urls.py b/readthedocs/v3/urls.py index a45de673243..2225051aea9 100644 --- a/readthedocs/v3/urls.py +++ b/readthedocs/v3/urls.py @@ -1,4 +1,6 @@ -from django.conf.urls import url +from django.conf.urls import url, include + +from rest_framework.documentation import include_docs_urls from .routers import DefaultRouterWithNesting from .views import ( @@ -56,9 +58,13 @@ ) urlpatterns = [ - url(r'^token/$', TokenObtainPairView.as_view(), name='token_obtain_pair'), - url(r'^token/refresh/$', TokenRefreshView.as_view(), name='token_refresh'), - url(r'^token/verify/$', TokenVerifyView.as_view(), name='token_verify'), + url(r'^docs/', include_docs_urls( + title='Read the Docs API', + patterns=[ + url(r'/api/v3/', include(router.urls)), + ], + public=True), + ), ] urlpatterns += router.urls From b1b38b380bc9d6055f6c4bff20cb4c32b10cf620 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 12 Mar 2019 11:52:27 +0100 Subject: [PATCH 34/96] Update Project filters and docs --- docs/api/v3.rst | 7 +++++-- readthedocs/v3/filters.py | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/docs/api/v3.rst b/docs/api/v3.rst index 6624f6791d1..1c432f45f5c 100644 --- a/docs/api/v3.rst +++ b/docs/api/v3.rst @@ -77,10 +77,13 @@ Projects list :>json string previous: URI for previous set of projects. :>json array results: array of ``project`` objects. - :query string privacy_level: one of ``public``, ``private``, ``protected``. + :query string name: name of the project. + :query string name__contains: part of the name of the project. + :query string slug: slug of the project. + :query string slug__contains: part of the slug of the project. :query string language: language code as ``en``, ``es``, ``ru``, etc. + :query string privacy_level: one of ``public``, ``private``, ``protected``. :query string programming_language: programming language code as ``py``, ``js``, etc. - :query string repository_url: URL of the repository. :query string repository_type: one of ``git``, ``hg``, ``bzr``, ``svn``. :requestheader Authorization: required token to authenticate. diff --git a/readthedocs/v3/filters.py b/readthedocs/v3/filters.py index 07adeb14904..68b64a74fef 100644 --- a/readthedocs/v3/filters.py +++ b/readthedocs/v3/filters.py @@ -7,6 +7,7 @@ class ProjectFilter(filters.FilterSet): name__contains = filters.CharFilter(field_name='name', lookup_expr='contains') slug__contains = filters.CharFilter(field_name='slug', lookup_expr='contains') + repository_type = filters.CharFilter(field_name='repo_type', lookup_expr='exact') class Meta: model = Project @@ -15,7 +16,10 @@ class Meta: 'name__contains', 'slug', 'slug__contains', + 'language', 'privacy_level', + 'programming_language', + 'repository_type', ] From 745558b19a2d9b60deaa964c0c0ff512d2431ae6 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 12 Mar 2019 11:58:47 +0100 Subject: [PATCH 35/96] Update docs for triggering a build --- docs/api/v3.rst | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/api/v3.rst b/docs/api/v3.rst index 1c432f45f5c..571ccbe6b99 100644 --- a/docs/api/v3.rst +++ b/docs/api/v3.rst @@ -347,7 +347,7 @@ Build details .. sourcecode:: bash - $ curl https://readthedocs.org/api/v3/projects/pip/builds/8592686/?include_config=true + $ curl https://readthedocs.org/api/v3/projects/pip/builds/8592686/?expand=config **Example response**: @@ -484,15 +484,22 @@ Build triggering .. http:post:: /api/v3/projects/(string:project_slug)/builds/ - Trigger a new build for this project. + Trigger a new build for the default version of this project. - **Example request**: + **Example response**: - .. sourcecode:: json + `See Build details <#build-details>`_ - { - "version": "latest", - } + :requestheader Authorization: required token to authenticate. + + :statuscode 202: Accepted + :statuscode 400: Some field is invalid + :statuscode 401: Not valid permissions + + +.. http:post:: /api/v3/projects/(string:project_slug)/versions/(string:version_slug)/builds/ + + Trigger a new build for the ``version_slug`` version of this project. **Example response**: @@ -500,7 +507,7 @@ Build triggering :requestheader Authorization: required token to authenticate. - :statuscode 201: Created sucessfully + :statuscode 202: Accepted :statuscode 400: Some field is invalid :statuscode 401: Not valid permissions From 31cc708bb677301d05dd552fba71a71951b2f3b5 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Thu, 21 Mar 2019 11:28:13 +0100 Subject: [PATCH 36/96] Linting --- readthedocs/v3/filters.py | 21 +++++++++++--- readthedocs/v3/renderer.py | 5 +++- readthedocs/v3/routers.py | 2 +- readthedocs/v3/serializers.py | 27 +++++++++--------- readthedocs/v3/urls.py | 26 ++++++++--------- readthedocs/v3/views.py | 54 ++++++++++++++++++++++++----------- 6 files changed, 85 insertions(+), 50 deletions(-) diff --git a/readthedocs/v3/filters.py b/readthedocs/v3/filters.py index 68b64a74fef..f670a2531c0 100644 --- a/readthedocs/v3/filters.py +++ b/readthedocs/v3/filters.py @@ -1,13 +1,23 @@ import django_filters.rest_framework as filters + from readthedocs.builds.constants import BUILD_STATE_FINISHED from readthedocs.builds.models import Build, Version from readthedocs.projects.models import Project class ProjectFilter(filters.FilterSet): - name__contains = filters.CharFilter(field_name='name', lookup_expr='contains') - slug__contains = filters.CharFilter(field_name='slug', lookup_expr='contains') - repository_type = filters.CharFilter(field_name='repo_type', lookup_expr='exact') + name__contains = filters.CharFilter( + field_name='name', + lookup_expr='contains', + ) + slug__contains = filters.CharFilter( + field_name='slug', + lookup_expr='contains', + ) + repository_type = filters.CharFilter( + field_name='repo_type', + lookup_expr='exact', + ) class Meta: model = Project @@ -28,7 +38,10 @@ class VersionFilter(filters.FilterSet): field_name='versbose_name', lookup_expr='contains', ) - slug__contains = filters.CharFilter(field_name='slug', lookup_expr='contains') + slug__contains = filters.CharFilter( + field_name='slug', + lookup_expr='contains', + ) class Meta: model = Version diff --git a/readthedocs/v3/renderer.py b/readthedocs/v3/renderer.py index fad520f9c61..a720aae20fd 100644 --- a/readthedocs/v3/renderer.py +++ b/readthedocs/v3/renderer.py @@ -1,6 +1,9 @@ import json + from rest_framework.compat import ( - INDENT_SEPARATORS, LONG_SEPARATORS, SHORT_SEPARATORS, + INDENT_SEPARATORS, + LONG_SEPARATORS, + SHORT_SEPARATORS, ) from rest_framework.renderers import JSONRenderer diff --git a/readthedocs/v3/routers.py b/readthedocs/v3/routers.py index 0bde9f1f020..4059ef38462 100644 --- a/readthedocs/v3/routers.py +++ b/readthedocs/v3/routers.py @@ -1,4 +1,4 @@ -from rest_framework.routers import DefaultRouter, APIRootView +from rest_framework.routers import APIRootView, DefaultRouter from rest_framework_extensions.routers import NestedRouterMixin diff --git a/readthedocs/v3/serializers.py b/readthedocs/v3/serializers.py index 1c9a22b4a4e..94294e59df0 100644 --- a/readthedocs/v3/serializers.py +++ b/readthedocs/v3/serializers.py @@ -2,16 +2,15 @@ import urllib from django.conf import settings -from django.core.urlresolvers import reverse from django.contrib.auth.models import User - +from django.core.urlresolvers import reverse +from rest_flex_fields import FlexFieldsModelSerializer +from rest_flex_fields.serializers import FlexFieldsSerializerMixin from rest_framework import serializers + +from readthedocs.builds.models import Build, Version from readthedocs.projects.constants import LANGUAGES, PROGRAMMING_LANGUAGES from readthedocs.projects.models import Project -from readthedocs.builds.models import Build, Version - -from rest_flex_fields import FlexFieldsModelSerializer -from rest_flex_fields.serializers import FlexFieldsSerializerMixin class UserSerializer(FlexFieldsModelSerializer): @@ -109,7 +108,8 @@ class BuildSerializer(FlexFieldsModelSerializer): expandable_fields = dict( config=( - BuildConfigSerializer, dict( + BuildConfigSerializer, + dict( source='config', ), ), @@ -186,9 +186,7 @@ class VersionURLsSerializer(serializers.Serializer): vcs = serializers.URLField(source='vcs_url') def get_documentation(self, obj): - return obj.project.get_docs_url( - version_slug=obj.slug, - ) + return obj.project.get_docs_url(version_slug=obj.slug,) class VersionSerializer(FlexFieldsModelSerializer): @@ -201,7 +199,8 @@ class VersionSerializer(FlexFieldsModelSerializer): expandable_fields = dict( last_build=( - BuildSerializer, dict( + BuildSerializer, + dict( source='last_build', ), ), @@ -384,13 +383,15 @@ class ProjectSerializer(FlexFieldsModelSerializer): expandable_fields = dict( users=( - UserSerializer, dict( + UserSerializer, + dict( source='users', many=True, ), ), active_versions=( - VersionSerializer, dict( + VersionSerializer, + dict( # NOTE: this has to be a Model method, can't be a # ``SerializerMethodField`` as far as I know source='active_versions', diff --git a/readthedocs/v3/urls.py b/readthedocs/v3/urls.py index 2225051aea9..82428fcd7f5 100644 --- a/readthedocs/v3/urls.py +++ b/readthedocs/v3/urls.py @@ -1,14 +1,9 @@ -from django.conf.urls import url, include - +from django.conf.urls import include, url from rest_framework.documentation import include_docs_urls from .routers import DefaultRouterWithNesting -from .views import ( - BuildsViewSet, - ProjectsViewSet, - VersionsViewSet, - UsersViewSet, -) +from .views import BuildsViewSet, ProjectsViewSet, UsersViewSet, VersionsViewSet + router = DefaultRouterWithNesting() @@ -58,12 +53,15 @@ ) urlpatterns = [ - url(r'^docs/', include_docs_urls( - title='Read the Docs API', - patterns=[ - url(r'/api/v3/', include(router.urls)), - ], - public=True), + url( + r'^docs/', + include_docs_urls( + title='Read the Docs API', + patterns=[ + url(r'/api/v3/', include(router.urls)), + ], + public=True, + ), ), ] diff --git a/readthedocs/v3/views.py b/readthedocs/v3/views.py index c8fd0e684c2..4c38fe9bdbe 100644 --- a/readthedocs/v3/views.py +++ b/readthedocs/v3/views.py @@ -1,25 +1,41 @@ +import django_filters.rest_framework as filters from django.contrib.auth.models import User from django.shortcuts import get_object_or_404 from django.utils.safestring import mark_safe -import django_filters.rest_framework as filters -from rest_framework.authentication import SessionAuthentication, TokenAuthentication +from rest_flex_fields.views import FlexFieldsMixin +from rest_framework.authentication import ( + SessionAuthentication, + TokenAuthentication, +) from rest_framework.decorators import action +from rest_framework.metadata import SimpleMetadata +from rest_framework.mixins import ( + CreateModelMixin, + ListModelMixin, + RetrieveModelMixin, + UpdateModelMixin, +) from rest_framework.permissions import IsAdminUser from rest_framework.renderers import BrowsableAPIRenderer from rest_framework.response import Response -from rest_framework.throttling import UserRateThrottle, AnonRateThrottle -from rest_framework.mixins import ListModelMixin, RetrieveModelMixin, UpdateModelMixin, CreateModelMixin +from rest_framework.throttling import AnonRateThrottle, UserRateThrottle from rest_framework.viewsets import GenericViewSet from rest_framework_extensions.mixins import NestedViewSetMixin -from rest_flex_fields.views import FlexFieldsMixin +from readthedocs.builds.models import Build, Version from readthedocs.core.utils import trigger_build -from readthedocs.builds.models import Version, Build from readthedocs.projects.models import Project -from rest_framework.metadata import SimpleMetadata -from .filters import ProjectFilter, VersionFilter, BuildFilter + +from .filters import BuildFilter, ProjectFilter, VersionFilter from .renderer import AlphaneticalSortedJSONRenderer -from .serializers import ProjectSerializer, VersionSerializer, VersionUpdateSerializer, BuildTriggerSerializer, BuildSerializer, UserSerializer +from .serializers import ( + BuildSerializer, + BuildTriggerSerializer, + ProjectSerializer, + UserSerializer, + VersionSerializer, + VersionUpdateSerializer, +) class APIv3Settings: @@ -32,7 +48,8 @@ class APIv3Settings: metadata_class = SimpleMetadata -class ProjectsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): +class ProjectsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, + ListModelMixin, RetrieveModelMixin, GenericViewSet): # Markdown docstring is automatically rendered by BrowsableAPIRenderer. @@ -109,9 +126,7 @@ def get_view_description(self, *args, **kwargs): project = self.request.user.projects.first() if project: # TODO: make the links clickable when ``kwargs.html=True`` - return mark_safe(description.format( - project_slug=project.slug, - )) + return mark_safe(description.format(project_slug=project.slug)) return description def get_queryset(self): @@ -148,7 +163,9 @@ def _related_projects(self, queryset): return Response(serializer.data) -class VersionsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, UpdateModelMixin, GenericViewSet): +class VersionsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, + ListModelMixin, RetrieveModelMixin, UpdateModelMixin, + GenericViewSet): model = Version lookup_field = 'slug' @@ -196,11 +213,13 @@ def update(self, request, *args, **kwargs): # Browsable API since SessionAuthentication can't be used because we set # ``httpOnly`` on our cookies and the ``PUT/PATCH`` method are triggered # via Javascript - response = super().update(request, *args, **kwargs) + super().update(request, *args, **kwargs) return Response(status=204) -class BuildsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, CreateModelMixin, GenericViewSet): +class BuildsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, + ListModelMixin, RetrieveModelMixin, CreateModelMixin, + GenericViewSet): model = Build lookup_field = 'pk' lookup_url_kwarg = 'build_pk' @@ -260,7 +279,8 @@ def create(self, request, **kwargs): return Response(data=data, status=status) -class UsersViewSet(APIv3Settings, NestedViewSetMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): +class UsersViewSet(APIv3Settings, NestedViewSetMixin, ListModelMixin, + RetrieveModelMixin, GenericViewSet): model = User lookup_field = 'username' lookup_url_kwarg = 'user_username' From 254def30b0781250b4627750231398f2b8e636ba Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 22 Apr 2019 14:58:49 +0200 Subject: [PATCH 37/96] Organize API v1, v2 and v3 structure --- readthedocs/{restapi => api/v1}/__init__.py | 0 readthedocs/{restapi => api/v2}/README.rst | 0 .../{restapi/views => api/v2}/__init__.py | 0 readthedocs/{restapi => api/v2}/client.py | 0 .../{restapi => api/v2}/permissions.py | 0 .../{restapi => api/v2}/serializers.py | 0 readthedocs/{restapi => api/v2}/signals.py | 0 .../v2}/templates/restapi/footer.html | 0 .../v2}/templates/restapi/log.txt | 0 readthedocs/{restapi => api/v2}/urls.py | 2 +- readthedocs/{restapi => api/v2}/utils.py | 0 readthedocs/{v3 => api/v2/views}/__init__.py | 0 .../{restapi => api/v2}/views/core_views.py | 0 .../{restapi => api/v2}/views/footer_views.py | 2 +- .../{restapi => api/v2}/views/integrations.py | 0 .../{restapi => api/v2}/views/model_views.py | 0 .../{restapi => api/v2}/views/task_views.py | 0 .../{v3/migrations => api/v3}/__init__.py | 0 readthedocs/{ => api}/v3/admin.py | 0 readthedocs/{ => api}/v3/apps.py | 0 readthedocs/api/v3/examples/__init__.py | 0 readthedocs/api/v3/examples/client.py | 10 +++ .../api/v3/examples/project_details.py | 81 +++++++++++++++++++ .../api/v3/examples/project_full_details.py | 4 + readthedocs/api/v3/examples/utils.py | 11 +++ readthedocs/{ => api}/v3/filters.py | 0 readthedocs/api/v3/migrations/__init__.py | 0 readthedocs/api/v3/mixins.py | 29 +++++++ readthedocs/{ => api}/v3/models.py | 0 readthedocs/{ => api}/v3/renderer.py | 0 readthedocs/{ => api}/v3/routers.py | 0 readthedocs/{ => api}/v3/serializers.py | 0 readthedocs/{ => api}/v3/tests.py | 0 readthedocs/{ => api}/v3/urls.py | 0 readthedocs/{ => api}/v3/views.py | 0 readthedocs/core/views/hooks.py | 8 +- readthedocs/doc_builder/backends/sphinx.py | 2 +- readthedocs/doc_builder/environments.py | 2 +- readthedocs/oauth/services/github.py | 2 +- readthedocs/projects/models.py | 2 +- readthedocs/projects/tasks.py | 2 +- readthedocs/projects/utils.py | 2 +- readthedocs/rtd_tests/mocks/mock_api.py | 2 +- readthedocs/rtd_tests/tests/test_api.py | 4 +- .../rtd_tests/tests/test_api_permissions.py | 2 +- .../tests/test_api_version_compare.py | 2 +- readthedocs/rtd_tests/tests/test_footer.py | 4 +- .../rtd_tests/tests/test_privacy_urls.py | 4 +- .../rtd_tests/tests/test_restapi_client.py | 2 +- readthedocs/settings/base.py | 6 +- readthedocs/sphinx_domains/api.py | 2 +- readthedocs/urls.py | 4 +- 52 files changed, 162 insertions(+), 29 deletions(-) rename readthedocs/{restapi => api/v1}/__init__.py (100%) rename readthedocs/{restapi => api/v2}/README.rst (100%) rename readthedocs/{restapi/views => api/v2}/__init__.py (100%) rename readthedocs/{restapi => api/v2}/client.py (100%) rename readthedocs/{restapi => api/v2}/permissions.py (100%) rename readthedocs/{restapi => api/v2}/serializers.py (100%) rename readthedocs/{restapi => api/v2}/signals.py (100%) rename readthedocs/{restapi => api/v2}/templates/restapi/footer.html (100%) rename readthedocs/{restapi => api/v2}/templates/restapi/log.txt (100%) rename readthedocs/{restapi => api/v2}/urls.py (98%) rename readthedocs/{restapi => api/v2}/utils.py (100%) rename readthedocs/{v3 => api/v2/views}/__init__.py (100%) rename readthedocs/{restapi => api/v2}/views/core_views.py (100%) rename readthedocs/{restapi => api/v2}/views/footer_views.py (99%) rename readthedocs/{restapi => api/v2}/views/integrations.py (100%) rename readthedocs/{restapi => api/v2}/views/model_views.py (100%) rename readthedocs/{restapi => api/v2}/views/task_views.py (100%) rename readthedocs/{v3/migrations => api/v3}/__init__.py (100%) rename readthedocs/{ => api}/v3/admin.py (100%) rename readthedocs/{ => api}/v3/apps.py (100%) create mode 100644 readthedocs/api/v3/examples/__init__.py create mode 100644 readthedocs/api/v3/examples/client.py create mode 100644 readthedocs/api/v3/examples/project_details.py create mode 100644 readthedocs/api/v3/examples/project_full_details.py create mode 100644 readthedocs/api/v3/examples/utils.py rename readthedocs/{ => api}/v3/filters.py (100%) create mode 100644 readthedocs/api/v3/migrations/__init__.py create mode 100644 readthedocs/api/v3/mixins.py rename readthedocs/{ => api}/v3/models.py (100%) rename readthedocs/{ => api}/v3/renderer.py (100%) rename readthedocs/{ => api}/v3/routers.py (100%) rename readthedocs/{ => api}/v3/serializers.py (100%) rename readthedocs/{ => api}/v3/tests.py (100%) rename readthedocs/{ => api}/v3/urls.py (100%) rename readthedocs/{ => api}/v3/views.py (100%) diff --git a/readthedocs/restapi/__init__.py b/readthedocs/api/v1/__init__.py similarity index 100% rename from readthedocs/restapi/__init__.py rename to readthedocs/api/v1/__init__.py diff --git a/readthedocs/restapi/README.rst b/readthedocs/api/v2/README.rst similarity index 100% rename from readthedocs/restapi/README.rst rename to readthedocs/api/v2/README.rst diff --git a/readthedocs/restapi/views/__init__.py b/readthedocs/api/v2/__init__.py similarity index 100% rename from readthedocs/restapi/views/__init__.py rename to readthedocs/api/v2/__init__.py diff --git a/readthedocs/restapi/client.py b/readthedocs/api/v2/client.py similarity index 100% rename from readthedocs/restapi/client.py rename to readthedocs/api/v2/client.py diff --git a/readthedocs/restapi/permissions.py b/readthedocs/api/v2/permissions.py similarity index 100% rename from readthedocs/restapi/permissions.py rename to readthedocs/api/v2/permissions.py diff --git a/readthedocs/restapi/serializers.py b/readthedocs/api/v2/serializers.py similarity index 100% rename from readthedocs/restapi/serializers.py rename to readthedocs/api/v2/serializers.py diff --git a/readthedocs/restapi/signals.py b/readthedocs/api/v2/signals.py similarity index 100% rename from readthedocs/restapi/signals.py rename to readthedocs/api/v2/signals.py diff --git a/readthedocs/restapi/templates/restapi/footer.html b/readthedocs/api/v2/templates/restapi/footer.html similarity index 100% rename from readthedocs/restapi/templates/restapi/footer.html rename to readthedocs/api/v2/templates/restapi/footer.html diff --git a/readthedocs/restapi/templates/restapi/log.txt b/readthedocs/api/v2/templates/restapi/log.txt similarity index 100% rename from readthedocs/restapi/templates/restapi/log.txt rename to readthedocs/api/v2/templates/restapi/log.txt diff --git a/readthedocs/restapi/urls.py b/readthedocs/api/v2/urls.py similarity index 98% rename from readthedocs/restapi/urls.py rename to readthedocs/api/v2/urls.py index 9958512786a..ea377493b55 100644 --- a/readthedocs/restapi/urls.py +++ b/readthedocs/api/v2/urls.py @@ -7,7 +7,7 @@ from rest_framework import routers from readthedocs.constants import pattern_opts -from readthedocs.restapi.views import ( +from readthedocs.api.v2.views import ( core_views, footer_views, integrations, diff --git a/readthedocs/restapi/utils.py b/readthedocs/api/v2/utils.py similarity index 100% rename from readthedocs/restapi/utils.py rename to readthedocs/api/v2/utils.py diff --git a/readthedocs/v3/__init__.py b/readthedocs/api/v2/views/__init__.py similarity index 100% rename from readthedocs/v3/__init__.py rename to readthedocs/api/v2/views/__init__.py diff --git a/readthedocs/restapi/views/core_views.py b/readthedocs/api/v2/views/core_views.py similarity index 100% rename from readthedocs/restapi/views/core_views.py rename to readthedocs/api/v2/views/core_views.py diff --git a/readthedocs/restapi/views/footer_views.py b/readthedocs/api/v2/views/footer_views.py similarity index 99% rename from readthedocs/restapi/views/footer_views.py rename to readthedocs/api/v2/views/footer_views.py index 1d8115d5835..ecb765bd690 100644 --- a/readthedocs/restapi/views/footer_views.py +++ b/readthedocs/api/v2/views/footer_views.py @@ -15,7 +15,7 @@ highest_version, parse_version_failsafe, ) -from readthedocs.restapi.signals import footer_response +from readthedocs.api.v2.signals import footer_response def get_version_compare_data(project, base_version=None): diff --git a/readthedocs/restapi/views/integrations.py b/readthedocs/api/v2/views/integrations.py similarity index 100% rename from readthedocs/restapi/views/integrations.py rename to readthedocs/api/v2/views/integrations.py diff --git a/readthedocs/restapi/views/model_views.py b/readthedocs/api/v2/views/model_views.py similarity index 100% rename from readthedocs/restapi/views/model_views.py rename to readthedocs/api/v2/views/model_views.py diff --git a/readthedocs/restapi/views/task_views.py b/readthedocs/api/v2/views/task_views.py similarity index 100% rename from readthedocs/restapi/views/task_views.py rename to readthedocs/api/v2/views/task_views.py diff --git a/readthedocs/v3/migrations/__init__.py b/readthedocs/api/v3/__init__.py similarity index 100% rename from readthedocs/v3/migrations/__init__.py rename to readthedocs/api/v3/__init__.py diff --git a/readthedocs/v3/admin.py b/readthedocs/api/v3/admin.py similarity index 100% rename from readthedocs/v3/admin.py rename to readthedocs/api/v3/admin.py diff --git a/readthedocs/v3/apps.py b/readthedocs/api/v3/apps.py similarity index 100% rename from readthedocs/v3/apps.py rename to readthedocs/api/v3/apps.py diff --git a/readthedocs/api/v3/examples/__init__.py b/readthedocs/api/v3/examples/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/readthedocs/api/v3/examples/client.py b/readthedocs/api/v3/examples/client.py new file mode 100644 index 00000000000..2f106ffa91b --- /dev/null +++ b/readthedocs/api/v3/examples/client.py @@ -0,0 +1,10 @@ +import requests +import slumber + + +def setup_api(token): + session = requests.Session() + session.headers.update({'Authorization': f'Token {token}'}) + return slumber.API('http://localhost:8000/api/v3/', session=session) + +api = setup_api(123) diff --git a/readthedocs/api/v3/examples/project_details.py b/readthedocs/api/v3/examples/project_details.py new file mode 100644 index 00000000000..c4624c79086 --- /dev/null +++ b/readthedocs/api/v3/examples/project_details.py @@ -0,0 +1,81 @@ +import time + +from client import api +from utils import p + + +# Get specific project by slug +p(api.projects('test-builds').get()) +input('Press Enter to continue...') + +# Get specific project by slug full expanded +# (all active versions with each last build object and its configuration) +p(api.projects('test-builds').get( + expand='active_versions,active_versions.last_build,active_versions.last_build.config', +)) +input('Press Enter to continue...') + + +# Get all active and built versions for a project selecting only needed fields: +# slug, urls, downloads +# (useful to create the versions menu on a theme) +p(api.projects('test-builds').versions.get( + expand='last_build', + fields='slug,urls,downloads', + active=True, + built=True, # filtering by built we avoid ending up with 404 links +)) +input('Press Enter to continue...') + +# Get all running builds for a project +# (useful for a status page of the project) +p(api.projects('test-builds').builds.get( + running=True, +)) +input('Press Enter to continue...') + +# Get all running builds for specific version of a project +p(api.projects('test-builds').versions('latest').builds.get( + running=True, +)) +input('Press Enter to continue...') + + +# trigger a build of default version and poll the status +# (useful on the release process to check that docs build before publishing) +# response = api.projects('test-builds').builds().post() + +# Trigger a build for a specific version +response = api.projects('test-builds').versions('use-py2').builds().post() +p(response) +if response['triggered']: + finished = response['build']['finished'] + build_id = response['build']['id'] + project_slug = response['project']['slug'] + build_url = response['build']['links']['_self'] + + while not finished: + time.sleep(5) + # NOTE: I already have the url for this on ``build_url`` but as I'm + # using slumber which already have the Authorization header, I don't + # know how to hit it directly and I need to rebuilt it here (this is a + # limitation of the client, not of API design) + response = api.projects(project_slug).builds(build_id).get() + state = response['state']['code'] + finished = response['finished'] + print(f'Current state: {state}') + print('Finished') + +input('Press Enter to continue...') + + +# Activate and make private a specific version of a project +# NOTE: slumber can't be used here since ``.patch`` send the data in the URL +api._store['session'].patch( + api.projects('test-builds').versions('submodule-https-scheme').url(), + data=dict( + active=True, + privacy_level='private', + ), +) +input('Press Enter to continue...') diff --git a/readthedocs/api/v3/examples/project_full_details.py b/readthedocs/api/v3/examples/project_full_details.py new file mode 100644 index 00000000000..11db0c477fa --- /dev/null +++ b/readthedocs/api/v3/examples/project_full_details.py @@ -0,0 +1,4 @@ +from client import api +from utils import p + +p(api.projects('test-builds').get(expand='')) diff --git a/readthedocs/api/v3/examples/utils.py b/readthedocs/api/v3/examples/utils.py new file mode 100644 index 00000000000..361d0689a59 --- /dev/null +++ b/readthedocs/api/v3/examples/utils.py @@ -0,0 +1,11 @@ +import json +from pygments import highlight +from pygments.formatters import TerminalTrueColorFormatter +from pygments.lexers import JsonLexer + + +def p(data): + j = json.dumps(data, sort_keys=True, indent=4) + print( + highlight(j, JsonLexer(), TerminalTrueColorFormatter()) + ) diff --git a/readthedocs/v3/filters.py b/readthedocs/api/v3/filters.py similarity index 100% rename from readthedocs/v3/filters.py rename to readthedocs/api/v3/filters.py diff --git a/readthedocs/api/v3/migrations/__init__.py b/readthedocs/api/v3/migrations/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py new file mode 100644 index 00000000000..73daa23624a --- /dev/null +++ b/readthedocs/api/v3/mixins.py @@ -0,0 +1,29 @@ +from rest_framework.response import Response + + +class UpdateOnlyPatchModelMixin: + + """ + Update a model instance (only allow PATCH method). + + Copied from https://github.com/encode/django-rest-framework/blob/master/rest_framework/mixins.py#L61 + """ + + def partial_update(self, request, *args, **kwargs): + partial = True + instance = self.get_object() + serializer = self.get_serializer( + instance, data=request.data, partial=partial + ) + serializer.is_valid(raise_exception=True) + self.perform_update(serializer) + + if getattr(instance, '_prefetched_objects_cache', None): + # If 'prefetch_related' has been applied to a queryset, we need to + # forcibly invalidate the prefetch cache on the instance. + instance._prefetched_objects_cache = {} + + return Response(serializer.data) + + def perform_update(self, serializer): + serializer.save() diff --git a/readthedocs/v3/models.py b/readthedocs/api/v3/models.py similarity index 100% rename from readthedocs/v3/models.py rename to readthedocs/api/v3/models.py diff --git a/readthedocs/v3/renderer.py b/readthedocs/api/v3/renderer.py similarity index 100% rename from readthedocs/v3/renderer.py rename to readthedocs/api/v3/renderer.py diff --git a/readthedocs/v3/routers.py b/readthedocs/api/v3/routers.py similarity index 100% rename from readthedocs/v3/routers.py rename to readthedocs/api/v3/routers.py diff --git a/readthedocs/v3/serializers.py b/readthedocs/api/v3/serializers.py similarity index 100% rename from readthedocs/v3/serializers.py rename to readthedocs/api/v3/serializers.py diff --git a/readthedocs/v3/tests.py b/readthedocs/api/v3/tests.py similarity index 100% rename from readthedocs/v3/tests.py rename to readthedocs/api/v3/tests.py diff --git a/readthedocs/v3/urls.py b/readthedocs/api/v3/urls.py similarity index 100% rename from readthedocs/v3/urls.py rename to readthedocs/api/v3/urls.py diff --git a/readthedocs/v3/views.py b/readthedocs/api/v3/views.py similarity index 100% rename from readthedocs/v3/views.py rename to readthedocs/api/v3/views.py diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index 3966ba62b2a..c76b96829bd 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -187,7 +187,7 @@ def github_build(request): # noqa: D205 GitHub webhook consumer. .. warning:: **DEPRECATED** - Use :py:class:`readthedocs.restapi.views.integrations.GitHubWebhookView` + Use :py:class:`readthedocs.api.v2.views.integrations.GitHubWebhookView` instead of this view function This will search for projects matching either a stripped down HTTP or SSH @@ -242,7 +242,7 @@ def gitlab_build(request): # noqa: D205 GitLab webhook consumer. .. warning:: **DEPRECATED** - Use :py:class:`readthedocs.restapi.views.integrations.GitLabWebhookView` + Use :py:class:`readthedocs.api.v2.views.integrations.GitLabWebhookView` instead of this view function Search project repository URLs using the site URL from GitLab webhook payload. @@ -277,7 +277,7 @@ def bitbucket_build(request): Consume webhooks from multiple versions of Bitbucket's API. .. warning:: **DEPRECATED** - Use :py:class:`readthedocs.restapi.views.integrations.BitbucketWebhookView` + Use :py:class:`readthedocs.api.v2.views.integrations.BitbucketWebhookView` instead of this view function New webhooks are set up with v2, but v1 webhooks will still point to this @@ -354,7 +354,7 @@ def generic_build(request, project_id_or_slug=None): .. warning:: **DEPRECATED** - Use :py:class:`readthedocs.restapi.views.integrations.GenericWebhookView` + Use :py:class:`readthedocs.api.v2.views.integrations.GenericWebhookView` instead of this view function """ try: diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index a71ac354b4c..71f6f0aa83a 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -23,7 +23,7 @@ from readthedocs.projects.exceptions import ProjectConfigurationError from readthedocs.projects.models import Feature from readthedocs.projects.utils import safe_write -from readthedocs.restapi.client import api +from readthedocs.api.v2.client import api from ..base import BaseBuilder, restoring_chdir from ..constants import PDF_RE diff --git a/readthedocs/doc_builder/environments.py b/readthedocs/doc_builder/environments.py index d93e43bce01..d40a2610ca5 100644 --- a/readthedocs/doc_builder/environments.py +++ b/readthedocs/doc_builder/environments.py @@ -25,7 +25,7 @@ from readthedocs.core.utils import slugify from readthedocs.projects.constants import LOG_TEMPLATE from readthedocs.projects.models import Feature -from readthedocs.restapi.client import api as api_v2 +from readthedocs.api.v2.client import api as api_v2 from .constants import ( DOCKER_HOSTNAME_MAX_LEN, diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index f565b019d60..4b471d0f3b5 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -13,7 +13,7 @@ from readthedocs.builds import utils as build_utils from readthedocs.integrations.models import Integration from readthedocs.integrations.utils import get_secret -from readthedocs.restapi.client import api +from readthedocs.api.v2.client import api from ..models import RemoteOrganization, RemoteRepository from .base import Service, SyncServiceError diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index db9aed3890a..41a7ed78546 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -37,7 +37,7 @@ validate_repository_url, ) from readthedocs.projects.version_handling import determine_stable_version -from readthedocs.restapi.client import api +from readthedocs.api.v2.client import api from readthedocs.search.parse_json import process_file from readthedocs.vcs_support.backends import backend_cls from readthedocs.vcs_support.utils import Lock, NonBlockingLock diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index a240770f3fb..40d055306f2 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -62,7 +62,7 @@ from readthedocs.doc_builder.python_environments import Conda, Virtualenv from readthedocs.sphinx_domains.models import SphinxDomain from readthedocs.projects.models import APIProject -from readthedocs.restapi.client import api as api_v2 +from readthedocs.api.v2.client import api as api_v2 from readthedocs.vcs_support import utils as vcs_support_utils from readthedocs.worker import app diff --git a/readthedocs/projects/utils.py b/readthedocs/projects/utils.py index 254e047516d..fd1916c58d7 100644 --- a/readthedocs/projects/utils.py +++ b/readthedocs/projects/utils.py @@ -14,7 +14,7 @@ # TODO make this a classmethod of Version def version_from_slug(slug, version): from readthedocs.builds.models import Version, APIVersion - from readthedocs.restapi.client import api + from readthedocs.api.v2.client import api if settings.DONT_HIT_DB: version_data = api.version().get( project=slug, diff --git a/readthedocs/rtd_tests/mocks/mock_api.py b/readthedocs/rtd_tests/mocks/mock_api.py index 7a2b09ced99..acd803b1c16 100644 --- a/readthedocs/rtd_tests/mocks/mock_api.py +++ b/readthedocs/rtd_tests/mocks/mock_api.py @@ -89,7 +89,7 @@ def command(self, _): @contextmanager def mock_api(repo): api_mock = MockApi(repo) - with mock.patch('readthedocs.restapi.client.api', api_mock), \ + with mock.patch('readthedocs.api.v2.client.api', api_mock), \ mock.patch('readthedocs.projects.tasks.api_v2', api_mock), \ mock.patch('readthedocs.doc_builder.environments.api_v2', api_mock): yield api_mock diff --git a/readthedocs/rtd_tests/tests/test_api.py b/readthedocs/rtd_tests/tests/test_api.py index af76b6072dd..848e3c52ae0 100644 --- a/readthedocs/rtd_tests/tests/test_api.py +++ b/readthedocs/rtd_tests/tests/test_api.py @@ -23,7 +23,7 @@ Feature, Project, ) -from readthedocs.restapi.views.integrations import ( +from readthedocs.api.v2.views.integrations import ( GITHUB_CREATE, GITHUB_DELETE, GITHUB_EVENT_HEADER, @@ -36,7 +36,7 @@ GitHubWebhookView, GitLabWebhookView, ) -from readthedocs.restapi.views.task_views import get_status_data +from readthedocs.api.v2.views.task_views import get_status_data super_auth = base64.b64encode(b'super:test').decode('utf-8') diff --git a/readthedocs/rtd_tests/tests/test_api_permissions.py b/readthedocs/rtd_tests/tests/test_api_permissions.py index 7367d8f5d6a..959f02b0cae 100644 --- a/readthedocs/rtd_tests/tests/test_api_permissions.py +++ b/readthedocs/rtd_tests/tests/test_api_permissions.py @@ -4,7 +4,7 @@ from mock import Mock -from readthedocs.restapi.permissions import APIRestrictedPermission +from readthedocs.api.v2.permissions import APIRestrictedPermission class APIRestrictedPermissionTests(TestCase): diff --git a/readthedocs/rtd_tests/tests/test_api_version_compare.py b/readthedocs/rtd_tests/tests/test_api_version_compare.py index 24b94ad39d6..85852fd5d10 100644 --- a/readthedocs/rtd_tests/tests/test_api_version_compare.py +++ b/readthedocs/rtd_tests/tests/test_api_version_compare.py @@ -3,7 +3,7 @@ from readthedocs.builds.constants import LATEST from readthedocs.projects.models import Project -from readthedocs.restapi.views.footer_views import get_version_compare_data +from readthedocs.api.v2.views.footer_views import get_version_compare_data class VersionCompareTests(TestCase): diff --git a/readthedocs/rtd_tests/tests/test_footer.py b/readthedocs/rtd_tests/tests/test_footer.py index da78ec3b0c0..87c8360e7bd 100644 --- a/readthedocs/rtd_tests/tests/test_footer.py +++ b/readthedocs/rtd_tests/tests/test_footer.py @@ -7,7 +7,7 @@ from readthedocs.builds.models import Version from readthedocs.core.middleware import FooterNoSessionMiddleware from readthedocs.projects.models import Project -from readthedocs.restapi.views.footer_views import ( +from readthedocs.api.v2.views.footer_views import ( footer_html, get_version_compare_data, ) @@ -46,7 +46,7 @@ def test_footer(self): self.assertEqual(r.status_code, 200) def test_footer_uses_version_compare(self): - version_compare = 'readthedocs.restapi.views.footer_views.get_version_compare_data' # noqa + version_compare = 'readthedocs.api.v2.views.footer_views.get_version_compare_data' # noqa with mock.patch(version_compare) as get_version_compare_data: get_version_compare_data.return_value = { 'MOCKED': True, diff --git a/readthedocs/rtd_tests/tests/test_privacy_urls.py b/readthedocs/rtd_tests/tests/test_privacy_urls.py index 23319888b28..10014c7f655 100644 --- a/readthedocs/rtd_tests/tests/test_privacy_urls.py +++ b/readthedocs/rtd_tests/tests/test_privacy_urls.py @@ -375,9 +375,9 @@ def setUp(self): class APIUnauthAccessTest(APIMixin, TestCase): - @mock.patch('readthedocs.restapi.views.task_views.get_public_task_data') + @mock.patch('readthedocs.api.v2.views.task_views.get_public_task_data') def test_api_urls(self, get_public_task_data): - from readthedocs.restapi.urls import urlpatterns + from readthedocs.api.v2.urls import urlpatterns get_public_task_data.side_effect = TaskNoPermission('Nope') self._test_url(urlpatterns) diff --git a/readthedocs/rtd_tests/tests/test_restapi_client.py b/readthedocs/rtd_tests/tests/test_restapi_client.py index cf555b74883..8e5b5386d53 100644 --- a/readthedocs/rtd_tests/tests/test_restapi_client.py +++ b/readthedocs/rtd_tests/tests/test_restapi_client.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from django.test import TestCase -from readthedocs.restapi.client import DrfJsonSerializer +from readthedocs.api.v2.client import DrfJsonSerializer class TestDrfJsonSerializer(TestCase): diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index d1a3c19dc20..fe4e468c15e 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -127,10 +127,8 @@ def INSTALLED_APPS(self): # noqa 'readthedocs.oauth', 'readthedocs.redirects', 'readthedocs.rtd_tests', - 'readthedocs.restapi', - - # TODO: refactor this module to be ``api.v3`` - 'readthedocs.v3', + 'readthedocs.api.v2', + 'readthedocs.api.v3', 'readthedocs.gold', 'readthedocs.payments', diff --git a/readthedocs/sphinx_domains/api.py b/readthedocs/sphinx_domains/api.py index 285605fa1e2..7857aee7e78 100644 --- a/readthedocs/sphinx_domains/api.py +++ b/readthedocs/sphinx_domains/api.py @@ -2,7 +2,7 @@ from rest_framework import serializers -from readthedocs.restapi.views.model_views import UserSelectViewSet +from readthedocs.api.v2.views.model_views import UserSelectViewSet from .models import SphinxDomain diff --git a/readthedocs/urls.py b/readthedocs/urls.py index 27b29e0b1a0..14b2a7276a4 100644 --- a/readthedocs/urls.py +++ b/readthedocs/urls.py @@ -59,14 +59,14 @@ ] api_urls = [ - url(r'^api/v2/', include('readthedocs.restapi.urls')), + url(r'^api/v2/', include('readthedocs.api.v2.urls')), # Keep the `doc_search` at root level, so the test does not fail for other API url(r'^api/v2/docsearch/$', PageSearchAPIView.as_view(), name='doc_search'), url( r'^api/auth/', include('rest_framework.urls', namespace='rest_framework') ), - url(r'^api/v3/', include('readthedocs.v3.urls')), + url(r'^api/v3/', include('readthedocs.api.v3.urls')), ] i18n_urls = [ From 0b9ab9ef8aa97a594519f264386fc9235b8ef386 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 22 Apr 2019 15:21:20 +0200 Subject: [PATCH 38/96] Revert api auth URL to keep compatibility --- readthedocs/urls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/urls.py b/readthedocs/urls.py index 14b2a7276a4..abf46fd034a 100644 --- a/readthedocs/urls.py +++ b/readthedocs/urls.py @@ -63,7 +63,7 @@ # Keep the `doc_search` at root level, so the test does not fail for other API url(r'^api/v2/docsearch/$', PageSearchAPIView.as_view(), name='doc_search'), url( - r'^api/auth/', + r'^api-auth/', include('rest_framework.urls', namespace='rest_framework') ), url(r'^api/v3/', include('readthedocs.api.v3.urls')), From 7fdec50f8d46f0cc3ad13fed2d061dd062556a78 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 22 Apr 2019 15:23:47 +0200 Subject: [PATCH 39/96] Remove autogenerated URL docs --- readthedocs/api/v3/urls.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/readthedocs/api/v3/urls.py b/readthedocs/api/v3/urls.py index 82428fcd7f5..1223db2541d 100644 --- a/readthedocs/api/v3/urls.py +++ b/readthedocs/api/v3/urls.py @@ -1,6 +1,3 @@ -from django.conf.urls import include, url -from rest_framework.documentation import include_docs_urls - from .routers import DefaultRouterWithNesting from .views import BuildsViewSet, ProjectsViewSet, UsersViewSet, VersionsViewSet @@ -52,17 +49,5 @@ parents_query_lookups=['projects__slug'], ) -urlpatterns = [ - url( - r'^docs/', - include_docs_urls( - title='Read the Docs API', - patterns=[ - url(r'/api/v3/', include(router.urls)), - ], - public=True, - ), - ), -] - +urlpatterns = [] urlpatterns += router.urls From e014d86792fec145df5d32f6875107d8b951a336 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 22 Apr 2019 15:24:04 +0200 Subject: [PATCH 40/96] Cleanup --- readthedocs/api/v3/admin.py | 3 --- readthedocs/api/v3/migrations/__init__.py | 0 readthedocs/api/v3/mixins.py | 29 ----------------------- readthedocs/api/v3/models.py | 3 --- readthedocs/settings/base.py | 4 ---- 5 files changed, 39 deletions(-) delete mode 100644 readthedocs/api/v3/admin.py delete mode 100644 readthedocs/api/v3/migrations/__init__.py delete mode 100644 readthedocs/api/v3/mixins.py delete mode 100644 readthedocs/api/v3/models.py diff --git a/readthedocs/api/v3/admin.py b/readthedocs/api/v3/admin.py deleted file mode 100644 index 8c38f3f3dad..00000000000 --- a/readthedocs/api/v3/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/readthedocs/api/v3/migrations/__init__.py b/readthedocs/api/v3/migrations/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py deleted file mode 100644 index 73daa23624a..00000000000 --- a/readthedocs/api/v3/mixins.py +++ /dev/null @@ -1,29 +0,0 @@ -from rest_framework.response import Response - - -class UpdateOnlyPatchModelMixin: - - """ - Update a model instance (only allow PATCH method). - - Copied from https://github.com/encode/django-rest-framework/blob/master/rest_framework/mixins.py#L61 - """ - - def partial_update(self, request, *args, **kwargs): - partial = True - instance = self.get_object() - serializer = self.get_serializer( - instance, data=request.data, partial=partial - ) - serializer.is_valid(raise_exception=True) - self.perform_update(serializer) - - if getattr(instance, '_prefetched_objects_cache', None): - # If 'prefetch_related' has been applied to a queryset, we need to - # forcibly invalidate the prefetch cache on the instance. - instance._prefetched_objects_cache = {} - - return Response(serializer.data) - - def perform_update(self, serializer): - serializer.save() diff --git a/readthedocs/api/v3/models.py b/readthedocs/api/v3/models.py deleted file mode 100644 index 71a83623907..00000000000 --- a/readthedocs/api/v3/models.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.db import models - -# Create your models here. diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index fe4e468c15e..82fd0b7bf6e 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -1,9 +1,6 @@ # -*- coding: utf-8 -*- # pylint: disable=missing-docstring -from __future__ import ( - absolute_import, division, print_function, unicode_literals) - import getpass import os import datetime @@ -115,7 +112,6 @@ def INSTALLED_APPS(self): # noqa 'django_extensions', 'crispy_forms', 'messages_extends', - 'django_filters', 'django_elasticsearch_dsl', 'django_filters', From 86c63b3526cde602fc59d35b0329535e5aebdaf2 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 22 Apr 2019 15:26:10 +0200 Subject: [PATCH 41/96] Typo fixed --- readthedocs/api/v3/renderer.py | 2 +- readthedocs/api/v3/views.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/api/v3/renderer.py b/readthedocs/api/v3/renderer.py index a720aae20fd..07877244da4 100644 --- a/readthedocs/api/v3/renderer.py +++ b/readthedocs/api/v3/renderer.py @@ -8,7 +8,7 @@ from rest_framework.renderers import JSONRenderer -class AlphaneticalSortedJSONRenderer(JSONRenderer): +class AlphabeticalSortedJSONRenderer(JSONRenderer): """ Renderer that sort they keys from the JSON alphabetically. diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 4c38fe9bdbe..04fc64455be 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -27,7 +27,7 @@ from readthedocs.projects.models import Project from .filters import BuildFilter, ProjectFilter, VersionFilter -from .renderer import AlphaneticalSortedJSONRenderer +from .renderer import AlphabeticalSortedJSONRenderer from .serializers import ( BuildSerializer, BuildTriggerSerializer, @@ -42,7 +42,7 @@ class APIv3Settings: authentication_classes = (SessionAuthentication, TokenAuthentication) permission_classes = (IsAdminUser,) - renderer_classes = (AlphaneticalSortedJSONRenderer, BrowsableAPIRenderer) + renderer_classes = (AlphabeticalSortedJSONRenderer, BrowsableAPIRenderer) throttle_classes = (UserRateThrottle, AnonRateThrottle) filter_backends = (filters.DjangoFilterBackend,) metadata_class = SimpleMetadata From 1e9355c6c7df8ba77c2b7bf8b905cc2cb8fcaf7a Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 22 Apr 2019 15:27:19 +0200 Subject: [PATCH 42/96] Do not expose names on API results --- readthedocs/api/v3/serializers.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index 94294e59df0..9494ef94af3 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -15,18 +15,12 @@ class UserSerializer(FlexFieldsModelSerializer): - # TODO: return ``null`` when ``last_name`` or ``first_name`` are `''``. I'm - # thinking on writing a decorator or similar that dynamically creates the - # methods based on a field with a list - class Meta: model = User fields = [ 'username', 'date_joined', 'last_login', - 'first_name', - 'last_name', ] From 39ff39cde2f9284468a697a1ec31f74dc57ed173 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 22 Apr 2019 15:45:05 +0200 Subject: [PATCH 43/96] Extend docs a little --- readthedocs/api/v3/views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 04fc64455be..8e211b7de65 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -87,6 +87,7 @@ class ProjectsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, ### Examples: * List my projects: ``/api/v3/projects/`` + * List my projects with offset and limit: ``/api/v3/projects/?offset=10&limit=25`` * Filter list: ``/api/v3/projects/?name__contains=test`` * Retrieve only needed data: ``/api/v3/projects/?fields=slug,created`` * Retrieve specific project: ``/api/v3/projects/{project_slug}/`` @@ -94,6 +95,8 @@ class ProjectsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, * Translations of a project: ``/api/v3/projects/{project_slug}/translations/`` * Subprojects of a project: ``/api/v3/projects/{project_slug}/subprojects/`` * Superproject of a project: ``/api/v3/projects/{project_slug}/superproject/`` + + Go to https://docs.readthedocs.io/en/stable/api/v3.html for a complete documentation of the APIv3. """ model = Project From b4b8bab061fd7ce357610bdc7aab644ff707d560 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 22 Apr 2019 15:45:17 +0200 Subject: [PATCH 44/96] Use our own querysets/managers permission backend --- readthedocs/api/v3/views.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 8e211b7de65..44b7e5447a2 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -134,12 +134,16 @@ def get_view_description(self, *args, **kwargs): def get_queryset(self): queryset = super().get_queryset() - return queryset.filter(users=self.request.user) + return queryset.api(user=self.request.user) @action(detail=True, methods=['get']) def translations(self, request, project_slug): project = self.get_object() - return self._related_projects(project.translations.all()) + return self._related_projects( + project.translations.api( + user=request.user, + ), + ) @action(detail=True, methods=['get']) def superproject(self, request, project_slug): @@ -154,7 +158,11 @@ def superproject(self, request, project_slug): @action(detail=True, methods=['get']) def subprojects(self, request, project_slug): project = self.get_object() - return self._related_projects(project.superprojects.all()) + return self._related_projects( + project.superprojects.api( + user=request.user, + ), + ) def _related_projects(self, queryset): page = self.paginate_queryset(queryset) @@ -201,8 +209,7 @@ def get_queryset(self): queryset = super().get_queryset() # we force to filter only by the versions the user has access to - user = self.request.user - queryset = queryset.filter(project__users=user) + queryset = queryset.api(user=self.request.user) return queryset def update(self, request, *args, **kwargs): @@ -237,8 +244,7 @@ def get_queryset(self): queryset = super().get_queryset() # we force to filter only by the versions the user has access to - user = self.request.user - queryset = queryset.filter(project__users=user) + queryset = queryset.api(user=self.request.user) return queryset def get_serializer_class(self): @@ -294,7 +300,8 @@ def get_queryset(self): # ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` queryset = super().get_queryset() - # we force to filter only by the projects the user has access to - user = self.request.user - queryset = queryset.filter(projects__in=user.projects.all()).distinct() + # the user can only see profiles from people they share a project with + queryset = queryset.filter( + projects__in=Project.objects.api(user=self.request.user), + ).distinct() return queryset From b676988d86ce8fcbbedad97560c07c88f1720f47 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 22 Apr 2019 15:48:30 +0200 Subject: [PATCH 45/96] Rename field --- readthedocs/api/v3/serializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index 9494ef94af3..7b08945131c 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -271,9 +271,9 @@ def get_name(self, programming_language): class ProjectURLsSerializer(serializers.Serializer): documentation = serializers.CharField(source='get_docs_url') - project = serializers.SerializerMethodField() + project_homepage = serializers.SerializerMethodField() - def get_project(self, obj): + def get_project_homepage(self, obj): # Overridden only to return ``None`` when the description is ``''`` return obj.project_url or None From 1029b2d3d7864799e06d8cdda821fbec4e3a8240 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 22 Apr 2019 15:48:38 +0200 Subject: [PATCH 46/96] Typo --- readthedocs/api/v3/filters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/api/v3/filters.py b/readthedocs/api/v3/filters.py index f670a2531c0..177759804ef 100644 --- a/readthedocs/api/v3/filters.py +++ b/readthedocs/api/v3/filters.py @@ -35,7 +35,7 @@ class Meta: class VersionFilter(filters.FilterSet): verbose_name__contains = filters.CharFilter( - field_name='versbose_name', + field_name='verbose_name', lookup_expr='contains', ) slug__contains = filters.CharFilter( From db51c32a69517d3162f821d96a744626309fd570 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 23 Apr 2019 19:28:37 +0200 Subject: [PATCH 47/96] Handle detail/list permissions in a Mixin --- readthedocs/api/v3/mixins.py | 8 +++++ readthedocs/api/v3/urls.py | 2 ++ readthedocs/api/v3/views.py | 52 +++++++++++-------------------- readthedocs/builds/querysets.py | 20 +++++++++--- readthedocs/projects/querysets.py | 10 ++++-- 5 files changed, 53 insertions(+), 39 deletions(-) create mode 100644 readthedocs/api/v3/mixins.py diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py new file mode 100644 index 00000000000..4bc6a02f309 --- /dev/null +++ b/readthedocs/api/v3/mixins.py @@ -0,0 +1,8 @@ +class APIAuthMixin: + + def get_queryset(self): + # ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` + # we need to have defined the class attribute as ``queryset = Model.objects.all()`` + queryset = super().get_queryset() + + return queryset.api(user=self.request.user, detail=self.detail) diff --git a/readthedocs/api/v3/urls.py b/readthedocs/api/v3/urls.py index 1223db2541d..5f08bc18e53 100644 --- a/readthedocs/api/v3/urls.py +++ b/readthedocs/api/v3/urls.py @@ -42,6 +42,8 @@ parents_query_lookups=['project__slug'], ) +# allows /api/v3/projects/pip/users/ +# allows /api/v3/projects/pip/users/humitos/ projects.register( r'users', UsersViewSet, diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 44b7e5447a2..27444c92a6d 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -8,6 +8,7 @@ TokenAuthentication, ) from rest_framework.decorators import action +from rest_framework.exceptions import PermissionDenied from rest_framework.metadata import SimpleMetadata from rest_framework.mixins import ( CreateModelMixin, @@ -27,6 +28,7 @@ from readthedocs.projects.models import Project from .filters import BuildFilter, ProjectFilter, VersionFilter +from .mixins import APIAuthMixin from .renderer import AlphabeticalSortedJSONRenderer from .serializers import ( BuildSerializer, @@ -48,8 +50,9 @@ class APIv3Settings: metadata_class = SimpleMetadata -class ProjectsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, - ListModelMixin, RetrieveModelMixin, GenericViewSet): +class ProjectsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, + FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, + GenericViewSet): # Markdown docstring is automatically rendered by BrowsableAPIRenderer. @@ -132,10 +135,6 @@ def get_view_description(self, *args, **kwargs): return mark_safe(description.format(project_slug=project.slug)) return description - def get_queryset(self): - queryset = super().get_queryset() - return queryset.api(user=self.request.user) - @action(detail=True, methods=['get']) def translations(self, request, project_slug): project = self.get_object() @@ -174,9 +173,9 @@ def _related_projects(self, queryset): return Response(serializer.data) -class VersionsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, - ListModelMixin, RetrieveModelMixin, UpdateModelMixin, - GenericViewSet): +class VersionsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, + FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, + UpdateModelMixin, GenericViewSet): model = Version lookup_field = 'slug' @@ -204,14 +203,6 @@ def get_serializer_class(self): return VersionSerializer return VersionUpdateSerializer - def get_queryset(self): - # ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` - queryset = super().get_queryset() - - # we force to filter only by the versions the user has access to - queryset = queryset.api(user=self.request.user) - return queryset - def update(self, request, *args, **kwargs): """ Make PUT/PATCH behaves in the same way. @@ -227,9 +218,9 @@ def update(self, request, *args, **kwargs): return Response(status=204) -class BuildsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, - ListModelMixin, RetrieveModelMixin, CreateModelMixin, - GenericViewSet): +class BuildsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, + FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, + CreateModelMixin, GenericViewSet): model = Build lookup_field = 'pk' lookup_url_kwarg = 'build_pk' @@ -239,14 +230,6 @@ class BuildsViewSet(APIv3Settings, NestedViewSetMixin, FlexFieldsMixin, 'config', ] - def get_queryset(self): - # ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` - queryset = super().get_queryset() - - # we force to filter only by the versions the user has access to - queryset = queryset.api(user=self.request.user) - return queryset - def get_serializer_class(self): if self.action in ('list', 'retrieve'): return BuildSerializer @@ -300,8 +283,11 @@ def get_queryset(self): # ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` queryset = super().get_queryset() - # the user can only see profiles from people they share a project with - queryset = queryset.filter( - projects__in=Project.objects.api(user=self.request.user), - ).distinct() - return queryset + if self.detail: + return queryset + + # give access to the user if it's maintainer of the project + if self.request.user in self.model.objects.filter(**self.get_parents_query_dict()): + return queryset + + raise PermissionDenied diff --git a/readthedocs/builds/querysets.py b/readthedocs/builds/querysets.py index ae7e5a8fb79..f5d8b360b5c 100644 --- a/readthedocs/builds/querysets.py +++ b/readthedocs/builds/querysets.py @@ -58,8 +58,14 @@ def private(self, user=None, project=None, only_active=True): queryset = queryset.filter(active=True) return queryset - def api(self, user=None): - return self.public(user, only_active=False) + def api(self, user=None, detail=True): + if detail: + return self.public(user, only_active=False) + + queryset = self.none() + if user: + queryset = self._add_user_repos(queryset, user) + return queryset def for_project(self, project): """Return all versions for a project, including translations.""" @@ -100,8 +106,14 @@ def public(self, user=None, project=None): queryset = queryset.filter(project=project) return queryset - def api(self, user=None): - return self.public(user) + def api(self, user=None, detail=True): + if detail: + return self.public(user) + + queryset = self.none() + if user: + queryset = self._add_user_repos(queryset, user) + return queryset class BuildQuerySet(SettingsOverrideObject): diff --git a/readthedocs/projects/querysets.py b/readthedocs/projects/querysets.py index 3d04667d0ff..388179f68f3 100644 --- a/readthedocs/projects/querysets.py +++ b/readthedocs/projects/querysets.py @@ -82,8 +82,14 @@ def is_active(self, project): def dashboard(self, user=None): return self.for_admin_user(user) - def api(self, user=None): - return self.public(user) + def api(self, user=None, detail=True): + if detail: + return self.public(user) + + queryset = self.none() + if user: + return self._add_user_repos(queryset, user) + return queryset class ProjectQuerySet(SettingsOverrideObject): From 2249f6ebfc9fd81d41d1944f71e363c8f6a0c7b2 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Tue, 23 Apr 2019 19:38:02 +0200 Subject: [PATCH 48/96] Allow authenticated users and authenticate them only via Token --- readthedocs/api/v3/views.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 27444c92a6d..f4203e8e27d 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -16,7 +16,7 @@ RetrieveModelMixin, UpdateModelMixin, ) -from rest_framework.permissions import IsAdminUser +from rest_framework.permissions import IsAuthenticated from rest_framework.renderers import BrowsableAPIRenderer from rest_framework.response import Response from rest_framework.throttling import AnonRateThrottle, UserRateThrottle @@ -42,8 +42,11 @@ class APIv3Settings: - authentication_classes = (SessionAuthentication, TokenAuthentication) - permission_classes = (IsAdminUser,) + # Using only ``TokenAuthentication`` for now, so we can give access to + # specific carefully selected users only + authentication_classes = (TokenAuthentication,) + + permission_classes = (IsAuthenticated,) renderer_classes = (AlphabeticalSortedJSONRenderer, BrowsableAPIRenderer) throttle_classes = (UserRateThrottle, AnonRateThrottle) filter_backends = (filters.DjangoFilterBackend,) From 289d927e1171fce9d52bf940ce2e9ddcf88ead74 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 24 Apr 2019 12:32:11 +0200 Subject: [PATCH 49/96] Use proper queryset for ProjectViewSet.subprojects --- readthedocs/api/v3/views.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index f4203e8e27d..8c2e4ee2b3f 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -160,11 +160,15 @@ def superproject(self, request, project_slug): @action(detail=True, methods=['get']) def subprojects(self, request, project_slug): project = self.get_object() - return self._related_projects( - project.superprojects.api( + queryset = self.get_queryset().filter( + pk__in=project.subprojects.api( user=request.user, - ), + # ``detail`` is not implemented in + # ``RelatedProjectQuerySetBase`` yet + # detail=self.detail, + ).values_list('child__pk', flat=True), ) + return self._related_projects(queryset) def _related_projects(self, queryset): page = self.paginate_queryset(queryset) From eb3d358c8571e99eb030686e9170c5628be8bf07 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 24 Apr 2019 12:33:00 +0200 Subject: [PATCH 50/96] Use pagination_class on APIv3 settings --- readthedocs/api/v3/views.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 8c2e4ee2b3f..bb43836f9da 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -16,6 +16,7 @@ RetrieveModelMixin, UpdateModelMixin, ) +from rest_framework.pagination import LimitOffsetPagination from rest_framework.permissions import IsAuthenticated from rest_framework.renderers import BrowsableAPIRenderer from rest_framework.response import Response @@ -42,11 +43,26 @@ class APIv3Settings: + """ + Django REST Framework settings for APIv3. + + Override global DRF settings for APIv3 in particular. All ViewSet should + inherit from this class to share/apply the same settings all over the APIv3. + + .. note:: + + The only settings used from ``settings.REST_FRAMEWORK`` is + ``DEFAULT_THROTTLE_RATES`` since it's not possible to define here. + """ + # Using only ``TokenAuthentication`` for now, so we can give access to # specific carefully selected users only authentication_classes = (TokenAuthentication,) - permission_classes = (IsAuthenticated,) + + pagination_class = LimitOffsetPagination + LimitOffsetPagination.default_limit = 10 + renderer_classes = (AlphabeticalSortedJSONRenderer, BrowsableAPIRenderer) throttle_classes = (UserRateThrottle, AnonRateThrottle) filter_backends = (filters.DjangoFilterBackend,) From bbe4113ef2100327882b34e82328d4bc780f8011 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 24 Apr 2019 14:33:42 +0200 Subject: [PATCH 51/96] Different approach to centralize authentication on APIAuthMixin --- readthedocs/api/v3/mixins.py | 57 ++++++++++++++++++++++++++++++++++-- readthedocs/api/v3/views.py | 56 ++++++++++++++++------------------- 2 files changed, 80 insertions(+), 33 deletions(-) diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py index 4bc6a02f309..48d631408f8 100644 --- a/readthedocs/api/v3/mixins.py +++ b/readthedocs/api/v3/mixins.py @@ -1,8 +1,59 @@ -class APIAuthMixin: +from django.contrib.auth.models import User +from django.shortcuts import get_object_or_404 +from rest_framework.exceptions import PermissionDenied +from rest_framework_extensions.settings import extensions_api_settings + +from readthedocs.projects.models import Project + + +class NestedParentProjectMixin: + + def get_parent_project(self): + + project_slug = None + for kwarg_name, kwarg_value in self.kwargs.items(): + if kwarg_name.startswith(extensions_api_settings.DEFAULT_PARENT_LOOKUP_KWARG_NAME_PREFIX) and 'project' in kwarg_name: + project_slug = kwarg_value + + return get_object_or_404(Project, slug=project_slug) + + +class APIAuthMixin(NestedParentProjectMixin): + + """ + Mixin to define queryset permissions for ViewSet only in one place. + + All APIv3 ViewSet should inherit this mixin, unless specific permissions + required. In that case, an specific mixin for that case should be defined. + """ def get_queryset(self): - # ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` + """ + Filter results based on user permissions. + + 1. filters by parent ``project_slug`` (NestedViewSetMixin). + 2. return those results if it's a detail view. + 3. if it's a list view, it checks if the user is admin of the parent + object (project) and return the same results. + 4. raise a ``PermissionDenied`` exception if the user is not an admin. + """ + + # NOTE: ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` # we need to have defined the class attribute as ``queryset = Model.objects.all()`` queryset = super().get_queryset() - return queryset.api(user=self.request.user, detail=self.detail) + # Filter results by user + # NOTE: we don't override the manager in User model, so we don't have + # ``.api`` method there + if self.model is not User: + queryset = queryset.api(user=self.request.user) + + # Detail requests are public + if self.detail: + return queryset + + # List view are only allowed if user is owner + if self.get_parent_project() in Project.objects.for_admin_user(user=self.request.user): + return queryset + + raise PermissionDenied diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index bb43836f9da..cd96c918686 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -4,7 +4,6 @@ from django.utils.safestring import mark_safe from rest_flex_fields.views import FlexFieldsMixin from rest_framework.authentication import ( - SessionAuthentication, TokenAuthentication, ) from rest_framework.decorators import action @@ -134,9 +133,6 @@ class ProjectsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, 'active_versions.last_build.config', ] - # NOTE: accessing a existent project when we don't have permissions to - # access it, returns 404 instead of 403. - def get_view_description(self, *args, **kwargs): """ Make valid links for the user's documentation browseable API. @@ -176,15 +172,28 @@ def superproject(self, request, project_slug): @action(detail=True, methods=['get']) def subprojects(self, request, project_slug): project = self.get_object() - queryset = self.get_queryset().filter( - pk__in=project.subprojects.api( - user=request.user, - # ``detail`` is not implemented in - # ``RelatedProjectQuerySetBase`` yet - # detail=self.detail, - ).values_list('child__pk', flat=True), - ) - return self._related_projects(queryset) + # queryset = self.get_queryset().filter( + # pk__in=project.subprojects.api( + # user=request.user, + # # ``detail`` is not implemented in + # # ``RelatedProjectQuerySetBase`` yet + # # detail=self.detail, + # ).values_list('child__pk', flat=True), + # ) + # return self._related_projects(queryset) + + # HACK: ``NestedRouterMixin`` does not generate the proper URL when + # ``detail=False`` on the decorator. + self.detail = False + + if project in self.get_queryset(): + queryset = self.get_queryset().filter( + pk__in=project.subprojects.all().values_list( + 'child__pk', flat=True), + ) + return self._related_projects(queryset) + + raise PermissionDenied def _related_projects(self, queryset): page = self.paginate_queryset(queryset) @@ -196,9 +205,9 @@ def _related_projects(self, queryset): return Response(serializer.data) -class VersionsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, - FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, - UpdateModelMixin, GenericViewSet): +class VersionsViewSet(APIv3Settings, APIAuthMixin, + NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, + RetrieveModelMixin, UpdateModelMixin, GenericViewSet): model = Version lookup_field = 'slug' @@ -294,23 +303,10 @@ def create(self, request, **kwargs): return Response(data=data, status=status) -class UsersViewSet(APIv3Settings, NestedViewSetMixin, ListModelMixin, +class UsersViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): model = User lookup_field = 'username' lookup_url_kwarg = 'user_username' serializer_class = UserSerializer queryset = User.objects.all() - - def get_queryset(self): - # ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` - queryset = super().get_queryset() - - if self.detail: - return queryset - - # give access to the user if it's maintainer of the project - if self.request.user in self.model.objects.filter(**self.get_parents_query_dict()): - return queryset - - raise PermissionDenied From 512b5df74f3253559850b9ca57c3ce6e5ee5164b Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 24 Apr 2019 17:52:50 +0200 Subject: [PATCH 52/96] Tests showing small problems on some endpoints --- pytest.ini | 3 + readthedocs/api/v3/tests.py | 130 +++++++++++++++++++++++++++++++++++- 2 files changed, 132 insertions(+), 1 deletion(-) diff --git a/pytest.ini b/pytest.ini index 1c0ef5e8ae6..ad73d65a6ff 100644 --- a/pytest.ini +++ b/pytest.ini @@ -15,3 +15,6 @@ filterwarnings = ignore:Pagination may yield inconsistent results with an unordered object_list.*:django.core.paginator.UnorderedObjectListWarning # docutils ignore:'U' mode is deprecated:DeprecationWarning + + # rest_framework_extensions.routers raises these + ignore::rest_framework.RemovedInDRF311Warning diff --git a/readthedocs/api/v3/tests.py b/readthedocs/api/v3/tests.py index 7ce503c2dd9..0f471398ec4 100644 --- a/readthedocs/api/v3/tests.py +++ b/readthedocs/api/v3/tests.py @@ -1,3 +1,131 @@ +import pytest +from django.contrib.auth.models import User from django.test import TestCase +from django.urls import reverse -# Create your tests here. +import django_dynamic_fixture as fixture +from rest_framework.authtoken.models import Token +from rest_framework.test import APIClient + +from readthedocs.builds.models import Version +from readthedocs.projects.models import Project + + +class APIEndpointTests(TestCase): + + def setUp(self): + + self.me = fixture.get(User) + self.token = fixture.get(Token, key='me', user=self.me) + self.project = fixture.get(Project, slug='project', users=[self.me]) + self.subproject = fixture.get(Project, slug='subproject') + # self.translation = fixture.get(Project, slug='translation') + + self.project.add_subproject(self.subproject) + # self.project.add_translation(self.translation) + + self.version = fixture.get(Version, slug='v1.0', project=self.project) + + self.other = fixture.get(User) + self.others_token = fixture.get(Token, key='other', user=self.other) + self.others_project = fixture.get(Project, slug='others_project', users=[self.other]) + + self.client = APIClient() + + @pytest.mark.xfail(strict=True) + def test_list_my_projects(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse('projects-list'), + ) + + # returns 404 because ``get_parent_project`` does not found any project. + self.assertEqual(response.status_code, 200) + + @pytest.mark.xfail(strict=True) + def test_subprojects(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-subprojects', + kwargs={ + 'project_slug': self.project.slug, + }), + + ) + + # ``get_parent_object`` is not receiving any ``parent_lookup_project__slug`` kwargs + self.assertEqual(response.status_code, 200) + + def test_detail_own_project(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-detail', + kwargs={ + 'project_slug': self.project.slug, + }), + + ) + self.assertEqual(response.status_code, 200) + + def test_detail_others_project(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-detail', + kwargs={ + 'project_slug': self.others_project.slug, + }), + + ) + self.assertEqual(response.status_code, 200) + + def test_unauthed_detail_others_project(self): + response = self.client.get( + reverse( + 'projects-detail', + kwargs={ + 'project_slug': self.others_project.slug, + }), + + ) + self.assertEqual(response.status_code, 401) + + def test_detail_nonexistent_project(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-detail', + kwargs={ + 'project_slug': 'nonexistent', + }), + + ) + self.assertEqual(response.status_code, 404) + + def test_detail_version_of_project(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-versions-detail', + kwargs={ + 'parent_lookup_project__slug': self.project.slug, + 'version_slug': 'v1.0', + }), + + ) + self.assertEqual(response.status_code, 200) + + def test_detail_version_of_nonexistent_project(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-versions-detail', + kwargs={ + 'parent_lookup_project__slug': 'nonexistent', + 'version_slug': 'latest', + }), + + ) + self.assertEqual(response.status_code, 404) From 995fd09d5436bf15f176c50e359c67e6502e429a Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 24 Apr 2019 18:39:22 +0200 Subject: [PATCH 53/96] Define /projects/pip/subprojects/ as a nested listing view --- readthedocs/api/v3/mixins.py | 1 - readthedocs/api/v3/serializers.py | 4 +-- readthedocs/api/v3/tests.py | 4 +-- readthedocs/api/v3/urls.py | 10 ++++++- readthedocs/api/v3/views.py | 44 ++++++++++++------------------- 5 files changed, 30 insertions(+), 33 deletions(-) diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py index 48d631408f8..a159fbff81a 100644 --- a/readthedocs/api/v3/mixins.py +++ b/readthedocs/api/v3/mixins.py @@ -9,7 +9,6 @@ class NestedParentProjectMixin: def get_parent_project(self): - project_slug = None for kwarg_name, kwarg_value in self.kwargs.items(): if kwarg_name.startswith(extensions_api_settings.DEFAULT_PARENT_LOOKUP_KWARG_NAME_PREFIX) and 'project' in kwarg_name: diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index 7b08945131c..5d2720938bb 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -328,9 +328,9 @@ def get_builds(self, obj): def get_subprojects(self, obj): path = reverse( - 'projects-subprojects', + 'projects-subprojects-list', kwargs={ - 'project_slug': obj.slug, + 'parent_lookup_parent__slug': obj.slug, }, ) return self._absolute_url(path) diff --git a/readthedocs/api/v3/tests.py b/readthedocs/api/v3/tests.py index 0f471398ec4..aa406490a8d 100644 --- a/readthedocs/api/v3/tests.py +++ b/readthedocs/api/v3/tests.py @@ -47,9 +47,9 @@ def test_subprojects(self): self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') response = self.client.get( reverse( - 'projects-subprojects', + 'projects-subprojects-list', kwargs={ - 'project_slug': self.project.slug, + 'parent_lookup_parent__slug': self.project.slug, }), ) diff --git a/readthedocs/api/v3/urls.py b/readthedocs/api/v3/urls.py index 5f08bc18e53..e3b73701838 100644 --- a/readthedocs/api/v3/urls.py +++ b/readthedocs/api/v3/urls.py @@ -1,5 +1,5 @@ from .routers import DefaultRouterWithNesting -from .views import BuildsViewSet, ProjectsViewSet, UsersViewSet, VersionsViewSet +from .views import BuildsViewSet, ProjectsViewSet, UsersViewSet, VersionsViewSet, SubprojectRelationshipViewSet router = DefaultRouterWithNesting() @@ -12,6 +12,14 @@ basename='projects', ) +# allows /api/v3/projects/pip/subprojects/ +subprojects = projects.register( + r'subprojects', + SubprojectRelationshipViewSet, + base_name='projects-subprojects', + parents_query_lookups=['parent__slug'], +) + # allows /api/v3/projects/pip/versions/ # allows /api/v3/projects/pip/versions/latest/ versions = projects.register( diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index cd96c918686..34381a27979 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -25,7 +25,7 @@ from readthedocs.builds.models import Build, Version from readthedocs.core.utils import trigger_build -from readthedocs.projects.models import Project +from readthedocs.projects.models import Project, ProjectRelationship from .filters import BuildFilter, ProjectFilter, VersionFilter from .mixins import APIAuthMixin @@ -169,32 +169,6 @@ def superproject(self, request, project_slug): return Response(data) return Response(status=404) - @action(detail=True, methods=['get']) - def subprojects(self, request, project_slug): - project = self.get_object() - # queryset = self.get_queryset().filter( - # pk__in=project.subprojects.api( - # user=request.user, - # # ``detail`` is not implemented in - # # ``RelatedProjectQuerySetBase`` yet - # # detail=self.detail, - # ).values_list('child__pk', flat=True), - # ) - # return self._related_projects(queryset) - - # HACK: ``NestedRouterMixin`` does not generate the proper URL when - # ``detail=False`` on the decorator. - self.detail = False - - if project in self.get_queryset(): - queryset = self.get_queryset().filter( - pk__in=project.subprojects.all().values_list( - 'child__pk', flat=True), - ) - return self._related_projects(queryset) - - raise PermissionDenied - def _related_projects(self, queryset): page = self.paginate_queryset(queryset) if page is not None: @@ -205,6 +179,22 @@ def _related_projects(self, queryset): return Response(serializer.data) +class SubprojectRelationshipViewSet(APIv3Settings, APIAuthMixin, + NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, + RetrieveModelMixin, GenericViewSet): + + model = ProjectRelationship + lookup_field = 'child__slug' + lookup_url_kwarg = 'project_slug' + serializer_class = ProjectSerializer + queryset = ProjectRelationship.objects.all() + + def get_queryset(self): + # HACK: to use the same ProjectSerializer over ProjectRelationship + queryset = super().get_queryset() + return [related.child for related in queryset] + + class VersionsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, UpdateModelMixin, GenericViewSet): From b3736cbacaf83f449c6c789c2efb1925df489599 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 11:09:46 +0200 Subject: [PATCH 54/96] Refactor and exception for /api/v3/projects/ listing --- readthedocs/api/v3/mixins.py | 31 ++++++++++++++++++++++++------- readthedocs/api/v3/tests.py | 7 ------- readthedocs/api/v3/views.py | 1 - 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py index a159fbff81a..db80e3cb71d 100644 --- a/readthedocs/api/v3/mixins.py +++ b/readthedocs/api/v3/mixins.py @@ -1,18 +1,28 @@ from django.contrib.auth.models import User from django.shortcuts import get_object_or_404 from rest_framework.exceptions import PermissionDenied -from rest_framework_extensions.settings import extensions_api_settings from readthedocs.projects.models import Project class NestedParentProjectMixin: - def get_parent_project(self): + # Lookup names defined on ``readthedocs/api/v3/urls.py`` when defining the + # mapping between URLs and views through the router. + LOOKUP_NAMES = [ + 'project__slug', + 'projects__slug', + 'parent__slug', + ] + + def _get_parent_project(self): project_slug = None - for kwarg_name, kwarg_value in self.kwargs.items(): - if kwarg_name.startswith(extensions_api_settings.DEFAULT_PARENT_LOOKUP_KWARG_NAME_PREFIX) and 'project' in kwarg_name: - project_slug = kwarg_value + query_dict = self.get_parents_query_dict() + for lookup in self.LOOKUP_NAMES: + value = query_dict.get(lookup) + if value: + project_slug = value + break return get_object_or_404(Project, slug=project_slug) @@ -51,8 +61,15 @@ def get_queryset(self): if self.detail: return queryset - # List view are only allowed if user is owner - if self.get_parent_project() in Project.objects.for_admin_user(user=self.request.user): + allowed_projects = Project.objects.for_admin_user(user=self.request.user) + + # Allow hitting ``/api/v3/projects/`` to list their own projects + if self.basename == 'projects' and self.action == 'list': + return allowed_projects + + # List view are only allowed if user is owner of parent project + project = self._get_parent_project() + if project in allowed_projects: return queryset raise PermissionDenied diff --git a/readthedocs/api/v3/tests.py b/readthedocs/api/v3/tests.py index aa406490a8d..39c29eba998 100644 --- a/readthedocs/api/v3/tests.py +++ b/readthedocs/api/v3/tests.py @@ -1,4 +1,3 @@ -import pytest from django.contrib.auth.models import User from django.test import TestCase from django.urls import reverse @@ -32,17 +31,13 @@ def setUp(self): self.client = APIClient() - @pytest.mark.xfail(strict=True) def test_list_my_projects(self): self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') response = self.client.get( reverse('projects-list'), ) - - # returns 404 because ``get_parent_project`` does not found any project. self.assertEqual(response.status_code, 200) - @pytest.mark.xfail(strict=True) def test_subprojects(self): self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') response = self.client.get( @@ -53,8 +48,6 @@ def test_subprojects(self): }), ) - - # ``get_parent_object`` is not receiving any ``parent_lookup_project__slug`` kwargs self.assertEqual(response.status_code, 200) def test_detail_own_project(self): diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 34381a27979..bfd651cbd28 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -7,7 +7,6 @@ TokenAuthentication, ) from rest_framework.decorators import action -from rest_framework.exceptions import PermissionDenied from rest_framework.metadata import SimpleMetadata from rest_framework.mixins import ( CreateModelMixin, From 894ba159efb7a53f404fa3da99a18178d6745f32 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 11:40:09 +0200 Subject: [PATCH 55/96] Make /projects/pip/translations/ a proper listing view (nested) --- readthedocs/api/v3/mixins.py | 1 + readthedocs/api/v3/serializers.py | 4 ++-- readthedocs/api/v3/urls.py | 11 ++++++++++- readthedocs/api/v3/views.py | 32 ++++++++++++------------------- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py index db80e3cb71d..3600e461f80 100644 --- a/readthedocs/api/v3/mixins.py +++ b/readthedocs/api/v3/mixins.py @@ -13,6 +13,7 @@ class NestedParentProjectMixin: 'project__slug', 'projects__slug', 'parent__slug', + 'main_language_project__slug', ] def _get_parent_project(self): diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index 5d2720938bb..4a903012295 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -346,9 +346,9 @@ def get_superproject(self, obj): def get_translations(self, obj): path = reverse( - 'projects-translations', + 'projects-translations-list', kwargs={ - 'project_slug': obj.slug, + 'parent_lookup_main_language_project__slug': obj.slug, }, ) return self._absolute_url(path) diff --git a/readthedocs/api/v3/urls.py b/readthedocs/api/v3/urls.py index e3b73701838..75906911f91 100644 --- a/readthedocs/api/v3/urls.py +++ b/readthedocs/api/v3/urls.py @@ -1,11 +1,12 @@ from .routers import DefaultRouterWithNesting -from .views import BuildsViewSet, ProjectsViewSet, UsersViewSet, VersionsViewSet, SubprojectRelationshipViewSet +from .views import BuildsViewSet, ProjectsViewSet, UsersViewSet, VersionsViewSet, SubprojectRelationshipViewSet, TranslationRelationshipViewSet router = DefaultRouterWithNesting() # allows /api/v3/projects/ # allows /api/v3/projects/pip/ +# allows /api/v3/projects/pip/superproject/ projects = router.register( r'projects', ProjectsViewSet, @@ -20,6 +21,14 @@ parents_query_lookups=['parent__slug'], ) +# allows /api/v3/projects/pip/translations/ +translations = projects.register( + r'translations', + TranslationRelationshipViewSet, + base_name='projects-translations', + parents_query_lookups=['main_language_project__slug'], +) + # allows /api/v3/projects/pip/versions/ # allows /api/v3/projects/pip/versions/latest/ versions = projects.register( diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index bfd651cbd28..465cd802419 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -149,38 +149,19 @@ def get_view_description(self, *args, **kwargs): return mark_safe(description.format(project_slug=project.slug)) return description - @action(detail=True, methods=['get']) - def translations(self, request, project_slug): - project = self.get_object() - return self._related_projects( - project.translations.api( - user=request.user, - ), - ) - @action(detail=True, methods=['get']) def superproject(self, request, project_slug): project = self.get_object() superproject = getattr(project, 'main_project', None) - data = None if superproject: data = self.get_serializer(superproject).data return Response(data) return Response(status=404) - def _related_projects(self, queryset): - page = self.paginate_queryset(queryset) - if page is not None: - serializer = self.get_serializer(page, many=True) - return self.get_paginated_response(serializer.data) - - serializer = self.get_serializer(queryset, many=True) - return Response(serializer.data) - class SubprojectRelationshipViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, - RetrieveModelMixin, GenericViewSet): + GenericViewSet): model = ProjectRelationship lookup_field = 'child__slug' @@ -194,6 +175,17 @@ def get_queryset(self): return [related.child for related in queryset] +class TranslationRelationshipViewSet(APIv3Settings, APIAuthMixin, + NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, + GenericViewSet): + + model = Project + lookup_field = 'slug' + lookup_url_kwarg = 'project_slug' + serializer_class = ProjectSerializer + queryset = Project.objects.all() + + class VersionsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, UpdateModelMixin, GenericViewSet): From 13e4d6620a7f805a7af605f8503db27fdb3ced1a Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 11:47:02 +0200 Subject: [PATCH 56/96] Better query/modeling for subprojects --- readthedocs/api/v3/mixins.py | 2 +- readthedocs/api/v3/serializers.py | 2 +- readthedocs/api/v3/tests.py | 2 +- readthedocs/api/v3/urls.py | 2 +- readthedocs/api/v3/views.py | 11 +++-------- 5 files changed, 7 insertions(+), 12 deletions(-) diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py index 3600e461f80..0c63503911f 100644 --- a/readthedocs/api/v3/mixins.py +++ b/readthedocs/api/v3/mixins.py @@ -12,7 +12,7 @@ class NestedParentProjectMixin: LOOKUP_NAMES = [ 'project__slug', 'projects__slug', - 'parent__slug', + 'subprojects__parent__slug', 'main_language_project__slug', ] diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index 4a903012295..98df915b1b4 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -330,7 +330,7 @@ def get_subprojects(self, obj): path = reverse( 'projects-subprojects-list', kwargs={ - 'parent_lookup_parent__slug': obj.slug, + 'parent_lookup_subprojects__parent__slug': obj.slug, }, ) return self._absolute_url(path) diff --git a/readthedocs/api/v3/tests.py b/readthedocs/api/v3/tests.py index 39c29eba998..d4d415b09f4 100644 --- a/readthedocs/api/v3/tests.py +++ b/readthedocs/api/v3/tests.py @@ -44,7 +44,7 @@ def test_subprojects(self): reverse( 'projects-subprojects-list', kwargs={ - 'parent_lookup_parent__slug': self.project.slug, + 'parent_lookup_subprojects__parent__slug': self.project.slug, }), ) diff --git a/readthedocs/api/v3/urls.py b/readthedocs/api/v3/urls.py index 75906911f91..78ef5769aeb 100644 --- a/readthedocs/api/v3/urls.py +++ b/readthedocs/api/v3/urls.py @@ -18,7 +18,7 @@ r'subprojects', SubprojectRelationshipViewSet, base_name='projects-subprojects', - parents_query_lookups=['parent__slug'], + parents_query_lookups=['subprojects__parent__slug'], ) # allows /api/v3/projects/pip/translations/ diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 465cd802419..0b84bab4419 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -163,16 +163,11 @@ class SubprojectRelationshipViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, GenericViewSet): - model = ProjectRelationship - lookup_field = 'child__slug' + model = Project + lookup_field = 'slug' lookup_url_kwarg = 'project_slug' serializer_class = ProjectSerializer - queryset = ProjectRelationship.objects.all() - - def get_queryset(self): - # HACK: to use the same ProjectSerializer over ProjectRelationship - queryset = super().get_queryset() - return [related.child for related in queryset] + queryset = Project.objects.all() class TranslationRelationshipViewSet(APIv3Settings, APIAuthMixin, From 9c0dda65706ef57c8c0f2262248c252774a14ebc Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 12:05:48 +0200 Subject: [PATCH 57/96] Fix subprojects query --- readthedocs/api/v3/mixins.py | 2 +- readthedocs/api/v3/serializers.py | 2 +- readthedocs/api/v3/tests.py | 2 +- readthedocs/api/v3/urls.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py index 0c63503911f..9eddbc7982f 100644 --- a/readthedocs/api/v3/mixins.py +++ b/readthedocs/api/v3/mixins.py @@ -12,7 +12,7 @@ class NestedParentProjectMixin: LOOKUP_NAMES = [ 'project__slug', 'projects__slug', - 'subprojects__parent__slug', + 'superprojects__parent__slug', 'main_language_project__slug', ] diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index 98df915b1b4..2dfb2c03e53 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -330,7 +330,7 @@ def get_subprojects(self, obj): path = reverse( 'projects-subprojects-list', kwargs={ - 'parent_lookup_subprojects__parent__slug': obj.slug, + 'parent_lookup_superprojects__parent__slug': obj.slug, }, ) return self._absolute_url(path) diff --git a/readthedocs/api/v3/tests.py b/readthedocs/api/v3/tests.py index d4d415b09f4..2012e09092e 100644 --- a/readthedocs/api/v3/tests.py +++ b/readthedocs/api/v3/tests.py @@ -44,7 +44,7 @@ def test_subprojects(self): reverse( 'projects-subprojects-list', kwargs={ - 'parent_lookup_subprojects__parent__slug': self.project.slug, + 'parent_lookup_superprojects__parent__slug': self.project.slug, }), ) diff --git a/readthedocs/api/v3/urls.py b/readthedocs/api/v3/urls.py index 78ef5769aeb..708188e1afa 100644 --- a/readthedocs/api/v3/urls.py +++ b/readthedocs/api/v3/urls.py @@ -18,7 +18,7 @@ r'subprojects', SubprojectRelationshipViewSet, base_name='projects-subprojects', - parents_query_lookups=['subprojects__parent__slug'], + parents_query_lookups=['superprojects__parent__slug'], ) # allows /api/v3/projects/pip/translations/ From 6abd0538c186b49a112832275c9de6a997e1eb7d Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 12:06:01 +0200 Subject: [PATCH 58/96] Render subproject/superproject in detail --- readthedocs/api/v3/serializers.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index 2dfb2c03e53..f37623526e4 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -427,13 +427,11 @@ def get_description(self, obj): return obj.description or None def get_translation_of(self, obj): - try: - return obj.main_language_project.slug - except Exception: - return None + if obj.main_language_project: + return self.__class__(obj.main_language_project).data def get_subproject_of(self, obj): try: - return obj.superprojects.first().slug + return self.__class__(obj.superprojects.first().parent).data except Exception: return None From e57843ec25df0b5147b74a2bd8ea004ec8f0a757 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 12:07:57 +0200 Subject: [PATCH 59/96] Remove unused import --- readthedocs/api/v3/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 0b84bab4419..cb19db3f0df 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -24,7 +24,7 @@ from readthedocs.builds.models import Build, Version from readthedocs.core.utils import trigger_build -from readthedocs.projects.models import Project, ProjectRelationship +from readthedocs.projects.models import Project from .filters import BuildFilter, ProjectFilter, VersionFilter from .mixins import APIAuthMixin From b473ddc8906d13f46b8a763412bc568f4a70299a Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 12:17:32 +0200 Subject: [PATCH 60/96] Properly generation of vcs_url for Version object --- readthedocs/builds/models.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 3bea7868de0..512c35258d5 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -137,6 +137,11 @@ def ref(self): @property def vcs_url(self): + """ + Generate VCS (github, gitlab, bitbucket) URL for this version. + + Example: https://github.com/rtfd/readthedocs.org/tree/3.4.2/. + """ url = '' if self.slug == STABLE: slug_url = self.ref @@ -145,10 +150,10 @@ def vcs_url(self): else: slug_url = self.slug - if self.project.repo_type in ('git', 'gitlab'): + if ('github' in self.project.repo) or ('gitlab' in self.project.repo): url = f'/tree/{slug_url}/' - if self.project.repo_type == 'bitbucket': + if 'bitbucket' in self.project.repo: slug_url = self.identifier url = f'/src/{slug_url}' From df82ddd9b9e48e41a288943e2f6d25837addbb6c Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 13:03:53 +0200 Subject: [PATCH 61/96] Prefetch relations on Project queryset --- readthedocs/api/v3/views.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index cb19db3f0df..c1aa2347fe1 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -132,6 +132,17 @@ class ProjectsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, 'active_versions.last_build.config', ] + def get_queryset(self): + # This could be a class attribute and managed on the ``APIAuthMixin`` in + # case we want to extend the ``prefetch_related`` to other views as + # well. + queryset = super().get_queryset() + return queryset.prefetch_related( + 'related_projects', + 'domains', + 'tags', + ) + def get_view_description(self, *args, **kwargs): """ Make valid links for the user's documentation browseable API. From f519787b08a3697d59aa2f6b0fc43622aa1d4a10 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 13:19:13 +0200 Subject: [PATCH 62/96] Serializer docstrings --- readthedocs/api/v3/serializers.py | 23 ++++++++++++++++++++++- readthedocs/api/v3/views.py | 4 ++-- 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index f37623526e4..f2948a10b46 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -32,7 +32,13 @@ def _absolute_url(self, path): return urllib.parse.urlunparse((scheme, domain, path, '', '', '')) -class BuildTriggerSerializer(serializers.ModelSerializer): +class BuildCreateSerializer(serializers.ModelSerializer): + + """ + Used when triggering (create action) a ``Build`` for a specific ``Version``. + + This serializer validates that no field is sent at all in the request. + """ class Meta: model = Build @@ -76,6 +82,15 @@ def get_project(self, obj): class BuildConfigSerializer(FlexFieldsSerializerMixin, serializers.Serializer): + """ + Render ``Build.config`` property without modifying it. + + .. note:: + + Any change on the output of that property will be reflected here, + which may produce incompatible changes in the API. + """ + def to_representation(self, obj): # For now, we want to return the ``config`` object as it is without # manipulating it. @@ -231,6 +246,12 @@ def get_downloads(self, obj): class VersionUpdateSerializer(serializers.ModelSerializer): + """ + Used when modifying (update action) a ``Version``. + + It only allows to make the Version active/non-active and private/public. + """ + class Meta: model = Version fields = [ diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index c1aa2347fe1..610821e697d 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -31,7 +31,7 @@ from .renderer import AlphabeticalSortedJSONRenderer from .serializers import ( BuildSerializer, - BuildTriggerSerializer, + BuildCreateSerializer, ProjectSerializer, UserSerializer, VersionSerializer, @@ -252,7 +252,7 @@ class BuildsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, def get_serializer_class(self): if self.action in ('list', 'retrieve'): return BuildSerializer - return BuildTriggerSerializer + return BuildCreateSerializer def create(self, request, **kwargs): parent_lookup_project__slug = kwargs.get('parent_lookup_project__slug') From 854530fa308d8356170a7fa97ad591303e060024 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 14:12:26 +0200 Subject: [PATCH 63/96] Rely on a custom permission class We need both things, 1. permission class to handle methods that does not pass over `get_queryset` (POST, for example) 2. a mixin with `get_queryset` to return the proper queryset for the logged in user --- readthedocs/api/v3/mixins.py | 89 ++++++++++++++++++++----------- readthedocs/api/v3/permissions.py | 20 +++++++ readthedocs/api/v3/views.py | 20 ++----- 3 files changed, 83 insertions(+), 46 deletions(-) create mode 100644 readthedocs/api/v3/permissions.py diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py index 9eddbc7982f..10daa195f72 100644 --- a/readthedocs/api/v3/mixins.py +++ b/readthedocs/api/v3/mixins.py @@ -1,34 +1,45 @@ from django.contrib.auth.models import User from django.shortcuts import get_object_or_404 -from rest_framework.exceptions import PermissionDenied +from rest_framework.exceptions import NotFound +from readthedocs.builds.models import Version from readthedocs.projects.models import Project -class NestedParentProjectMixin: +class NestedParentObjectMixin: # Lookup names defined on ``readthedocs/api/v3/urls.py`` when defining the # mapping between URLs and views through the router. - LOOKUP_NAMES = [ + PROJECT_LOOKUP_NAMES = [ 'project__slug', 'projects__slug', 'superprojects__parent__slug', 'main_language_project__slug', ] - def _get_parent_project(self): + VERSION_LOOKUP_NAMES = [ + 'version__slug', + ] + + def _get_parent_object(self, model, lookup_names): project_slug = None query_dict = self.get_parents_query_dict() - for lookup in self.LOOKUP_NAMES: + for lookup in lookup_names: value = query_dict.get(lookup) if value: - project_slug = value + slug = value break - return get_object_or_404(Project, slug=project_slug) + return get_object_or_404(model, slug=slug) + + def _get_parent_project(self): + return self._get_parent_object(Project, self.PROJECT_LOOKUP_NAMES) + def _get_parent_version(self): + return self._get_parent_object(Version, self.VERSION_LOOKUP_NAMES) -class APIAuthMixin(NestedParentProjectMixin): + +class APIAuthMixin(NestedParentObjectMixin): """ Mixin to define queryset permissions for ViewSet only in one place. @@ -37,40 +48,58 @@ class APIAuthMixin(NestedParentProjectMixin): required. In that case, an specific mixin for that case should be defined. """ + def detail_objects(self, queryset, user): + # Filter results by user + # NOTE: we don't override the manager in User model, so we don't have + # ``.api`` method there + if self.model is not User: + queryset = queryset.api(user=user) + + return queryset + + def listing_objects(self, queryset, user): + project = self._get_parent_project() + if self.has_admin_permission(user, project): + return queryset + + def has_admin_permission(self, user, project): + if project in self.admin_projects(user): + return True + + return False + + def admin_projects(self, user): + return Project.objects.for_admin_user(user=user) + def get_queryset(self): """ Filter results based on user permissions. - 1. filters by parent ``project_slug`` (NestedViewSetMixin). - 2. return those results if it's a detail view. - 3. if it's a list view, it checks if the user is admin of the parent - object (project) and return the same results. - 4. raise a ``PermissionDenied`` exception if the user is not an admin. + 1. returns ``Projects`` where the user is admin if ``/projects/`` is hit + 2. filters by parent ``project_slug`` (NestedViewSetMixin) + 2. returns ``detail_objects`` results if it's a detail view + 3. returns ``listing_objects`` results if it's a listing view + 4. raise a ``NotFound`` exception otherwise """ + # Allow hitting ``/api/v3/projects/`` to list their own projects + if self.basename == 'projects' and self.action == 'list': + # We force returning ``Project`` objects here because it's under the + # ``projects`` view. This could be moved to a specific + # ``get_queryset`` in the view. + return self.admin_projects(self.request.user) + # NOTE: ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` # we need to have defined the class attribute as ``queryset = Model.objects.all()`` queryset = super().get_queryset() - # Filter results by user - # NOTE: we don't override the manager in User model, so we don't have - # ``.api`` method there - if self.model is not User: - queryset = queryset.api(user=self.request.user) - # Detail requests are public if self.detail: - return queryset - - allowed_projects = Project.objects.for_admin_user(user=self.request.user) - - # Allow hitting ``/api/v3/projects/`` to list their own projects - if self.basename == 'projects' and self.action == 'list': - return allowed_projects + return self.detail_objects(queryset, self.request.user) # List view are only allowed if user is owner of parent project - project = self._get_parent_project() - if project in allowed_projects: - return queryset + listing_objects = self.listing_objects(queryset, self.request.user) + if listing_objects: + return listing_objects - raise PermissionDenied + raise NotFound diff --git a/readthedocs/api/v3/permissions.py b/readthedocs/api/v3/permissions.py new file mode 100644 index 00000000000..09fa44e5d20 --- /dev/null +++ b/readthedocs/api/v3/permissions.py @@ -0,0 +1,20 @@ +from rest_framework.permissions import IsAuthenticated + + +class PublicDetailPrivateListing(IsAuthenticated): + + def has_permission(self, request, view): + is_authenticated = super().has_permission(request, view) + if is_authenticated: + if view.basename == 'projects' and view.action == 'list': + # hitting ``/projects/``, allowing + return True + + if view.detail: + return True + + project = view._get_parent_project() + if view.has_admin_permission(request.user, project): + return True + + return False diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 610821e697d..227b1198739 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -28,6 +28,7 @@ from .filters import BuildFilter, ProjectFilter, VersionFilter from .mixins import APIAuthMixin +from .permissions import PublicDetailPrivateListing from .renderer import AlphabeticalSortedJSONRenderer from .serializers import ( BuildSerializer, @@ -56,7 +57,7 @@ class APIv3Settings: # Using only ``TokenAuthentication`` for now, so we can give access to # specific carefully selected users only authentication_classes = (TokenAuthentication,) - permission_classes = (IsAuthenticated,) + permission_classes = (PublicDetailPrivateListing,) pagination_class = LimitOffsetPagination LimitOffsetPagination.default_limit = 10 @@ -255,21 +256,8 @@ def get_serializer_class(self): return BuildCreateSerializer def create(self, request, **kwargs): - parent_lookup_project__slug = kwargs.get('parent_lookup_project__slug') - parent_lookup_version__slug = kwargs.get('parent_lookup_version__slug') - - version = None - project = get_object_or_404( - Project, - slug=parent_lookup_project__slug, - users=request.user, - ) - - if parent_lookup_version__slug: - version = get_object_or_404( - project.versions.all(), - slug=parent_lookup_version__slug, - ) + project = self._get_parent_project() + version = self._get_parent_version() _, build = trigger_build(project, version=version) From 4406687889b7288ef3596e026a1e2cf98b3f1ce3 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 14:14:55 +0200 Subject: [PATCH 64/96] Some docstrings --- readthedocs/api/v3/permissions.py | 8 ++++++++ readthedocs/api/v3/views.py | 20 +++++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/readthedocs/api/v3/permissions.py b/readthedocs/api/v3/permissions.py index 09fa44e5d20..d3e686aa08e 100644 --- a/readthedocs/api/v3/permissions.py +++ b/readthedocs/api/v3/permissions.py @@ -3,6 +3,14 @@ class PublicDetailPrivateListing(IsAuthenticated): + """ + Permission class for our custom use case. + + * Always give permission for a ``detail`` request + * Only give permission for ``listing`` request if user is admin of the project + * Allow access to ``/projects`` (user's projects listing) + """ + def has_permission(self, request, view): is_authenticated = super().has_permission(request, view) if is_authenticated: diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 227b1198739..cd94e2ea39c 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -175,6 +175,13 @@ class SubprojectRelationshipViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, GenericViewSet): + """ + List subprojects of a ``Project``. + + The main query is done via the ``NestedViewSetMixin`` using the + ``parents_query_lookups`` defined when registering the urls. + """ + model = Project lookup_field = 'slug' lookup_url_kwarg = 'project_slug' @@ -186,6 +193,13 @@ class TranslationRelationshipViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, GenericViewSet): + """ + List translations of a ``Project``. + + The main query is done via the ``NestedViewSetMixin`` using the + ``parents_query_lookups`` defined when registering the urls. + """ + model = Project lookup_field = 'slug' lookup_url_kwarg = 'project_slug' @@ -211,10 +225,6 @@ class VersionsViewSet(APIv3Settings, APIAuthMixin, 'last_build.config', ] - # NOTE: ``NestedViewSetMixin`` is really good, but if the ``project.slug`` - # does not exist it does not return 404, but 200 instead: - # /api/v3/projects/nonexistent/versions/ - def get_serializer_class(self): """ Return correct serializer depending on the action (GET or PUT/PATCH/POST). @@ -230,7 +240,7 @@ def update(self, request, *args, **kwargs): Force to return 204 is the update was good. """ - # NOTE: ``Authorization: `` is mandatory to use this method from + # NOTE: ``Authorization:`` header is mandatory to use this method from # Browsable API since SessionAuthentication can't be used because we set # ``httpOnly`` on our cookies and the ``PUT/PATCH`` method are triggered # via Javascript From e6585e2c1481db5af2e39c3d791da13096d2fa0c Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 15:47:15 +0200 Subject: [PATCH 65/96] Cleanup after rebase --- readthedocs/api/v1/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 readthedocs/api/v1/__init__.py diff --git a/readthedocs/api/v1/__init__.py b/readthedocs/api/v1/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 From ebbe613ca2ea8aff9cd0dbf67e2c9cd1eaa86498 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 17:40:00 +0200 Subject: [PATCH 66/96] Response (list/detail) test cases --- readthedocs/api/v3/tests.py | 124 ---------- readthedocs/api/v3/tests/__init__.py | 0 .../v3/tests/responses/projects-detail.json | 98 ++++++++ .../api/v3/tests/responses/projects-list.json | 53 ++++ .../responses/projects-subprojects-list.json | 94 +++++++ .../responses/projects-versions-detail.json | 25 ++ readthedocs/api/v3/tests/test_projects.py | 231 ++++++++++++++++++ 7 files changed, 501 insertions(+), 124 deletions(-) delete mode 100644 readthedocs/api/v3/tests.py create mode 100644 readthedocs/api/v3/tests/__init__.py create mode 100644 readthedocs/api/v3/tests/responses/projects-detail.json create mode 100644 readthedocs/api/v3/tests/responses/projects-list.json create mode 100644 readthedocs/api/v3/tests/responses/projects-subprojects-list.json create mode 100644 readthedocs/api/v3/tests/responses/projects-versions-detail.json create mode 100644 readthedocs/api/v3/tests/test_projects.py diff --git a/readthedocs/api/v3/tests.py b/readthedocs/api/v3/tests.py deleted file mode 100644 index 2012e09092e..00000000000 --- a/readthedocs/api/v3/tests.py +++ /dev/null @@ -1,124 +0,0 @@ -from django.contrib.auth.models import User -from django.test import TestCase -from django.urls import reverse - -import django_dynamic_fixture as fixture -from rest_framework.authtoken.models import Token -from rest_framework.test import APIClient - -from readthedocs.builds.models import Version -from readthedocs.projects.models import Project - - -class APIEndpointTests(TestCase): - - def setUp(self): - - self.me = fixture.get(User) - self.token = fixture.get(Token, key='me', user=self.me) - self.project = fixture.get(Project, slug='project', users=[self.me]) - self.subproject = fixture.get(Project, slug='subproject') - # self.translation = fixture.get(Project, slug='translation') - - self.project.add_subproject(self.subproject) - # self.project.add_translation(self.translation) - - self.version = fixture.get(Version, slug='v1.0', project=self.project) - - self.other = fixture.get(User) - self.others_token = fixture.get(Token, key='other', user=self.other) - self.others_project = fixture.get(Project, slug='others_project', users=[self.other]) - - self.client = APIClient() - - def test_list_my_projects(self): - self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') - response = self.client.get( - reverse('projects-list'), - ) - self.assertEqual(response.status_code, 200) - - def test_subprojects(self): - self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') - response = self.client.get( - reverse( - 'projects-subprojects-list', - kwargs={ - 'parent_lookup_superprojects__parent__slug': self.project.slug, - }), - - ) - self.assertEqual(response.status_code, 200) - - def test_detail_own_project(self): - self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') - response = self.client.get( - reverse( - 'projects-detail', - kwargs={ - 'project_slug': self.project.slug, - }), - - ) - self.assertEqual(response.status_code, 200) - - def test_detail_others_project(self): - self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') - response = self.client.get( - reverse( - 'projects-detail', - kwargs={ - 'project_slug': self.others_project.slug, - }), - - ) - self.assertEqual(response.status_code, 200) - - def test_unauthed_detail_others_project(self): - response = self.client.get( - reverse( - 'projects-detail', - kwargs={ - 'project_slug': self.others_project.slug, - }), - - ) - self.assertEqual(response.status_code, 401) - - def test_detail_nonexistent_project(self): - self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') - response = self.client.get( - reverse( - 'projects-detail', - kwargs={ - 'project_slug': 'nonexistent', - }), - - ) - self.assertEqual(response.status_code, 404) - - def test_detail_version_of_project(self): - self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') - response = self.client.get( - reverse( - 'projects-versions-detail', - kwargs={ - 'parent_lookup_project__slug': self.project.slug, - 'version_slug': 'v1.0', - }), - - ) - self.assertEqual(response.status_code, 200) - - def test_detail_version_of_nonexistent_project(self): - self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') - response = self.client.get( - reverse( - 'projects-versions-detail', - kwargs={ - 'parent_lookup_project__slug': 'nonexistent', - 'version_slug': 'latest', - }), - - ) - self.assertEqual(response.status_code, 404) diff --git a/readthedocs/api/v3/tests/__init__.py b/readthedocs/api/v3/tests/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/readthedocs/api/v3/tests/responses/projects-detail.json b/readthedocs/api/v3/tests/responses/projects-detail.json new file mode 100644 index 00000000000..0ee649be291 --- /dev/null +++ b/readthedocs/api/v3/tests/responses/projects-detail.json @@ -0,0 +1,98 @@ +{ + "active_versions": [ + { + "active": true, + "built": true, + "downloads": {}, + "id": 3, + "identifier": "a1b2c3", + "last_build": { + "builder": "builder01", + "cold_storage": null, + "commit": "a1b2c3", + "config": { + "property": "test value" + }, + "created": "2019-04-29T10:00:00Z", + "duration": 60, + "error": "", + "finished": "2019-04-29T10:01:00Z", + "id": 1, + "links": { + "_self": "https://readthedocs.org/api/v3/projects/project/builds/1/", + "project": "https://readthedocs.org/api/v3/projects/project/", + "version": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/" + }, + "project": "project", + "state": { + "code": "finished", + "name": "Finished" + }, + "success": true, + "version": "v1.0" + }, + "links": { + "_self": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/", + "builds": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/builds/", + "project": "https://readthedocs.org/api/v3/projects/project/" + }, + "privacy_level": { + "code": "public", + "name": "Public" + }, + "ref": null, + "slug": "v1.0", + "type": "tag", + "uploaded": false, + "urls": { + "documentation": "http://readthedocs.org/docs/project/en/v1.0/", + "vcs": "https://github.com/rtfd/project/tree/v1.0/" + }, + "verbose_name": "v1.0" + } + ], + "created": "2019-04-29T10:00:00Z", + "default_branch": "master", + "default_version": "latest", + "description": "Project description", + "id": 1, + "language": { + "code": "en", + "name": "English" + }, + "links": { + "_self": "https://readthedocs.org/api/v3/projects/project/", + "builds": "https://readthedocs.org/api/v3/projects/project/builds/", + "subprojects": "https://readthedocs.org/api/v3/projects/project/subprojects/", + "superproject": "https://readthedocs.org/api/v3/projects/project/superproject/", + "translations": "https://readthedocs.org/api/v3/projects/project/translations/", + "users": "https://readthedocs.org/api/v3/projects/project/users/", + "versions": "https://readthedocs.org/api/v3/projects/project/versions/" + }, + "modified": "2019-04-29T12:00:00Z", + "name": "project", + "privacy_level": { + "code": "public", + "name": "Public" + }, + "programming_language": { + "code": "words", + "name": "Only Words" + }, + "repository": { + "type": "git", + "url": "https://github.com/rtfd/project" + }, + "slug": "project", + "subproject_of": null, + "tags": [ + "tag", + "project", + "test" + ], + "translation_of": null, + "urls": { + "documentation": "http://readthedocs.org/docs/project/en/latest/", + "project_homepage": "http://project.com" + } +} diff --git a/readthedocs/api/v3/tests/responses/projects-list.json b/readthedocs/api/v3/tests/responses/projects-list.json new file mode 100644 index 00000000000..8fcd6134c11 --- /dev/null +++ b/readthedocs/api/v3/tests/responses/projects-list.json @@ -0,0 +1,53 @@ +{ + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "id": 1, + "name": "project", + "slug": "project", + "description": "Project description", + "created": "2019-04-29T10:00:00Z", + "modified": "2019-04-29T12:00:00Z", + "language": { + "code": "en", + "name": "English" + }, + "programming_language": { + "code": "words", + "name": "Only Words" + }, + "repository": { + "url": "https://github.com/rtfd/project", + "type": "git" + }, + "default_version": "latest", + "default_branch": "master", + "privacy_level": { + "code": "public", + "name": "Public" + }, + "subproject_of": null, + "translation_of": null, + "urls": { + "documentation": "http://readthedocs.org/docs/project/en/latest/", + "project_homepage": "http://project.com" + }, + "tags": [ + "tag", + "project", + "test" + ], + "links": { + "_self": "https://readthedocs.org/api/v3/projects/project/", + "users": "https://readthedocs.org/api/v3/projects/project/users/", + "versions": "https://readthedocs.org/api/v3/projects/project/versions/", + "builds": "https://readthedocs.org/api/v3/projects/project/builds/", + "subprojects": "https://readthedocs.org/api/v3/projects/project/subprojects/", + "superproject": "https://readthedocs.org/api/v3/projects/project/superproject/", + "translations": "https://readthedocs.org/api/v3/projects/project/translations/" + } + } + ] +} diff --git a/readthedocs/api/v3/tests/responses/projects-subprojects-list.json b/readthedocs/api/v3/tests/responses/projects-subprojects-list.json new file mode 100644 index 00000000000..dc0d9d08c3a --- /dev/null +++ b/readthedocs/api/v3/tests/responses/projects-subprojects-list.json @@ -0,0 +1,94 @@ +{ + "count": 1, + "next": null, + "previous": null, + "results": [ + { + "id": 2, + "name": "subproject", + "slug": "subproject", + "description": "SubProject description", + "created": "2019-04-29T10:00:00Z", + "modified": "2019-04-29T12:00:00Z", + "language": { + "code": "en", + "name": "English" + }, + "programming_language": { + "code": "words", + "name": "Only Words" + }, + "repository": { + "url": "https://github.com/rtfd/subproject", + "type": "git" + }, + "default_version": "latest", + "default_branch": "master", + "privacy_level": { + "code": "public", + "name": "Public" + }, + "subproject_of": { + "id": 1, + "name": "project", + "slug": "project", + "description": "Project description", + "created": "2019-04-29T10:00:00Z", + "modified": "2019-04-29T12:00:00Z", + "language": { + "code": "en", + "name": "English" + }, + "programming_language": { + "code": "words", + "name": "Only Words" + }, + "repository": { + "url": "https://github.com/rtfd/project", + "type": "git" + }, + "default_version": "latest", + "default_branch": "master", + "privacy_level": { + "code": "public", + "name": "Public" + }, + "subproject_of": null, + "translation_of": null, + "urls": { + "documentation": "http://readthedocs.org/docs/project/en/latest/", + "project_homepage": "http://project.com" + }, + "tags": [ + "tag", + "project", + "test" + ], + "links": { + "_self": "https://readthedocs.org/api/v3/projects/project/", + "users": "https://readthedocs.org/api/v3/projects/project/users/", + "versions": "https://readthedocs.org/api/v3/projects/project/versions/", + "builds": "https://readthedocs.org/api/v3/projects/project/builds/", + "subprojects": "https://readthedocs.org/api/v3/projects/project/subprojects/", + "superproject": "https://readthedocs.org/api/v3/projects/project/superproject/", + "translations": "https://readthedocs.org/api/v3/projects/project/translations/" + } + }, + "translation_of": null, + "urls": { + "documentation": "http://readthedocs.org/docs/project/projects/subproject/en/latest/", + "project_homepage": "http://subproject.com" + }, + "tags": [], + "links": { + "_self": "https://readthedocs.org/api/v3/projects/subproject/", + "users": "https://readthedocs.org/api/v3/projects/subproject/users/", + "versions": "https://readthedocs.org/api/v3/projects/subproject/versions/", + "builds": "https://readthedocs.org/api/v3/projects/subproject/builds/", + "subprojects": "https://readthedocs.org/api/v3/projects/subproject/subprojects/", + "superproject": "https://readthedocs.org/api/v3/projects/subproject/superproject/", + "translations": "https://readthedocs.org/api/v3/projects/subproject/translations/" + } + } + ] +} diff --git a/readthedocs/api/v3/tests/responses/projects-versions-detail.json b/readthedocs/api/v3/tests/responses/projects-versions-detail.json new file mode 100644 index 00000000000..e7677416097 --- /dev/null +++ b/readthedocs/api/v3/tests/responses/projects-versions-detail.json @@ -0,0 +1,25 @@ +{ + "active": true, + "built": true, + "downloads": {}, + "id": 3, + "identifier": "a1b2c3", + "links": { + "_self": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/", + "builds": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/builds/", + "project": "https://readthedocs.org/api/v3/projects/project/" + }, + "privacy_level": { + "code": "public", + "name": "Public" + }, + "ref": null, + "slug": "v1.0", + "type": "tag", + "uploaded": false, + "urls": { + "documentation": "http://readthedocs.org/docs/project/en/v1.0/", + "vcs": "https://github.com/rtfd/project/tree/v1.0/" + }, + "verbose_name": "v1.0" +} diff --git a/readthedocs/api/v3/tests/test_projects.py b/readthedocs/api/v3/tests/test_projects.py new file mode 100644 index 00000000000..613599c8b5a --- /dev/null +++ b/readthedocs/api/v3/tests/test_projects.py @@ -0,0 +1,231 @@ +import datetime +import json + +from pathlib import Path + + +from django.contrib.auth.models import User +from django.test import TestCase +from django.urls import reverse + +import django_dynamic_fixture as fixture +from rest_framework.authtoken.models import Token +from rest_framework.test import APIClient + +from readthedocs.builds.models import Version, Build +from readthedocs.projects.models import Project + + +class APIEndpointTests(TestCase): + + fixtures = [] + + def setUp(self): + + self.me = fixture.get(User, projects=[]) + self.token = fixture.get(Token, key='me', user=self.me) + # Defining all the defaults helps to avoid creating ghost / unwanted + # objects (like a Project for translations/subprojects) + self.project = fixture.get( + Project, + pub_date=datetime.datetime(2019, 4, 29, 10, 0, 0), + modified_date=datetime.datetime(2019, 4, 29, 12, 0, 0), + description='Project description', + repo='https://github.com/rtfd/project', + project_url='http://project.com', + name='project', + slug='project', + related_projects=[], + main_language_project=None, + users=[self.me], + versions=[], + ) + for tag in ('tag', 'project', 'test'): + self.project.tags.add(tag) + + self.subproject = fixture.get( + Project, + pub_date=datetime.datetime(2019, 4, 29, 10, 0, 0), + modified_date=datetime.datetime(2019, 4, 29, 12, 0, 0), + description='SubProject description', + repo='https://github.com/rtfd/subproject', + project_url='http://subproject.com', + name='subproject', + slug='subproject', + related_projects=[], + main_language_project=None, + users=[], + versions=[], + ) + # self.translation = fixture.get(Project, slug='translation') + + self.project.add_subproject(self.subproject) + # self.project.add_translation(self.translation) + + self.version = fixture.get( + Version, + slug='v1.0', + verbose_name='v1.0', + identifier='a1b2c3', + project=self.project, + active=True, + built=True, + type='tag', + ) + + self.build = fixture.get( + Build, + date=datetime.datetime(2019, 4, 29, 10, 0, 0), + type='html', + state='finished', + error='', + success=True, + _config = {'property': 'test value'}, + version=self.version, + project=self.project, + builder='builder01', + commit='a1b2c3', + length=60, + ) + + self.other = fixture.get(User, projects=[]) + self.others_token = fixture.get(Token, key='other', user=self.other) + self.others_project = fixture.get( + Project, + slug='others_project', + related_projects=[], + main_language_project=None, + users=[self.other], + versions=[], + ) + + self.client = APIClient() + + def _get_response_dict(self, view_name): + filename = Path(__file__).absolute().parent / 'responses' / f'{view_name}.json' + return json.load(open(filename)) + + def assertDictEqual(self, d1, d2): + """ + Show the differences between the dicts in a human readable way. + + It's just a helper for debugging API responses. + """ + import datadiff + return super().assertDictEqual(d1, d2, datadiff.diff(d1, d2)) + + def test_projects_list(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse('projects-list'), + ) + self.assertEqual(response.status_code, 200) + self.assertDictEqual( + response.json(), + self._get_response_dict('projects-list'), + ) + + def test_subprojects_list(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-subprojects-list', + kwargs={ + 'parent_lookup_superprojects__parent__slug': self.project.slug, + }), + + ) + self.assertEqual(response.status_code, 200) + self.assertDictEqual( + response.json(), + self._get_response_dict('projects-subprojects-list'), + ) + + def test_detail_own_project(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-detail', + kwargs={ + 'project_slug': self.project.slug, + }), + { + 'expand': ( + 'active_versions,' + 'active_versions.last_build,' + 'active_versions.last_build.config' + ), + }, + ) + self.assertEqual(response.status_code, 200) + + self.assertDictEqual( + response.json(), + self._get_response_dict('projects-detail'), + ) + + def test_detail_others_project(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-detail', + kwargs={ + 'project_slug': self.others_project.slug, + }), + + ) + self.assertEqual(response.status_code, 200) + + def test_unauthed_detail_others_project(self): + response = self.client.get( + reverse( + 'projects-detail', + kwargs={ + 'project_slug': self.others_project.slug, + }), + + ) + self.assertEqual(response.status_code, 401) + + def test_detail_nonexistent_project(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-detail', + kwargs={ + 'project_slug': 'nonexistent', + }), + + ) + self.assertEqual(response.status_code, 404) + + def test_detail_version_of_project(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-versions-detail', + kwargs={ + 'parent_lookup_project__slug': self.project.slug, + 'version_slug': 'v1.0', + }), + + ) + self.assertEqual(response.status_code, 200) + self.assertDictEqual( + response.json(), + self._get_response_dict('projects-versions-detail'), + ) + + + def test_detail_version_of_nonexistent_project(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-versions-detail', + kwargs={ + 'parent_lookup_project__slug': 'nonexistent', + 'version_slug': 'latest', + }), + + ) + self.assertEqual(response.status_code, 404) From 443f277b43cabb4f3047977c5a22f5001ed71660 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 18:07:18 +0200 Subject: [PATCH 67/96] More tests and superproject endpoint fixed --- .../responses/projects-builds-detail.json | 22 +++ .../responses/projects-superproject.json | 46 ++++++ readthedocs/api/v3/tests/test_projects.py | 134 +++++++++++++++--- readthedocs/api/v3/views.py | 7 +- 4 files changed, 185 insertions(+), 24 deletions(-) create mode 100644 readthedocs/api/v3/tests/responses/projects-builds-detail.json create mode 100644 readthedocs/api/v3/tests/responses/projects-superproject.json diff --git a/readthedocs/api/v3/tests/responses/projects-builds-detail.json b/readthedocs/api/v3/tests/responses/projects-builds-detail.json new file mode 100644 index 00000000000..2a6d9e29b74 --- /dev/null +++ b/readthedocs/api/v3/tests/responses/projects-builds-detail.json @@ -0,0 +1,22 @@ +{ + "builder": "builder01", + "cold_storage": null, + "commit": "a1b2c3", + "created": "2019-04-29T10:00:00Z", + "duration": 60, + "error": "", + "finished": "2019-04-29T10:01:00Z", + "id": 1, + "links": { + "_self": "https://readthedocs.org/api/v3/projects/project/builds/1/", + "project": "https://readthedocs.org/api/v3/projects/project/", + "version": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/" + }, + "project": "project", + "state": { + "code": "finished", + "name": "Finished" + }, + "success": true, + "version": "v1.0" +} diff --git a/readthedocs/api/v3/tests/responses/projects-superproject.json b/readthedocs/api/v3/tests/responses/projects-superproject.json new file mode 100644 index 00000000000..05ff4b25e4c --- /dev/null +++ b/readthedocs/api/v3/tests/responses/projects-superproject.json @@ -0,0 +1,46 @@ +{ + "created": "2019-04-29T10:00:00Z", + "default_branch": "master", + "default_version": "latest", + "description": "Project description", + "id": 1, + "language": { + "code": "en", + "name": "English" + }, + "links": { + "_self": "https://readthedocs.org/api/v3/projects/project/", + "builds": "https://readthedocs.org/api/v3/projects/project/builds/", + "subprojects": "https://readthedocs.org/api/v3/projects/project/subprojects/", + "superproject": "https://readthedocs.org/api/v3/projects/project/superproject/", + "translations": "https://readthedocs.org/api/v3/projects/project/translations/", + "users": "https://readthedocs.org/api/v3/projects/project/users/", + "versions": "https://readthedocs.org/api/v3/projects/project/versions/" + }, + "modified": "2019-04-29T12:00:00Z", + "name": "project", + "privacy_level": { + "code": "public", + "name": "Public" + }, + "programming_language": { + "code": "words", + "name": "Only Words" + }, + "repository": { + "type": "git", + "url": "https://github.com/rtfd/project" + }, + "slug": "project", + "subproject_of": null, + "tags": [ + "tag", + "project", + "test" + ], + "translation_of": null, + "urls": { + "documentation": "http://readthedocs.org/docs/project/en/latest/", + "project_homepage": "http://project.com" + } +} diff --git a/readthedocs/api/v3/tests/test_projects.py b/readthedocs/api/v3/tests/test_projects.py index 613599c8b5a..17d7d3db517 100644 --- a/readthedocs/api/v3/tests/test_projects.py +++ b/readthedocs/api/v3/tests/test_projects.py @@ -125,7 +125,47 @@ def test_projects_list(self): self._get_response_dict('projects-list'), ) - def test_subprojects_list(self): + def test_own_projects_detail(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-detail', + kwargs={ + 'project_slug': self.project.slug, + }), + { + 'expand': ( + 'active_versions,' + 'active_versions.last_build,' + 'active_versions.last_build.config' + ), + }, + ) + self.assertEqual(response.status_code, 200) + + self.assertDictEqual( + response.json(), + self._get_response_dict('projects-detail'), + ) + + def test_projects_superproject(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-superproject', + kwargs={ + 'project_slug': self.subproject.slug, + }), + + ) + self.assertEqual(response.status_code, 200) + + self.assertDictEqual( + response.json(), + self._get_response_dict('projects-superproject'), + ) + + def test_projects_subprojects_list(self): self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') response = self.client.get( reverse( @@ -141,30 +181,31 @@ def test_subprojects_list(self): self._get_response_dict('projects-subprojects-list'), ) - def test_detail_own_project(self): + def test_others_projects_builds_list(self): self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') response = self.client.get( reverse( - 'projects-detail', + 'projects-builds-list', kwargs={ - 'project_slug': self.project.slug, + 'parent_lookup_project__slug': self.others_project.slug, }), - { - 'expand': ( - 'active_versions,' - 'active_versions.last_build,' - 'active_versions.last_build.config' - ), - }, + ) - self.assertEqual(response.status_code, 200) + self.assertEqual(response.status_code, 403) + + def test_others_projects_users_list(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-users-list', + kwargs={ + 'parent_lookup_projects__slug': self.others_project.slug, + }), - self.assertDictEqual( - response.json(), - self._get_response_dict('projects-detail'), ) + self.assertEqual(response.status_code, 403) - def test_detail_others_project(self): + def test_others_projects_detail(self): self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') response = self.client.get( reverse( @@ -176,7 +217,7 @@ def test_detail_others_project(self): ) self.assertEqual(response.status_code, 200) - def test_unauthed_detail_others_project(self): + def test_unauthed_others_projects_detail(self): response = self.client.get( reverse( 'projects-detail', @@ -187,7 +228,7 @@ def test_unauthed_detail_others_project(self): ) self.assertEqual(response.status_code, 401) - def test_detail_nonexistent_project(self): + def test_nonexistent_projects_detail(self): self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') response = self.client.get( reverse( @@ -199,7 +240,31 @@ def test_detail_nonexistent_project(self): ) self.assertEqual(response.status_code, 404) - def test_detail_version_of_project(self): + def test_projects_versions_list(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-versions-list', + kwargs={ + 'parent_lookup_project__slug': self.project.slug, + }), + + ) + self.assertEqual(response.status_code, 200) + + def test_others_projects_versions_list(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-versions-list', + kwargs={ + 'parent_lookup_project__slug': self.others_project.slug, + }), + + ) + self.assertEqual(response.status_code, 403) + + def test_projects_versions_detail(self): self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') response = self.client.get( reverse( @@ -216,8 +281,7 @@ def test_detail_version_of_project(self): self._get_response_dict('projects-versions-detail'), ) - - def test_detail_version_of_nonexistent_project(self): + def test_nonexistent_project_version_detail(self): self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') response = self.client.get( reverse( @@ -229,3 +293,31 @@ def test_detail_version_of_nonexistent_project(self): ) self.assertEqual(response.status_code, 404) + + def test_projects_builds_list(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-builds-list', + kwargs={ + 'parent_lookup_project__slug': self.project.slug, + }), + ) + self.assertEqual(response.status_code, 200) + + def test_projects_builds_detail(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-builds-detail', + kwargs={ + 'parent_lookup_project__slug': self.project.slug, + 'build_pk': self.build.pk, + }), + ) + self.assertEqual(response.status_code, 200) + + self.assertDictEqual( + response.json(), + self._get_response_dict('projects-builds-detail'), + ) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index cd94e2ea39c..04cebc3f4a5 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -164,11 +164,12 @@ def get_view_description(self, *args, **kwargs): @action(detail=True, methods=['get']) def superproject(self, request, project_slug): project = self.get_object() - superproject = getattr(project, 'main_project', None) - if superproject: + try: + superproject = project.superprojects.first().parent data = self.get_serializer(superproject).data return Response(data) - return Response(status=404) + except Exception: + return Response(status=404) class SubprojectRelationshipViewSet(APIv3Settings, APIAuthMixin, From d09c09ff853ac3b80a2897d77b8bc6bf5b78bc55 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 18:12:48 +0200 Subject: [PATCH 68/96] Remove old examples --- readthedocs/api/v3/examples/__init__.py | 0 readthedocs/api/v3/examples/client.py | 10 --- .../api/v3/examples/project_details.py | 81 ------------------- .../api/v3/examples/project_full_details.py | 4 - readthedocs/api/v3/examples/utils.py | 11 --- 5 files changed, 106 deletions(-) delete mode 100644 readthedocs/api/v3/examples/__init__.py delete mode 100644 readthedocs/api/v3/examples/client.py delete mode 100644 readthedocs/api/v3/examples/project_details.py delete mode 100644 readthedocs/api/v3/examples/project_full_details.py delete mode 100644 readthedocs/api/v3/examples/utils.py diff --git a/readthedocs/api/v3/examples/__init__.py b/readthedocs/api/v3/examples/__init__.py deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/readthedocs/api/v3/examples/client.py b/readthedocs/api/v3/examples/client.py deleted file mode 100644 index 2f106ffa91b..00000000000 --- a/readthedocs/api/v3/examples/client.py +++ /dev/null @@ -1,10 +0,0 @@ -import requests -import slumber - - -def setup_api(token): - session = requests.Session() - session.headers.update({'Authorization': f'Token {token}'}) - return slumber.API('http://localhost:8000/api/v3/', session=session) - -api = setup_api(123) diff --git a/readthedocs/api/v3/examples/project_details.py b/readthedocs/api/v3/examples/project_details.py deleted file mode 100644 index c4624c79086..00000000000 --- a/readthedocs/api/v3/examples/project_details.py +++ /dev/null @@ -1,81 +0,0 @@ -import time - -from client import api -from utils import p - - -# Get specific project by slug -p(api.projects('test-builds').get()) -input('Press Enter to continue...') - -# Get specific project by slug full expanded -# (all active versions with each last build object and its configuration) -p(api.projects('test-builds').get( - expand='active_versions,active_versions.last_build,active_versions.last_build.config', -)) -input('Press Enter to continue...') - - -# Get all active and built versions for a project selecting only needed fields: -# slug, urls, downloads -# (useful to create the versions menu on a theme) -p(api.projects('test-builds').versions.get( - expand='last_build', - fields='slug,urls,downloads', - active=True, - built=True, # filtering by built we avoid ending up with 404 links -)) -input('Press Enter to continue...') - -# Get all running builds for a project -# (useful for a status page of the project) -p(api.projects('test-builds').builds.get( - running=True, -)) -input('Press Enter to continue...') - -# Get all running builds for specific version of a project -p(api.projects('test-builds').versions('latest').builds.get( - running=True, -)) -input('Press Enter to continue...') - - -# trigger a build of default version and poll the status -# (useful on the release process to check that docs build before publishing) -# response = api.projects('test-builds').builds().post() - -# Trigger a build for a specific version -response = api.projects('test-builds').versions('use-py2').builds().post() -p(response) -if response['triggered']: - finished = response['build']['finished'] - build_id = response['build']['id'] - project_slug = response['project']['slug'] - build_url = response['build']['links']['_self'] - - while not finished: - time.sleep(5) - # NOTE: I already have the url for this on ``build_url`` but as I'm - # using slumber which already have the Authorization header, I don't - # know how to hit it directly and I need to rebuilt it here (this is a - # limitation of the client, not of API design) - response = api.projects(project_slug).builds(build_id).get() - state = response['state']['code'] - finished = response['finished'] - print(f'Current state: {state}') - print('Finished') - -input('Press Enter to continue...') - - -# Activate and make private a specific version of a project -# NOTE: slumber can't be used here since ``.patch`` send the data in the URL -api._store['session'].patch( - api.projects('test-builds').versions('submodule-https-scheme').url(), - data=dict( - active=True, - privacy_level='private', - ), -) -input('Press Enter to continue...') diff --git a/readthedocs/api/v3/examples/project_full_details.py b/readthedocs/api/v3/examples/project_full_details.py deleted file mode 100644 index 11db0c477fa..00000000000 --- a/readthedocs/api/v3/examples/project_full_details.py +++ /dev/null @@ -1,4 +0,0 @@ -from client import api -from utils import p - -p(api.projects('test-builds').get(expand='')) diff --git a/readthedocs/api/v3/examples/utils.py b/readthedocs/api/v3/examples/utils.py deleted file mode 100644 index 361d0689a59..00000000000 --- a/readthedocs/api/v3/examples/utils.py +++ /dev/null @@ -1,11 +0,0 @@ -import json -from pygments import highlight -from pygments.formatters import TerminalTrueColorFormatter -from pygments.lexers import JsonLexer - - -def p(data): - j = json.dumps(data, sort_keys=True, indent=4) - print( - highlight(j, JsonLexer(), TerminalTrueColorFormatter()) - ) From 41ef2d3290f3cd93a9191499a1f4bae02bc146e7 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 18:15:19 +0200 Subject: [PATCH 69/96] Requirements updated --- requirements/pip.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements/pip.txt b/requirements/pip.txt index 59020336a8d..5d4afd9e1dd 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -17,9 +17,8 @@ Sphinx==1.8.5 # pyup: <2.0.0 # Filtering for the REST API django-filter==2.1.0 -drf-flex-fields==0.3.5 +drf-flex-fields==0.5.0 drf-extensions==0.4.0 -djangorestframework-simplejwt==4.0.0 django-vanilla-views==1.0.6 jsonfield==2.0.2 From 55cbf4689c7cda679dfd5c61433d4f956bb43dcc Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 29 Apr 2019 22:11:31 +0200 Subject: [PATCH 70/96] Avoid throttling on tests --- readthedocs/api/v3/tests/test_projects.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/readthedocs/api/v3/tests/test_projects.py b/readthedocs/api/v3/tests/test_projects.py index 17d7d3db517..e37fddeef67 100644 --- a/readthedocs/api/v3/tests/test_projects.py +++ b/readthedocs/api/v3/tests/test_projects.py @@ -4,6 +4,7 @@ from pathlib import Path +from django.core.cache import cache from django.contrib.auth.models import User from django.test import TestCase from django.urls import reverse @@ -101,6 +102,10 @@ def setUp(self): self.client = APIClient() + def tearDown(self): + # Cleanup cache to avoid throttling on tests + cache.clear() + def _get_response_dict(self, view_name): filename = Path(__file__).absolute().parent / 'responses' / f'{view_name}.json' return json.load(open(filename)) From 4e7420587e5ef857cfdd922d36aea06390cc3151 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 1 May 2019 14:15:49 +0200 Subject: [PATCH 71/96] Variable name --- readthedocs/api/v3/mixins.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py index 10daa195f72..0724c18fafa 100644 --- a/readthedocs/api/v3/mixins.py +++ b/readthedocs/api/v3/mixins.py @@ -22,15 +22,15 @@ class NestedParentObjectMixin: ] def _get_parent_object(self, model, lookup_names): - project_slug = None + object_slug = None query_dict = self.get_parents_query_dict() for lookup in lookup_names: value = query_dict.get(lookup) if value: - slug = value + object_slug = value break - return get_object_or_404(model, slug=slug) + return get_object_or_404(model, slug=object_slug) def _get_parent_project(self): return self._get_parent_object(Project, self.PROJECT_LOOKUP_NAMES) From 86f89f08bd4b381244d29dfc809267c5b32af353 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 1 May 2019 14:17:52 +0200 Subject: [PATCH 72/96] Explicit return --- readthedocs/api/v3/mixins.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py index 0724c18fafa..3a4737b5cfc 100644 --- a/readthedocs/api/v3/mixins.py +++ b/readthedocs/api/v3/mixins.py @@ -62,6 +62,8 @@ def listing_objects(self, queryset, user): if self.has_admin_permission(user, project): return queryset + return queryset.none() + def has_admin_permission(self, user, project): if project in self.admin_projects(user): return True From 2035ea176fd2c123e8a485078854f2ba414da290 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 1 May 2019 14:18:59 +0200 Subject: [PATCH 73/96] Rename file --- readthedocs/api/v3/{renderer.py => renderers.py} | 0 readthedocs/api/v3/views.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename readthedocs/api/v3/{renderer.py => renderers.py} (100%) diff --git a/readthedocs/api/v3/renderer.py b/readthedocs/api/v3/renderers.py similarity index 100% rename from readthedocs/api/v3/renderer.py rename to readthedocs/api/v3/renderers.py diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 04cebc3f4a5..b1d16aa7703 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -29,7 +29,7 @@ from .filters import BuildFilter, ProjectFilter, VersionFilter from .mixins import APIAuthMixin from .permissions import PublicDetailPrivateListing -from .renderer import AlphabeticalSortedJSONRenderer +from .renderers import AlphabeticalSortedJSONRenderer from .serializers import ( BuildSerializer, BuildCreateSerializer, From 8d11b9d2c7399eb2398e2ed2c7343082760b3639 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 1 May 2019 14:24:34 +0200 Subject: [PATCH 74/96] Remove unused import --- readthedocs/settings/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index 82fd0b7bf6e..fd59d34e4aa 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -3,7 +3,6 @@ import getpass import os -import datetime from celery.schedules import crontab From 99c7954d6cd4ee5520c2b0fed0cb6ff0c42c9ddb Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 1 May 2019 14:44:06 +0200 Subject: [PATCH 75/96] Lint fixes --- readthedocs/api/v3/filters.py | 4 ++-- readthedocs/api/v3/routers.py | 2 +- readthedocs/api/v3/serializers.py | 4 ++-- readthedocs/api/v3/views.py | 12 ++++++++---- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/readthedocs/api/v3/filters.py b/readthedocs/api/v3/filters.py index 177759804ef..b24ffb31b63 100644 --- a/readthedocs/api/v3/filters.py +++ b/readthedocs/api/v3/filters.py @@ -70,5 +70,5 @@ class Meta: def get_running(self, queryset, name, value): if value: return queryset.exclude(state=BUILD_STATE_FINISHED) - else: - return queryset.filter(state=BUILD_STATE_FINISHED) + + return queryset.filter(state=BUILD_STATE_FINISHED) diff --git a/readthedocs/api/v3/routers.py b/readthedocs/api/v3/routers.py index 4059ef38462..47bfa613d12 100644 --- a/readthedocs/api/v3/routers.py +++ b/readthedocs/api/v3/routers.py @@ -10,7 +10,7 @@ class DocsAPIRootView(APIRootView): Read the Docs APIv3 root endpoint. Full documentation at [https://docs.readthedocs.io/en/latest/api/v3.html](https://docs.readthedocs.io/en/latest/api/v3.html). - """ + """ # noqa def get_view_name(self): return 'Read the Docs APIv3' diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index f2948a10b46..0a460dccc67 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -91,10 +91,10 @@ class BuildConfigSerializer(FlexFieldsSerializerMixin, serializers.Serializer): which may produce incompatible changes in the API. """ - def to_representation(self, obj): + def to_representation(self, instance): # For now, we want to return the ``config`` object as it is without # manipulating it. - return obj + return instance class BuildStateSerializer(serializers.Serializer): diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index b1d16aa7703..5610e85a7d4 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -117,7 +117,8 @@ class ProjectsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, * Subprojects of a project: ``/api/v3/projects/{project_slug}/subprojects/`` * Superproject of a project: ``/api/v3/projects/{project_slug}/superproject/`` - Go to https://docs.readthedocs.io/en/stable/api/v3.html for a complete documentation of the APIv3. + Go to https://docs.readthedocs.io/en/stable/api/v3.html + for a complete documentation of the APIv3. """ model = Project @@ -144,7 +145,7 @@ def get_queryset(self): 'tags', ) - def get_view_description(self, *args, **kwargs): + def get_view_description(self, *args, **kwargs): # pylint: disable=arguments-differ """ Make valid links for the user's documentation browseable API. @@ -228,7 +229,10 @@ class VersionsViewSet(APIv3Settings, APIAuthMixin, def get_serializer_class(self): """ - Return correct serializer depending on the action (GET or PUT/PATCH/POST). + Return correct serializer depending on the action. + + For GET it returns a serializer with many fields and on PUT/PATCH/POST, + it return a serializer to validate just a few fields. """ if self.action in ('list', 'retrieve'): return VersionSerializer @@ -266,7 +270,7 @@ def get_serializer_class(self): return BuildSerializer return BuildCreateSerializer - def create(self, request, **kwargs): + def create(self, request, **kwargs): # pylint: disable=arguments-differ project = self._get_parent_project() version = self._get_parent_version() From 276a979d3c9b96ac0380dfc1c90134687ce14874 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 1 May 2019 14:50:51 +0200 Subject: [PATCH 76/96] pre-commit auto style --- readthedocs/api/v2/client.py | 2 -- readthedocs/api/v2/permissions.py | 2 -- readthedocs/api/v2/serializers.py | 2 -- readthedocs/api/v2/signals.py | 2 -- readthedocs/api/v2/urls.py | 6 ++--- readthedocs/api/v2/utils.py | 2 -- readthedocs/api/v2/views/core_views.py | 2 -- readthedocs/api/v2/views/footer_views.py | 2 +- readthedocs/api/v2/views/integrations.py | 12 ++++----- readthedocs/api/v2/views/model_views.py | 4 +-- readthedocs/api/v2/views/task_views.py | 2 -- readthedocs/api/v3/tests/test_projects.py | 9 +++---- readthedocs/api/v3/urls.py | 9 ++++++- readthedocs/api/v3/views.py | 26 ++++++++----------- readthedocs/builds/querysets.py | 2 -- readthedocs/config/models.py | 8 +++--- readthedocs/core/views/hooks.py | 2 -- readthedocs/doc_builder/backends/sphinx.py | 9 ++----- readthedocs/doc_builder/environments.py | 4 +-- readthedocs/oauth/services/github.py | 4 +-- readthedocs/projects/models.py | 6 ++--- readthedocs/projects/querysets.py | 2 -- readthedocs/projects/tasks.py | 18 ++++++++----- readthedocs/projects/urls/public.py | 2 -- readthedocs/projects/utils.py | 2 -- readthedocs/rtd_tests/mocks/mock_api.py | 1 - readthedocs/rtd_tests/tests/test_api.py | 21 +++++++-------- .../rtd_tests/tests/test_api_permissions.py | 1 - .../tests/test_api_version_compare.py | 3 +-- readthedocs/rtd_tests/tests/test_footer.py | 9 +++---- .../rtd_tests/tests/test_privacy_urls.py | 1 - .../rtd_tests/tests/test_restapi_client.py | 1 - readthedocs/urls.py | 1 - requirements/pip.txt | 2 +- 34 files changed, 71 insertions(+), 110 deletions(-) diff --git a/readthedocs/api/v2/client.py b/readthedocs/api/v2/client.py index 98920c6da11..c0085b6ebee 100644 --- a/readthedocs/api/v2/client.py +++ b/readthedocs/api/v2/client.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Simple client to access our API with Slumber credentials.""" import logging diff --git a/readthedocs/api/v2/permissions.py b/readthedocs/api/v2/permissions.py index 93d4695a7cc..67bc8b212f6 100644 --- a/readthedocs/api/v2/permissions.py +++ b/readthedocs/api/v2/permissions.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Defines access permissions for the API.""" from rest_framework import permissions diff --git a/readthedocs/api/v2/serializers.py b/readthedocs/api/v2/serializers.py index 0b9504ec74b..6d9a1cd48a6 100644 --- a/readthedocs/api/v2/serializers.py +++ b/readthedocs/api/v2/serializers.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Defines serializers for each of our models.""" from allauth.socialaccount.models import SocialAccount diff --git a/readthedocs/api/v2/signals.py b/readthedocs/api/v2/signals.py index 65509fc551d..419e78f43f5 100644 --- a/readthedocs/api/v2/signals.py +++ b/readthedocs/api/v2/signals.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """We define custom Django signals to trigger when a footer is rendered.""" import django.dispatch diff --git a/readthedocs/api/v2/urls.py b/readthedocs/api/v2/urls.py index ea377493b55..4d4c835d3cc 100644 --- a/readthedocs/api/v2/urls.py +++ b/readthedocs/api/v2/urls.py @@ -1,18 +1,17 @@ -# -*- coding: utf-8 -*- - """Define routes between URL paths and views/endpoints.""" from django.conf import settings from django.conf.urls import include, url from rest_framework import routers -from readthedocs.constants import pattern_opts from readthedocs.api.v2.views import ( core_views, footer_views, integrations, task_views, ) +from readthedocs.constants import pattern_opts +from readthedocs.sphinx_domains.api import SphinxDomainAPIView from .views.model_views import ( BuildCommandViewSet, @@ -25,7 +24,6 @@ SocialAccountViewSet, VersionViewSet, ) -from readthedocs.sphinx_domains.api import SphinxDomainAPIView router = routers.DefaultRouter() diff --git a/readthedocs/api/v2/utils.py b/readthedocs/api/v2/utils.py index f30655f46e0..b620bfe930c 100644 --- a/readthedocs/api/v2/utils.py +++ b/readthedocs/api/v2/utils.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Utility functions that are used by both views and celery tasks.""" import logging diff --git a/readthedocs/api/v2/views/core_views.py b/readthedocs/api/v2/views/core_views.py index 1e4afdd8a7b..305baba4b7c 100644 --- a/readthedocs/api/v2/views/core_views.py +++ b/readthedocs/api/v2/views/core_views.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Utility endpoints relating to canonical urls, embedded content, etc.""" from django.shortcuts import get_object_or_404 diff --git a/readthedocs/api/v2/views/footer_views.py b/readthedocs/api/v2/views/footer_views.py index ecb765bd690..09709ac1bb1 100644 --- a/readthedocs/api/v2/views/footer_views.py +++ b/readthedocs/api/v2/views/footer_views.py @@ -8,6 +8,7 @@ from rest_framework.response import Response from rest_framework_jsonp.renderers import JSONPRenderer +from readthedocs.api.v2.signals import footer_response from readthedocs.builds.constants import LATEST, TAG from readthedocs.builds.models import Version from readthedocs.projects.models import Project @@ -15,7 +16,6 @@ highest_version, parse_version_failsafe, ) -from readthedocs.api.v2.signals import footer_response def get_version_compare_data(project, base_version=None): diff --git a/readthedocs/api/v2/views/integrations.py b/readthedocs/api/v2/views/integrations.py index 5a96caf55b2..4424238a78b 100644 --- a/readthedocs/api/v2/views/integrations.py +++ b/readthedocs/api/v2/views/integrations.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Endpoints integrating with Github, Bitbucket, and other webhooks.""" import hashlib @@ -77,7 +75,7 @@ def post(self, request, project_slug): ) return Response( {'detail': self.invalid_payload_msg}, - status=HTTP_400_BAD_REQUEST + status=HTTP_400_BAD_REQUEST, ) resp = self.handle_webhook() if resp is None: @@ -232,7 +230,7 @@ def is_payload_valid(self): if not secret: log.info( 'Skipping payload validation for project: %s', - self.project.slug + self.project.slug, ) return True if not signature: @@ -241,7 +239,7 @@ def is_payload_valid(self): digest = GitHubWebhookView.get_digest(secret, msg) result = hmac.compare_digest( b'sha1=' + digest.encode(), - signature.encode() + signature.encode(), ) return result @@ -251,7 +249,7 @@ def get_digest(secret, msg): digest = hmac.new( secret.encode(), msg=msg.encode(), - digestmod=hashlib.sha1 + digestmod=hashlib.sha1, ) return digest.hexdigest() @@ -318,7 +316,7 @@ def is_payload_valid(self): if not secret: log.info( 'Skipping payload validation for project: %s', - self.project.slug + self.project.slug, ) return True if not token: diff --git a/readthedocs/api/v2/views/model_views.py b/readthedocs/api/v2/views/model_views.py index d955218653d..cbb459cf42e 100644 --- a/readthedocs/api/v2/views/model_views.py +++ b/readthedocs/api/v2/views/model_views.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Endpoints for listing Projects, Versions, Builds, etc.""" import logging @@ -8,7 +6,7 @@ from django.shortcuts import get_object_or_404 from django.template.loader import render_to_string from rest_framework import decorators, permissions, status, viewsets -from rest_framework.parsers import MultiPartParser, JSONParser, FormParser +from rest_framework.parsers import JSONParser, MultiPartParser from rest_framework.renderers import BaseRenderer, JSONRenderer from rest_framework.response import Response diff --git a/readthedocs/api/v2/views/task_views.py b/readthedocs/api/v2/views/task_views.py index 8bf9d3843e4..dfb34adc6c2 100644 --- a/readthedocs/api/v2/views/task_views.py +++ b/readthedocs/api/v2/views/task_views.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Endpoints relating to task/job status, etc.""" import logging diff --git a/readthedocs/api/v3/tests/test_projects.py b/readthedocs/api/v3/tests/test_projects.py index e37fddeef67..cc66366ed4f 100644 --- a/readthedocs/api/v3/tests/test_projects.py +++ b/readthedocs/api/v3/tests/test_projects.py @@ -1,19 +1,16 @@ import datetime import json - from pathlib import Path - -from django.core.cache import cache +import django_dynamic_fixture as fixture from django.contrib.auth.models import User +from django.core.cache import cache from django.test import TestCase from django.urls import reverse - -import django_dynamic_fixture as fixture from rest_framework.authtoken.models import Token from rest_framework.test import APIClient -from readthedocs.builds.models import Version, Build +from readthedocs.builds.models import Build, Version from readthedocs.projects.models import Project diff --git a/readthedocs/api/v3/urls.py b/readthedocs/api/v3/urls.py index 708188e1afa..d055dd85b98 100644 --- a/readthedocs/api/v3/urls.py +++ b/readthedocs/api/v3/urls.py @@ -1,5 +1,12 @@ from .routers import DefaultRouterWithNesting -from .views import BuildsViewSet, ProjectsViewSet, UsersViewSet, VersionsViewSet, SubprojectRelationshipViewSet, TranslationRelationshipViewSet +from .views import ( + BuildsViewSet, + ProjectsViewSet, + SubprojectRelationshipViewSet, + TranslationRelationshipViewSet, + UsersViewSet, + VersionsViewSet, +) router = DefaultRouterWithNesting() diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 5610e85a7d4..65ba90c6061 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -1,11 +1,8 @@ import django_filters.rest_framework as filters from django.contrib.auth.models import User -from django.shortcuts import get_object_or_404 from django.utils.safestring import mark_safe from rest_flex_fields.views import FlexFieldsMixin -from rest_framework.authentication import ( - TokenAuthentication, -) +from rest_framework.authentication import TokenAuthentication from rest_framework.decorators import action from rest_framework.metadata import SimpleMetadata from rest_framework.mixins import ( @@ -15,7 +12,6 @@ UpdateModelMixin, ) from rest_framework.pagination import LimitOffsetPagination -from rest_framework.permissions import IsAuthenticated from rest_framework.renderers import BrowsableAPIRenderer from rest_framework.response import Response from rest_framework.throttling import AnonRateThrottle, UserRateThrottle @@ -31,8 +27,8 @@ from .permissions import PublicDetailPrivateListing from .renderers import AlphabeticalSortedJSONRenderer from .serializers import ( - BuildSerializer, BuildCreateSerializer, + BuildSerializer, ProjectSerializer, UserSerializer, VersionSerializer, @@ -174,8 +170,8 @@ def superproject(self, request, project_slug): class SubprojectRelationshipViewSet(APIv3Settings, APIAuthMixin, - NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, - GenericViewSet): + NestedViewSetMixin, FlexFieldsMixin, + ListModelMixin, GenericViewSet): """ List subprojects of a ``Project``. @@ -192,8 +188,8 @@ class SubprojectRelationshipViewSet(APIv3Settings, APIAuthMixin, class TranslationRelationshipViewSet(APIv3Settings, APIAuthMixin, - NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, - GenericViewSet): + NestedViewSetMixin, FlexFieldsMixin, + ListModelMixin, GenericViewSet): """ List translations of a ``Project``. @@ -209,9 +205,9 @@ class TranslationRelationshipViewSet(APIv3Settings, APIAuthMixin, queryset = Project.objects.all() -class VersionsViewSet(APIv3Settings, APIAuthMixin, - NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, - RetrieveModelMixin, UpdateModelMixin, GenericViewSet): +class VersionsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, + FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, + UpdateModelMixin, GenericViewSet): model = Version lookup_field = 'slug' @@ -293,8 +289,8 @@ def create(self, request, **kwargs): # pylint: disable=arguments-differ return Response(data=data, status=status) -class UsersViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, ListModelMixin, - RetrieveModelMixin, GenericViewSet): +class UsersViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, + ListModelMixin, RetrieveModelMixin, GenericViewSet): model = User lookup_field = 'username' lookup_url_kwarg = 'user_username' diff --git a/readthedocs/builds/querysets.py b/readthedocs/builds/querysets.py index f5d8b360b5c..6b4c9132b1a 100644 --- a/readthedocs/builds/querysets.py +++ b/readthedocs/builds/querysets.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Build and Version QuerySet classes.""" from django.db import models diff --git a/readthedocs/config/models.py b/readthedocs/config/models.py index d0e99bc9b5a..ba11bce29d4 100644 --- a/readthedocs/config/models.py +++ b/readthedocs/config/models.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Models for the response of the configuration object.""" from readthedocs.config.utils import to_dict @@ -45,7 +43,11 @@ class PythonInstallRequirements(Base): class PythonInstall(Base): - __slots__ = ('path', 'method', 'extra_requirements',) + __slots__ = ( + 'path', + 'method', + 'extra_requirements', + ) class Conda(Base): diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index c76b96829bd..7d0f514505a 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Views pertaining to builds.""" import json diff --git a/readthedocs/doc_builder/backends/sphinx.py b/readthedocs/doc_builder/backends/sphinx.py index 71f6f0aa83a..dd00bfcbe15 100644 --- a/readthedocs/doc_builder/backends/sphinx.py +++ b/readthedocs/doc_builder/backends/sphinx.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """ Sphinx_ backend for building docs. @@ -10,7 +8,6 @@ import logging import os import shutil -import sys import zipfile from glob import glob from pathlib import Path @@ -19,11 +16,11 @@ from django.template import loader as template_loader from django.template.loader import render_to_string +from readthedocs.api.v2.client import api from readthedocs.builds import utils as version_utils from readthedocs.projects.exceptions import ProjectConfigurationError from readthedocs.projects.models import Feature from readthedocs.projects.utils import safe_write -from readthedocs.api.v2.client import api from ..base import BaseBuilder, restoring_chdir from ..constants import PDF_RE @@ -178,9 +175,7 @@ def append_conf(self, **__): ) outfile = codecs.open(self.config_file, encoding='utf-8', mode='a') except IOError: - raise ProjectConfigurationError( - ProjectConfigurationError.NOT_FOUND - ) + raise ProjectConfigurationError(ProjectConfigurationError.NOT_FOUND) # Append config to project conf file tmpl = template_loader.get_template('doc_builder/conf.py.tmpl') diff --git a/readthedocs/doc_builder/environments.py b/readthedocs/doc_builder/environments.py index d40a2610ca5..4c14ffb29a2 100644 --- a/readthedocs/doc_builder/environments.py +++ b/readthedocs/doc_builder/environments.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Documentation Builder Environments.""" import logging @@ -20,12 +18,12 @@ from requests_toolbelt.multipart.encoder import MultipartEncoder from slumber.exceptions import HttpClientError +from readthedocs.api.v2.client import api as api_v2 from readthedocs.builds.constants import BUILD_STATE_FINISHED from readthedocs.builds.models import BuildCommandResultMixin from readthedocs.core.utils import slugify from readthedocs.projects.constants import LOG_TEMPLATE from readthedocs.projects.models import Feature -from readthedocs.api.v2.client import api as api_v2 from .constants import ( DOCKER_HOSTNAME_MAX_LEN, diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index 4b471d0f3b5..1c29f7b3ae9 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -10,16 +10,14 @@ from django.urls import reverse from requests.exceptions import RequestException +from readthedocs.api.v2.client import api from readthedocs.builds import utils as build_utils from readthedocs.integrations.models import Integration -from readthedocs.integrations.utils import get_secret -from readthedocs.api.v2.client import api from ..models import RemoteOrganization, RemoteRepository from .base import Service, SyncServiceError - log = logging.getLogger(__name__) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index 41a7ed78546..4626c035b24 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -12,13 +12,14 @@ from django.db import models from django.db.models import Prefetch from django.urls import NoReverseMatch, reverse -from django.utils.translation import ugettext_lazy as _ from django.utils.functional import cached_property +from django.utils.translation import ugettext_lazy as _ from django_extensions.db.models import TimeStampedModel from guardian.shortcuts import assign from six.moves import shlex_quote from taggit.managers import TaggableManager +from readthedocs.api.v2.client import api from readthedocs.builds.constants import LATEST, STABLE from readthedocs.core.resolver import resolve, resolve_domain from readthedocs.core.utils import broadcast, slugify @@ -37,7 +38,6 @@ validate_repository_url, ) from readthedocs.projects.version_handling import determine_stable_version -from readthedocs.api.v2.client import api from readthedocs.search.parse_json import process_file from readthedocs.vcs_support.backends import backend_cls from readthedocs.vcs_support.utils import Lock, NonBlockingLock @@ -1002,7 +1002,7 @@ def get_parent_relationship(self): return self.superprojects.select_related('parent').first() def get_canonical_custom_domain(self): - """Get the canonical custom domain or None""" + """Get the canonical custom domain or None.""" if hasattr(self, '_canonical_domains'): # Cached custom domains if self._canonical_domains: diff --git a/readthedocs/projects/querysets.py b/readthedocs/projects/querysets.py index 388179f68f3..ece4e99b961 100644 --- a/readthedocs/projects/querysets.py +++ b/readthedocs/projects/querysets.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Project model QuerySet classes.""" from django.db import models diff --git a/readthedocs/projects/tasks.py b/readthedocs/projects/tasks.py index 40d055306f2..8a3f08da386 100644 --- a/readthedocs/projects/tasks.py +++ b/readthedocs/projects/tasks.py @@ -26,7 +26,7 @@ from slumber.exceptions import HttpClientError from sphinx.ext import intersphinx - +from readthedocs.api.v2.client import api as api_v2 from readthedocs.builds.constants import ( BUILD_STATE_BUILDING, BUILD_STATE_CLONING, @@ -60,9 +60,8 @@ ) from readthedocs.doc_builder.loader import get_builder_class from readthedocs.doc_builder.python_environments import Conda, Virtualenv -from readthedocs.sphinx_domains.models import SphinxDomain from readthedocs.projects.models import APIProject -from readthedocs.api.v2.client import api as api_v2 +from readthedocs.sphinx_domains.models import SphinxDomain from readthedocs.vcs_support import utils as vcs_support_utils from readthedocs.worker import app @@ -933,8 +932,15 @@ def is_type_sphinx(self): # Web tasks @app.task(queue='web') def sync_files( - project_pk, version_pk, doctype, hostname=None, html=False, - localmedia=False, search=False, pdf=False, epub=False, + project_pk, + version_pk, + doctype, + hostname=None, + html=False, + localmedia=False, + search=False, + pdf=False, + epub=False, delete_unsynced_media=False, ): """ @@ -1217,7 +1223,7 @@ def fileify(version_pk, commit): def _update_intersphinx_data(version, path, commit): """ - Update intersphinx data for this version + Update intersphinx data for this version. :param version: Version instance :param path: Path to search diff --git a/readthedocs/projects/urls/public.py b/readthedocs/projects/urls/public.py index 851764a3b43..0f8b201c9e8 100644 --- a/readthedocs/projects/urls/public.py +++ b/readthedocs/projects/urls/public.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Project URLS for public users.""" from django.conf.urls import url diff --git a/readthedocs/projects/utils.py b/readthedocs/projects/utils.py index fd1916c58d7..df539cbfb49 100644 --- a/readthedocs/projects/utils.py +++ b/readthedocs/projects/utils.py @@ -1,5 +1,3 @@ -# -*- coding: utf-8 -*- - """Utility functions used by projects.""" import logging diff --git a/readthedocs/rtd_tests/mocks/mock_api.py b/readthedocs/rtd_tests/mocks/mock_api.py index acd803b1c16..ef33f7d6d49 100644 --- a/readthedocs/rtd_tests/mocks/mock_api.py +++ b/readthedocs/rtd_tests/mocks/mock_api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """Mock versions of many API-related classes.""" import json from contextlib import contextmanager diff --git a/readthedocs/rtd_tests/tests/test_api.py b/readthedocs/rtd_tests/tests/test_api.py index 848e3c52ae0..3ee7820d24d 100644 --- a/readthedocs/rtd_tests/tests/test_api.py +++ b/readthedocs/rtd_tests/tests/test_api.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import base64 import datetime import json @@ -13,16 +12,6 @@ from rest_framework import status from rest_framework.test import APIClient -from readthedocs.builds.constants import LATEST -from readthedocs.builds.models import Build, BuildCommandResult, Version -from readthedocs.integrations.models import Integration -from readthedocs.oauth.models import RemoteOrganization, RemoteRepository -from readthedocs.projects.models import ( - APIProject, - EnvironmentVariable, - Feature, - Project, -) from readthedocs.api.v2.views.integrations import ( GITHUB_CREATE, GITHUB_DELETE, @@ -37,6 +26,16 @@ GitLabWebhookView, ) from readthedocs.api.v2.views.task_views import get_status_data +from readthedocs.builds.constants import LATEST +from readthedocs.builds.models import Build, BuildCommandResult, Version +from readthedocs.integrations.models import Integration +from readthedocs.oauth.models import RemoteOrganization, RemoteRepository +from readthedocs.projects.models import ( + APIProject, + EnvironmentVariable, + Feature, + Project, +) super_auth = base64.b64encode(b'super:test').decode('utf-8') diff --git a/readthedocs/rtd_tests/tests/test_api_permissions.py b/readthedocs/rtd_tests/tests/test_api_permissions.py index 959f02b0cae..4e91f419faa 100644 --- a/readthedocs/rtd_tests/tests/test_api_permissions.py +++ b/readthedocs/rtd_tests/tests/test_api_permissions.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from functools import partial from unittest import TestCase diff --git a/readthedocs/rtd_tests/tests/test_api_version_compare.py b/readthedocs/rtd_tests/tests/test_api_version_compare.py index 85852fd5d10..0f7628981e8 100644 --- a/readthedocs/rtd_tests/tests/test_api_version_compare.py +++ b/readthedocs/rtd_tests/tests/test_api_version_compare.py @@ -1,9 +1,8 @@ -# -*- coding: utf-8 -*- from django.test import TestCase +from readthedocs.api.v2.views.footer_views import get_version_compare_data from readthedocs.builds.constants import LATEST from readthedocs.projects.models import Project -from readthedocs.api.v2.views.footer_views import get_version_compare_data class VersionCompareTests(TestCase): diff --git a/readthedocs/rtd_tests/tests/test_footer.py b/readthedocs/rtd_tests/tests/test_footer.py index 87c8360e7bd..34c1fec1f92 100644 --- a/readthedocs/rtd_tests/tests/test_footer.py +++ b/readthedocs/rtd_tests/tests/test_footer.py @@ -1,16 +1,15 @@ -# -*- coding: utf-8 -*- import mock from django.test import TestCase from rest_framework.test import APIRequestFactory, APITestCase -from readthedocs.builds.constants import BRANCH, LATEST, TAG -from readthedocs.builds.models import Version -from readthedocs.core.middleware import FooterNoSessionMiddleware -from readthedocs.projects.models import Project from readthedocs.api.v2.views.footer_views import ( footer_html, get_version_compare_data, ) +from readthedocs.builds.constants import BRANCH, LATEST, TAG +from readthedocs.builds.models import Version +from readthedocs.core.middleware import FooterNoSessionMiddleware +from readthedocs.projects.models import Project from readthedocs.rtd_tests.mocks.paths import fake_paths_by_regex diff --git a/readthedocs/rtd_tests/tests/test_privacy_urls.py b/readthedocs/rtd_tests/tests/test_privacy_urls.py index 10014c7f655..b6448d50699 100644 --- a/readthedocs/rtd_tests/tests/test_privacy_urls.py +++ b/readthedocs/rtd_tests/tests/test_privacy_urls.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- import re import mock diff --git a/readthedocs/rtd_tests/tests/test_restapi_client.py b/readthedocs/rtd_tests/tests/test_restapi_client.py index 8e5b5386d53..fe094eee94e 100644 --- a/readthedocs/rtd_tests/tests/test_restapi_client.py +++ b/readthedocs/rtd_tests/tests/test_restapi_client.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- from django.test import TestCase from readthedocs.api.v2.client import DrfJsonSerializer diff --git a/readthedocs/urls.py b/readthedocs/urls.py index abf46fd034a..14708f1cfcc 100644 --- a/readthedocs/urls.py +++ b/readthedocs/urls.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # pylint: disable=missing-docstring import os from functools import reduce diff --git a/requirements/pip.txt b/requirements/pip.txt index 5d4afd9e1dd..9b423f92b73 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -78,7 +78,7 @@ stripe==2.27.0 git+https://github.com/mozilla/unicode-slugify@b696c37#egg=unicode-slugify==0.1.5 django-formtools==2.1 -django-crispy-forms==1.7.2 +django-crispy-forms==1.7.2 docker==3.7.2 From 5dfc7bbb01f009aef38d8db69facd947a963a376 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 1 May 2019 18:55:31 +0200 Subject: [PATCH 77/96] Improve/Fix build triggering --- readthedocs/api/v3/mixins.py | 19 ++-- .../projects-versions-builds-list_POST.json | 96 +++++++++++++++++++ readthedocs/api/v3/tests/test_projects.py | 21 ++++ readthedocs/api/v3/urls.py | 3 +- readthedocs/api/v3/views.py | 11 ++- 5 files changed, 136 insertions(+), 14 deletions(-) create mode 100644 readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py index 3a4737b5cfc..a6282d05df7 100644 --- a/readthedocs/api/v3/mixins.py +++ b/readthedocs/api/v3/mixins.py @@ -21,22 +21,25 @@ class NestedParentObjectMixin: 'version__slug', ] - def _get_parent_object(self, model, lookup_names): - object_slug = None + def _get_parent_object_lookup(self, lookup_names): query_dict = self.get_parents_query_dict() for lookup in lookup_names: value = query_dict.get(lookup) if value: - object_slug = value - break - - return get_object_or_404(model, slug=object_slug) + return value def _get_parent_project(self): - return self._get_parent_object(Project, self.PROJECT_LOOKUP_NAMES) + slug = self._get_parent_object_lookup(self.PROJECT_LOOKUP_NAMES) + return get_object_or_404(Project, slug=slug) def _get_parent_version(self): - return self._get_parent_object(Version, self.VERSION_LOOKUP_NAMES) + project_slug = self._get_parent_object_lookup(self.PROJECT_LOOKUP_NAMES) + slug = self._get_parent_object_lookup(self.VERSION_LOOKUP_NAMES) + return get_object_or_404( + Version, + slug=slug, + project__slug=project_slug, + ) class APIAuthMixin(NestedParentObjectMixin): diff --git a/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json b/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json new file mode 100644 index 00000000000..b0f3dc15dc5 --- /dev/null +++ b/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json @@ -0,0 +1,96 @@ +{ + "build": { + "builder": null, + "cold_storage": null, + "commit": null, + "created": "2019-04-29T14:00:00Z", + "duration": null, + "error": "", + "finished": null, + "id": 2, + "links": { + "_self": "https://readthedocs.org/api/v3/projects/project/builds/2/", + "project": "https://readthedocs.org/api/v3/projects/project/", + "version": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/" + }, + "project": "project", + "state": { + "code": "triggered", + "name": "Triggered" + }, + "success": true, + "version": "v1.0" + }, + "project": { + "created": "2019-04-29T10:00:00Z", + "default_branch": "master", + "default_version": "latest", + "description": "Project description", + "id": 1, + "language": { + "code": "en", + "name": "English" + }, + "links": { + "_self": "https://readthedocs.org/api/v3/projects/project/", + "builds": "https://readthedocs.org/api/v3/projects/project/builds/", + "subprojects": "https://readthedocs.org/api/v3/projects/project/subprojects/", + "superproject": "https://readthedocs.org/api/v3/projects/project/superproject/", + "translations": "https://readthedocs.org/api/v3/projects/project/translations/", + "users": "https://readthedocs.org/api/v3/projects/project/users/", + "versions": "https://readthedocs.org/api/v3/projects/project/versions/" + }, + "modified": "2019-04-29T12:00:00Z", + "name": "project", + "privacy_level": { + "code": "public", + "name": "Public" + }, + "programming_language": { + "code": "words", + "name": "Only Words" + }, + "repository": { + "type": "git", + "url": "https://github.com/rtfd/project" + }, + "slug": "project", + "subproject_of": null, + "tags": [ + "tag", + "project", + "test" + ], + "translation_of": null, + "urls": { + "documentation": "http://readthedocs.org/docs/project/en/latest/", + "project_homepage": "http://project.com" + } + }, + "triggered": true, + "version": { + "active": true, + "built": true, + "downloads": {}, + "id": 3, + "identifier": "a1b2c3", + "links": { + "_self": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/", + "builds": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/builds/", + "project": "https://readthedocs.org/api/v3/projects/project/" + }, + "privacy_level": { + "code": "public", + "name": "Public" + }, + "ref": null, + "slug": "v1.0", + "type": "tag", + "uploaded": false, + "urls": { + "documentation": "http://readthedocs.org/docs/project/en/v1.0/", + "vcs": "https://github.com/rtfd/project/tree/v1.0/" + }, + "verbose_name": "v1.0" + } +} diff --git a/readthedocs/api/v3/tests/test_projects.py b/readthedocs/api/v3/tests/test_projects.py index cc66366ed4f..c9307c5a000 100644 --- a/readthedocs/api/v3/tests/test_projects.py +++ b/readthedocs/api/v3/tests/test_projects.py @@ -323,3 +323,24 @@ def test_projects_builds_detail(self): response.json(), self._get_response_dict('projects-builds-detail'), ) + + def test_projects_versions_builds_list_post(self): + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + self.assertEqual(self.project.builds.count(), 1) + response = self.client.post( + reverse( + 'projects-versions-builds-list', + kwargs={ + 'parent_lookup_project__slug': self.project.slug, + 'parent_lookup_version__slug': self.version.slug, + }), + ) + self.assertEqual(response.status_code, 202) + self.assertEqual(self.project.builds.count(), 2) + + response_json = response.json() + response_json['build']['created'] = '2019-04-29T14:00:00Z' + self.assertDictEqual( + response_json, + self._get_response_dict('projects-versions-builds-list_POST'), + ) diff --git a/readthedocs/api/v3/urls.py b/readthedocs/api/v3/urls.py index d055dd85b98..abff5078db9 100644 --- a/readthedocs/api/v3/urls.py +++ b/readthedocs/api/v3/urls.py @@ -1,6 +1,7 @@ from .routers import DefaultRouterWithNesting from .views import ( BuildsViewSet, + BuildsCreateViewSet, ProjectsViewSet, SubprojectRelationshipViewSet, TranslationRelationshipViewSet, @@ -49,7 +50,7 @@ # allows /api/v3/projects/pip/versions/v3.6.2/builds/1053/ versions.register( r'builds', - BuildsViewSet, + BuildsCreateViewSet, base_name='projects-versions-builds', parents_query_lookups=[ 'project__slug', diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 65ba90c6061..a50105da5b4 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -251,20 +251,21 @@ def update(self, request, *args, **kwargs): class BuildsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, - CreateModelMixin, GenericViewSet): + GenericViewSet): model = Build lookup_field = 'pk' lookup_url_kwarg = 'build_pk' + serializer_class = BuildSerializer filterset_class = BuildFilter queryset = Build.objects.all() permit_list_expands = [ 'config', ] - def get_serializer_class(self): - if self.action in ('list', 'retrieve'): - return BuildSerializer - return BuildCreateSerializer + +class BuildsCreateViewSet(BuildsViewSet, CreateModelMixin): + + serializer_class = BuildCreateSerializer def create(self, request, **kwargs): # pylint: disable=arguments-differ project = self._get_parent_project() From c034972aee7abaf7e6d3e30b0d441fd07e8be6ad Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 1 May 2019 19:08:41 +0200 Subject: [PATCH 78/96] Fix BrowsableAPIRenderer /projects/ --- readthedocs/api/v3/permissions.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/readthedocs/api/v3/permissions.py b/readthedocs/api/v3/permissions.py index d3e686aa08e..d822b9a1fea 100644 --- a/readthedocs/api/v3/permissions.py +++ b/readthedocs/api/v3/permissions.py @@ -14,7 +14,10 @@ class PublicDetailPrivateListing(IsAuthenticated): def has_permission(self, request, view): is_authenticated = super().has_permission(request, view) if is_authenticated: - if view.basename == 'projects' and view.action == 'list': + if view.basename == 'projects' and any([ + view.action == 'list', + view.action is None, # needed for BrowsableAPIRenderer + ]): # hitting ``/projects/``, allowing return True From fa0dc1f7a1d1291878db37b1a3ae1978035323e7 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 1 May 2019 19:31:51 +0200 Subject: [PATCH 79/96] Fix serializer on Build --- readthedocs/api/v3/views.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index a50105da5b4..4ec768e50a7 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -265,7 +265,11 @@ class BuildsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, class BuildsCreateViewSet(BuildsViewSet, CreateModelMixin): - serializer_class = BuildCreateSerializer + def get_serializer_class(self): + if self.action == 'create': + return BuildCreateSerializer + + return super().get_serializer_class() def create(self, request, **kwargs): # pylint: disable=arguments-differ project = self._get_parent_project() From e0b716a9e11098cf574f325671547efbfdfc79db Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 1 May 2019 19:32:27 +0200 Subject: [PATCH 80/96] More docstrings --- readthedocs/api/v3/routers.py | 2 ++ readthedocs/api/v3/views.py | 21 +++++++++++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/readthedocs/api/v3/routers.py b/readthedocs/api/v3/routers.py index 47bfa613d12..5a2e225eb7f 100644 --- a/readthedocs/api/v3/routers.py +++ b/readthedocs/api/v3/routers.py @@ -9,6 +9,8 @@ class DocsAPIRootView(APIRootView): """ Read the Docs APIv3 root endpoint. + API is browseable by sending the header ``Authorization: Token `` on each request. + Full documentation at [https://docs.readthedocs.io/en/latest/api/v3.html](https://docs.readthedocs.io/en/latest/api/v3.html). """ # noqa diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 4ec768e50a7..53eee225e01 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -77,6 +77,7 @@ class ProjectsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, * Detailed object. Retrieving only needed data using ``?fields=`` URL attribute is allowed. + On the other hand, you can use ``?omit=`` and list the fields you want to skip in the response. ### Filters @@ -113,9 +114,9 @@ class ProjectsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, * Subprojects of a project: ``/api/v3/projects/{project_slug}/subprojects/`` * Superproject of a project: ``/api/v3/projects/{project_slug}/superproject/`` - Go to https://docs.readthedocs.io/en/stable/api/v3.html + Go to [https://docs.readthedocs.io/en/stable/api/v3.html](https://docs.readthedocs.io/en/stable/api/v3.html) for a complete documentation of the APIv3. - """ + """ # noqa model = Project lookup_field = 'slug' @@ -160,6 +161,9 @@ def get_view_description(self, *args, **kwargs): # pylint: disable=arguments-di @action(detail=True, methods=['get']) def superproject(self, request, project_slug): + """ + Return the superproject of a ``Project``. + """ project = self.get_object() try: superproject = project.superprojects.first().parent @@ -173,9 +177,13 @@ class SubprojectRelationshipViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, GenericViewSet): + # Markdown docstring exposed at BrowsableAPIRenderer. """ List subprojects of a ``Project``. + """ + # Private/Internal docstring + """ The main query is done via the ``NestedViewSetMixin`` using the ``parents_query_lookups`` defined when registering the urls. """ @@ -191,9 +199,13 @@ class TranslationRelationshipViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, GenericViewSet): + # Markdown docstring exposed at BrowsableAPIRenderer. """ List translations of a ``Project``. + """ + # Private/Internal docstring + """ The main query is done via the ``NestedViewSetMixin`` using the ``parents_query_lookups`` defined when registering the urls. """ @@ -296,6 +308,11 @@ def create(self, request, **kwargs): # pylint: disable=arguments-differ class UsersViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, ListModelMixin, RetrieveModelMixin, GenericViewSet): + + """ + List users of a ``Project``. + """ + model = User lookup_field = 'username' lookup_url_kwarg = 'user_username' From c4da1b71bf302e3c187118cdf7c9952a6a795765 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 1 May 2019 19:42:02 +0200 Subject: [PATCH 81/96] Valid success field on Build object --- readthedocs/api/v3/serializers.py | 12 ++++++++++++ .../projects-versions-builds-list_POST.json | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index 0a460dccc67..66856a8cd8b 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -111,6 +111,7 @@ class BuildSerializer(FlexFieldsModelSerializer): version = serializers.SlugRelatedField(slug_field='slug', read_only=True) created = serializers.DateTimeField(source='date') finished = serializers.SerializerMethodField() + success = serializers.SerializerMethodField() duration = serializers.IntegerField(source='length') state = BuildStateSerializer(source='*') links = BuildLinksSerializer(source='*') @@ -146,6 +147,17 @@ def get_finished(self, obj): if obj.date and obj.length: return obj.date + datetime.timedelta(seconds=obj.length) + def get_success(self, obj): + """ + Return ``None`` if the build is not finished. + + This is needed becase ``default=True`` in the model field. + """ + if obj.finished: + return obj.success + + return None + class PrivacyLevelSerializer(serializers.Serializer): code = serializers.CharField(source='privacy_level') diff --git a/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json b/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json index b0f3dc15dc5..d1b5a1e5b39 100644 --- a/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json +++ b/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json @@ -18,7 +18,7 @@ "code": "triggered", "name": "Triggered" }, - "success": true, + "success": null, "version": "v1.0" }, "project": { From e3a58e2aafd8140faece89b8d20d15c16fe9e47b Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 1 May 2019 20:10:03 +0200 Subject: [PATCH 82/96] Use `created` instead of `date_joined` on User response --- readthedocs/api/v3/serializers.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index 66856a8cd8b..5417e7ff434 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -15,11 +15,13 @@ class UserSerializer(FlexFieldsModelSerializer): + created = serializers.DateTimeField(source='date_joined') + class Meta: model = User fields = [ 'username', - 'date_joined', + 'created', 'last_login', ] From 86d8865e83cdbc53938f78a9e01825ac37f2bf79 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 1 May 2019 20:10:28 +0200 Subject: [PATCH 83/96] Update the APIv3 docs to the current state of the implementation --- docs/api/v3.rst | 172 +++++++----------------------------------------- 1 file changed, 22 insertions(+), 150 deletions(-) diff --git a/docs/api/v3.rst b/docs/api/v3.rst index 571ccbe6b99..6af4eb372c4 100644 --- a/docs/api/v3.rst +++ b/docs/api/v3.rst @@ -14,15 +14,7 @@ Authentication and authorization -------------------------------- Requests to the Read the Docs public API are for public and private information. -Some listing endpoints and some detailed ones are allowed to access without being authenticated. -Although, all endpoints that perform non-readonly operations, require authentication. - - -No authentication -~~~~~~~~~~~~~~~~~ - -Some endpoints on read-only requests won't require any kind of authentication. -Most of them are detailed information of specific listings for a particular object. +All endpoints require authentication. Token @@ -35,6 +27,10 @@ and have the same permissions that the user itself. Session ~~~~~~~ +.. warning:: + + Authentication via session is not enabled yet. + Session authentication is allowed on very specific endpoints, to allow hitting the API when reading documentation. @@ -86,7 +82,7 @@ Projects list :query string programming_language: programming language code as ``py``, ``js``, etc. :query string repository_type: one of ``git``, ``hg``, ``bzr``, ``svn``. - :requestheader Authorization: required token to authenticate. + :requestheader Authorization: token to authenticate. Project details @@ -100,7 +96,7 @@ Project details .. sourcecode:: bash - $ curl https://readthedocs.org/api/v3/projects/pip/ + $ curl -H "Authorization: Token " https://readthedocs.org/api/v3/projects/pip/ **Example response**: @@ -171,9 +167,7 @@ Project details .. TODO: complete the returned data docs once agreed on this. - :query boolean inactive_versions: whether or not include inactive versions. - - :requestheader Authorization: optional token to authenticate. + :requestheader Authorization: token to authenticate. :statuscode 200: Success :statuscode 404: There is no ``Project`` with this slug @@ -202,7 +196,7 @@ Versions listing .. sourcecode:: bash - $ curl https://readthedocs.org/api/v3/projects/pip/versions/ + $ curl -H "Authorization: Token " https://readthedocs.org/api/v3/projects/pip/versions/ **Example response**: @@ -228,7 +222,7 @@ Versions listing :query boolean active: whether return active versions only :query boolean built: whether return only built version - :requestheader Authorization: optional token to authenticate. + :requestheader Authorization: token to authenticate. Version detail @@ -242,7 +236,7 @@ Version detail .. sourcecode:: bash - $ curl https://readthedocs.org/api/v3/projects/pip/versions/stable/ + $ curl -H "Authorization: Token " https://readthedocs.org/api/v3/projects/pip/versions/stable/ **Example response**: @@ -290,7 +284,7 @@ Version detail :>json string last_build: Build object representing the last build of this version :>json array downloads: URLs to downloads of this version's documentation - :requestheader Authorization: optional token to authenticate. + :requestheader Authorization: token to authenticate. :statuscode 200: Success :statuscode 404: There is no ``Version`` with this slug for this project @@ -312,11 +306,7 @@ Version edit "privacy_level": "public" } - **Example response**: - - `See Version details <#version-detail>`_ - - :requestheader Authorization: required token to authenticate. + :requestheader Authorization: token to authenticate. :statuscode 204: Edited sucessfully :statuscode 400: Some field is invalid @@ -347,7 +337,7 @@ Build details .. sourcecode:: bash - $ curl https://readthedocs.org/api/v3/projects/pip/builds/8592686/?expand=config + $ curl -H "Authorization: Token " https://readthedocs.org/api/v3/projects/pip/builds/8592686/?expand=config **Example response**: @@ -425,29 +415,12 @@ Build details :query boolean include_config: whether or not include the configs used for this build. Default is ``false`` - :requestheader Authorization: optional token to authenticate. + :requestheader Authorization: token to authenticate. :statuscode 200: Success :statuscode 404: There is no ``Build`` with this ID -.. http:get:: /api/v3/projects/(str:project_slug)/builds/latest/ - - Retrieve details for latest build on this project. - - **Example request**: - - .. sourcecode:: bash - - $ curl https://readthedocs.org/api/v3/projects/pip/builds/latest/ - - **Example response**: - - `See Build details <#build-details>`_ - - :requestheader Authorization: optional token to authenticate. - - Builds listing ++++++++++++++ @@ -475,28 +448,13 @@ Builds listing :query string commit: commit hash to filter the builds returned by commit :query boolean running: whether or not to filter the builds returned by currently building - :requestheader Authorization: optional token to authenticate. + :requestheader Authorization: token to authenticate. Build triggering ++++++++++++++++ -.. http:post:: /api/v3/projects/(string:project_slug)/builds/ - - Trigger a new build for the default version of this project. - - **Example response**: - - `See Build details <#build-details>`_ - - :requestheader Authorization: required token to authenticate. - - :statuscode 202: Accepted - :statuscode 400: Some field is invalid - :statuscode 401: Not valid permissions - - .. http:post:: /api/v3/projects/(string:project_slug)/versions/(string:version_slug)/builds/ Trigger a new build for the ``version_slug`` version of this project. @@ -505,79 +463,13 @@ Build triggering `See Build details <#build-details>`_ - :requestheader Authorization: required token to authenticate. + :requestheader Authorization: token to authenticate. :statuscode 202: Accepted :statuscode 400: Some field is invalid :statuscode 401: Not valid permissions -Build commands listing -++++++++++++++++++++++ - -.. http:get:: /api/v3/projects/(str:project_slug)/builds/(int:build_id)/commands/ - - Retrieve build command list of a single build. - - **Example request**: - - .. sourcecode:: bash - - $ curl https://readthedocs.org/api/v3/projects/pip/builds/719263915/commands/ - - **Example response**: - - .. sourcecode:: json - - { - "count": 15, - "next": "/api/v3/projects/pip/builds/719263915/commands/?limit=10&offset=10", - "previous": null, - "results": ["BUILDCOMMAND"] - } - - :requestheader Authorization: optional token to authenticate. - - -Build command details -+++++++++++++++++++++ - -.. http:get:: /api/v3/projects/(str:project_slug)/builds/(int:build_id)/commands/(int:buildcommand_id) - - Retrieve build command detail. - - **Example request**: - - .. sourcecode:: bash - - $ curl https://readthedocs.org/api/v3/projects/pip/builds/719263915/commands/9182639172/ - - **Example response**: - - .. sourcecode:: json - - { - "id": 9182639172, - "build": 719263915, - "project": "pip", - "version": "stable", - "created": "2018-06-19T15:15:59+00:00", - "finished": "2018-06-19T15:16:58+00:00", - "duration": 59, - "command": "cat docs/config.py", - "output": "...", - "exit_code": 0, - "links": { - "self": "/api/v3/projects/pip/builds/719263915/commands/9182639172/", - "build": "/api/v3/projects/pip/builds/719263915/", - "version": "/api/v3/projects/pip/versions/stable/", - "project": "/api/v3/projects/pip/" - } - } - - :requestheader Authorization: optional token to authenticate. - - Users ~~~~~ @@ -600,36 +492,16 @@ User detail .. sourcecode:: json { - "id": 25, "username": "humitos", "created": "2008-10-23T18:12:31+00:00", - "last_login": "2010-10-23T18:12:31+00:00", - "first_name": "Manuel", - "last_name": "Kaufmann", - "email": "humitos@readthedocs.org", - "links": { - "self": "/api/v3/users/humitos/", - "projects": "/api/v3/projects/?user=humitos" - } + "last_login": "2010-10-23T18:12:31+00:00" } - .. TODO: considering that ``/api/v3/projects/`` will return only - the projects for the authenticated user, the ``projects`` link - here won't work. - - On the other hand, ``/api/v3/projects/all/?user=humitos`` can't - be used because we will be mixing ``all`` as project slug with - our endpoint URL. - - :>json integer id: ID for the user on the database. :>json string username: username for the user. :>json string created: date and time when the user was created. :>json string last_login: date and time for last time this user was logged in. - :>json string first_name: first name of the user. - :>json string last_name: last name of the user. - :>json string email: email of the user. - :requestheader Authorization: required token to authenticate. + :requestheader Authorization: token to authenticate. User listing @@ -662,7 +534,7 @@ User listing :>json string previous: URI for previous set of users. :>json array results: array of ``user`` objects. - :requestheader Authorization: optional token to authenticate. + :requestheader Authorization: token to authenticate. Subprojects @@ -698,7 +570,7 @@ Subprojects listing :>json string previous: URI for previous set of projects. :>json array results: array of ``project`` objects. - :requestheader Authorization: required token to authenticate. + :requestheader Authorization: token to authenticate. Translations @@ -734,4 +606,4 @@ Translations listing :>json string previous: URI for previous set of projects. :>json array results: array of ``project`` objects. - :requestheader Authorization: required token to authenticate. + :requestheader Authorization: token to authenticate. From 2e40de910fc9a6e71fb3de22d4a6e530fc1d10d6 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 6 May 2019 12:43:56 +0200 Subject: [PATCH 84/96] Increase logged in throttling to 60/minute --- readthedocs/settings/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/settings/base.py b/readthedocs/settings/base.py index fd59d34e4aa..88f28d0a8fe 100644 --- a/readthedocs/settings/base.py +++ b/readthedocs/settings/base.py @@ -477,7 +477,7 @@ def USE_PROMOS(self): # noqa 'DEFAULT_PAGINATION_CLASS': 'rest_framework.pagination.LimitOffsetPagination', # NOQA 'DEFAULT_THROTTLE_RATES': { 'anon': '5/minute', - 'user': '10/minute', + 'user': '60/minute', }, 'PAGE_SIZE': 10, } From 9ae181428f4be8696bb9d780ba658f4e31bf5ab9 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 6 May 2019 12:44:19 +0200 Subject: [PATCH 85/96] Typo --- readthedocs/api/v3/mixins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py index a6282d05df7..1332fd810a2 100644 --- a/readthedocs/api/v3/mixins.py +++ b/readthedocs/api/v3/mixins.py @@ -48,7 +48,7 @@ class APIAuthMixin(NestedParentObjectMixin): Mixin to define queryset permissions for ViewSet only in one place. All APIv3 ViewSet should inherit this mixin, unless specific permissions - required. In that case, an specific mixin for that case should be defined. + required. In that case, a specific mixin for that case should be defined. """ def detail_objects(self, queryset, user): From fa5c8237daad630ab2529231ecde386810811e5f Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 6 May 2019 13:20:01 +0200 Subject: [PATCH 86/96] Always expand users in Project details --- readthedocs/api/v3/serializers.py | 9 ++------ .../v3/tests/responses/projects-detail.json | 8 ++++++- readthedocs/api/v3/tests/test_projects.py | 21 ++++++++++++------- readthedocs/api/v3/views.py | 2 +- 4 files changed, 24 insertions(+), 16 deletions(-) diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index 5417e7ff434..9e0292e50b8 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -400,6 +400,7 @@ class ProjectSerializer(FlexFieldsModelSerializer): translation_of = serializers.SerializerMethodField() default_branch = serializers.CharField(source='get_default_branch') tags = serializers.StringRelatedField(many=True) + users = UserSerializer(many=True) description = serializers.SerializerMethodField() @@ -411,13 +412,6 @@ class ProjectSerializer(FlexFieldsModelSerializer): modified = serializers.DateTimeField(source='modified_date') expandable_fields = dict( - users=( - UserSerializer, - dict( - source='users', - many=True, - ), - ), active_versions=( VersionSerializer, dict( @@ -446,6 +440,7 @@ class Meta: 'privacy_level', 'subproject_of', 'translation_of', + 'users', 'urls', 'tags', diff --git a/readthedocs/api/v3/tests/responses/projects-detail.json b/readthedocs/api/v3/tests/responses/projects-detail.json index 0ee649be291..59500331cab 100644 --- a/readthedocs/api/v3/tests/responses/projects-detail.json +++ b/readthedocs/api/v3/tests/responses/projects-detail.json @@ -94,5 +94,11 @@ "urls": { "documentation": "http://readthedocs.org/docs/project/en/latest/", "project_homepage": "http://project.com" - } + }, + "users": [ + { + "created": "2019-04-29T10:00:00Z", + "username": "testuser" + } + ] } diff --git a/readthedocs/api/v3/tests/test_projects.py b/readthedocs/api/v3/tests/test_projects.py index c9307c5a000..b37f49540e8 100644 --- a/readthedocs/api/v3/tests/test_projects.py +++ b/readthedocs/api/v3/tests/test_projects.py @@ -19,15 +19,22 @@ class APIEndpointTests(TestCase): fixtures = [] def setUp(self): - - self.me = fixture.get(User, projects=[]) + created = datetime.datetime(2019, 4, 29, 10, 0, 0) + modified = datetime.datetime(2019, 4, 29, 12, 0, 0) + + self.me = fixture.get( + User, + date_joined=created, + username='testuser', + projects=[], + ) self.token = fixture.get(Token, key='me', user=self.me) # Defining all the defaults helps to avoid creating ghost / unwanted # objects (like a Project for translations/subprojects) self.project = fixture.get( Project, - pub_date=datetime.datetime(2019, 4, 29, 10, 0, 0), - modified_date=datetime.datetime(2019, 4, 29, 12, 0, 0), + pub_date=created, + modified_date=modified, description='Project description', repo='https://github.com/rtfd/project', project_url='http://project.com', @@ -43,8 +50,8 @@ def setUp(self): self.subproject = fixture.get( Project, - pub_date=datetime.datetime(2019, 4, 29, 10, 0, 0), - modified_date=datetime.datetime(2019, 4, 29, 12, 0, 0), + pub_date=created, + modified_date=modified, description='SubProject description', repo='https://github.com/rtfd/subproject', project_url='http://subproject.com', @@ -73,7 +80,7 @@ def setUp(self): self.build = fixture.get( Build, - date=datetime.datetime(2019, 4, 29, 10, 0, 0), + date=created, type='html', state='finished', error='', diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 53eee225e01..bbbed9fbbdb 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -125,7 +125,6 @@ class ProjectsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, filterset_class = ProjectFilter queryset = Project.objects.all() permit_list_expands = [ - 'users', 'active_versions', 'active_versions.last_build', 'active_versions.last_build.config', @@ -140,6 +139,7 @@ def get_queryset(self): 'related_projects', 'domains', 'tags', + 'users', ) def get_view_description(self, *args, **kwargs): # pylint: disable=arguments-differ From b634daa87dae80c3adc6193696b6b60aca5ef723 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 6 May 2019 13:20:47 +0200 Subject: [PATCH 87/96] Move specific list action on projects view to the specific view --- readthedocs/api/v3/mixins.py | 7 ------- readthedocs/api/v3/views.py | 6 ++++++ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py index 1332fd810a2..2d7e6e961da 100644 --- a/readthedocs/api/v3/mixins.py +++ b/readthedocs/api/v3/mixins.py @@ -87,13 +87,6 @@ def get_queryset(self): 4. raise a ``NotFound`` exception otherwise """ - # Allow hitting ``/api/v3/projects/`` to list their own projects - if self.basename == 'projects' and self.action == 'list': - # We force returning ``Project`` objects here because it's under the - # ``projects`` view. This could be moved to a specific - # ``get_queryset`` in the view. - return self.admin_projects(self.request.user) - # NOTE: ``super().get_queryset`` produces the filter by ``NestedViewSetMixin`` # we need to have defined the class attribute as ``queryset = Model.objects.all()`` queryset = super().get_queryset() diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index bbbed9fbbdb..a13d715006d 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -131,6 +131,12 @@ class ProjectsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, ] def get_queryset(self): + # Allow hitting ``/api/v3/projects/`` to list their own projects + if self.basename == 'projects' and self.action == 'list': + # We force returning ``Project`` objects here because it's under the + # ``projects`` view. + return self.admin_projects(self.request.user) + # This could be a class attribute and managed on the ``APIAuthMixin`` in # case we want to extend the ``prefetch_related`` to other views as # well. From 4f1f587a08bc705c4e2e486ac9b38c8f56a038c4 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 6 May 2019 13:22:19 +0200 Subject: [PATCH 88/96] Remove /projects/pip//users/ endpoint --- readthedocs/api/v3/urls.py | 10 ---------- readthedocs/api/v3/views.py | 16 ---------------- 2 files changed, 26 deletions(-) diff --git a/readthedocs/api/v3/urls.py b/readthedocs/api/v3/urls.py index abff5078db9..71b545a3201 100644 --- a/readthedocs/api/v3/urls.py +++ b/readthedocs/api/v3/urls.py @@ -5,7 +5,6 @@ ProjectsViewSet, SubprojectRelationshipViewSet, TranslationRelationshipViewSet, - UsersViewSet, VersionsViewSet, ) @@ -67,14 +66,5 @@ parents_query_lookups=['project__slug'], ) -# allows /api/v3/projects/pip/users/ -# allows /api/v3/projects/pip/users/humitos/ -projects.register( - r'users', - UsersViewSet, - base_name='projects-users', - parents_query_lookups=['projects__slug'], -) - urlpatterns = [] urlpatterns += router.urls diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index a13d715006d..53054ed695b 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -1,5 +1,4 @@ import django_filters.rest_framework as filters -from django.contrib.auth.models import User from django.utils.safestring import mark_safe from rest_flex_fields.views import FlexFieldsMixin from rest_framework.authentication import TokenAuthentication @@ -30,7 +29,6 @@ BuildCreateSerializer, BuildSerializer, ProjectSerializer, - UserSerializer, VersionSerializer, VersionUpdateSerializer, ) @@ -310,17 +308,3 @@ def create(self, request, **kwargs): # pylint: disable=arguments-differ data.update({'triggered': False}) status = 400 return Response(data=data, status=status) - - -class UsersViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, - ListModelMixin, RetrieveModelMixin, GenericViewSet): - - """ - List users of a ``Project``. - """ - - model = User - lookup_field = 'username' - lookup_url_kwarg = 'user_username' - serializer_class = UserSerializer - queryset = User.objects.all() From 741e01775f5de10b476560ebee8c98e23f8ac330 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 6 May 2019 13:22:39 +0200 Subject: [PATCH 89/96] Docstring URL typo --- readthedocs/api/v3/routers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readthedocs/api/v3/routers.py b/readthedocs/api/v3/routers.py index 5a2e225eb7f..90588c0997d 100644 --- a/readthedocs/api/v3/routers.py +++ b/readthedocs/api/v3/routers.py @@ -11,7 +11,7 @@ class DocsAPIRootView(APIRootView): API is browseable by sending the header ``Authorization: Token `` on each request. - Full documentation at [https://docs.readthedocs.io/en/latest/api/v3.html](https://docs.readthedocs.io/en/latest/api/v3.html). + Full documentation at [https://docs.readthedocs.io/page/api/v3.html](https://docs.readthedocs.io/page/api/v3.html). """ # noqa def get_view_name(self): From ade539242aaeee92ee8ba38c6ad64318fa4c3158 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 6 May 2019 13:26:45 +0200 Subject: [PATCH 90/96] User endpoint does not exist anymore No need to make this exception here. --- readthedocs/api/v3/mixins.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py index 2d7e6e961da..6c29bd0efda 100644 --- a/readthedocs/api/v3/mixins.py +++ b/readthedocs/api/v3/mixins.py @@ -53,12 +53,7 @@ class APIAuthMixin(NestedParentObjectMixin): def detail_objects(self, queryset, user): # Filter results by user - # NOTE: we don't override the manager in User model, so we don't have - # ``.api`` method there - if self.model is not User: - queryset = queryset.api(user=user) - - return queryset + return queryset.api(user=user) def listing_objects(self, queryset, user): project = self._get_parent_project() From 71a653915862f98248b0492d1a297c85d55e36ea Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 6 May 2019 13:42:50 +0200 Subject: [PATCH 91/96] Remove some fields from responses --- readthedocs/api/v3/serializers.py | 14 -------------- .../v3/tests/responses/projects-builds-detail.json | 2 -- .../api/v3/tests/responses/projects-detail.json | 4 ---- .../api/v3/tests/responses/projects-list.json | 7 ++++++- .../tests/responses/projects-subprojects-list.json | 11 ++++++++--- .../v3/tests/responses/projects-superproject.json | 9 +++++++-- .../projects-versions-builds-list_POST.json | 12 +++++++----- .../tests/responses/projects-versions-detail.json | 1 - readthedocs/api/v3/tests/test_projects.py | 12 ------------ 9 files changed, 28 insertions(+), 44 deletions(-) diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index 9e0292e50b8..2ddb1ad343c 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -22,7 +22,6 @@ class Meta: fields = [ 'username', 'created', - 'last_login', ] @@ -140,8 +139,6 @@ class Meta: 'success', 'error', 'commit', - 'builder', - 'cold_storage', 'links', ] @@ -239,7 +236,6 @@ class Meta: 'ref', 'built', 'active', - 'uploaded', 'privacy_level', 'type', 'downloads', @@ -323,7 +319,6 @@ class ProjectLinksSerializer(BaseLinksSerializer): _self = serializers.SerializerMethodField() - users = serializers.SerializerMethodField() versions = serializers.SerializerMethodField() builds = serializers.SerializerMethodField() subprojects = serializers.SerializerMethodField() @@ -334,15 +329,6 @@ def get__self(self, obj): path = reverse('projects-detail', kwargs={'project_slug': obj.slug}) return self._absolute_url(path) - def get_users(self, obj): - path = reverse( - 'projects-users-list', - kwargs={ - 'parent_lookup_projects__slug': obj.slug, - }, - ) - return self._absolute_url(path) - def get_versions(self, obj): path = reverse( 'projects-versions-list', diff --git a/readthedocs/api/v3/tests/responses/projects-builds-detail.json b/readthedocs/api/v3/tests/responses/projects-builds-detail.json index 2a6d9e29b74..95b6f3ff261 100644 --- a/readthedocs/api/v3/tests/responses/projects-builds-detail.json +++ b/readthedocs/api/v3/tests/responses/projects-builds-detail.json @@ -1,6 +1,4 @@ { - "builder": "builder01", - "cold_storage": null, "commit": "a1b2c3", "created": "2019-04-29T10:00:00Z", "duration": 60, diff --git a/readthedocs/api/v3/tests/responses/projects-detail.json b/readthedocs/api/v3/tests/responses/projects-detail.json index 59500331cab..93c511ddd00 100644 --- a/readthedocs/api/v3/tests/responses/projects-detail.json +++ b/readthedocs/api/v3/tests/responses/projects-detail.json @@ -7,8 +7,6 @@ "id": 3, "identifier": "a1b2c3", "last_build": { - "builder": "builder01", - "cold_storage": null, "commit": "a1b2c3", "config": { "property": "test value" @@ -43,7 +41,6 @@ "ref": null, "slug": "v1.0", "type": "tag", - "uploaded": false, "urls": { "documentation": "http://readthedocs.org/docs/project/en/v1.0/", "vcs": "https://github.com/rtfd/project/tree/v1.0/" @@ -66,7 +63,6 @@ "subprojects": "https://readthedocs.org/api/v3/projects/project/subprojects/", "superproject": "https://readthedocs.org/api/v3/projects/project/superproject/", "translations": "https://readthedocs.org/api/v3/projects/project/translations/", - "users": "https://readthedocs.org/api/v3/projects/project/users/", "versions": "https://readthedocs.org/api/v3/projects/project/versions/" }, "modified": "2019-04-29T12:00:00Z", diff --git a/readthedocs/api/v3/tests/responses/projects-list.json b/readthedocs/api/v3/tests/responses/projects-list.json index 8fcd6134c11..3244f976144 100644 --- a/readthedocs/api/v3/tests/responses/projects-list.json +++ b/readthedocs/api/v3/tests/responses/projects-list.json @@ -39,9 +39,14 @@ "project", "test" ], + "users": [ + { + "created": "2019-04-29T10:00:00Z", + "username": "testuser" + } + ], "links": { "_self": "https://readthedocs.org/api/v3/projects/project/", - "users": "https://readthedocs.org/api/v3/projects/project/users/", "versions": "https://readthedocs.org/api/v3/projects/project/versions/", "builds": "https://readthedocs.org/api/v3/projects/project/builds/", "subprojects": "https://readthedocs.org/api/v3/projects/project/subprojects/", diff --git a/readthedocs/api/v3/tests/responses/projects-subprojects-list.json b/readthedocs/api/v3/tests/responses/projects-subprojects-list.json index dc0d9d08c3a..30638f84861 100644 --- a/readthedocs/api/v3/tests/responses/projects-subprojects-list.json +++ b/readthedocs/api/v3/tests/responses/projects-subprojects-list.json @@ -66,13 +66,18 @@ ], "links": { "_self": "https://readthedocs.org/api/v3/projects/project/", - "users": "https://readthedocs.org/api/v3/projects/project/users/", "versions": "https://readthedocs.org/api/v3/projects/project/versions/", "builds": "https://readthedocs.org/api/v3/projects/project/builds/", "subprojects": "https://readthedocs.org/api/v3/projects/project/subprojects/", "superproject": "https://readthedocs.org/api/v3/projects/project/superproject/", "translations": "https://readthedocs.org/api/v3/projects/project/translations/" - } + }, + "users": [ + { + "created": "2019-04-29T10:00:00Z", + "username": "testuser" + } + ] }, "translation_of": null, "urls": { @@ -80,9 +85,9 @@ "project_homepage": "http://subproject.com" }, "tags": [], + "users": [], "links": { "_self": "https://readthedocs.org/api/v3/projects/subproject/", - "users": "https://readthedocs.org/api/v3/projects/subproject/users/", "versions": "https://readthedocs.org/api/v3/projects/subproject/versions/", "builds": "https://readthedocs.org/api/v3/projects/subproject/builds/", "subprojects": "https://readthedocs.org/api/v3/projects/subproject/subprojects/", diff --git a/readthedocs/api/v3/tests/responses/projects-superproject.json b/readthedocs/api/v3/tests/responses/projects-superproject.json index 05ff4b25e4c..3ccf2f9e13d 100644 --- a/readthedocs/api/v3/tests/responses/projects-superproject.json +++ b/readthedocs/api/v3/tests/responses/projects-superproject.json @@ -14,7 +14,6 @@ "subprojects": "https://readthedocs.org/api/v3/projects/project/subprojects/", "superproject": "https://readthedocs.org/api/v3/projects/project/superproject/", "translations": "https://readthedocs.org/api/v3/projects/project/translations/", - "users": "https://readthedocs.org/api/v3/projects/project/users/", "versions": "https://readthedocs.org/api/v3/projects/project/versions/" }, "modified": "2019-04-29T12:00:00Z", @@ -42,5 +41,11 @@ "urls": { "documentation": "http://readthedocs.org/docs/project/en/latest/", "project_homepage": "http://project.com" - } + }, + "users": [ + { + "created": "2019-04-29T10:00:00Z", + "username": "testuser" + } + ] } diff --git a/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json b/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json index d1b5a1e5b39..6362a3efd4e 100644 --- a/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json +++ b/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json @@ -1,7 +1,5 @@ { "build": { - "builder": null, - "cold_storage": null, "commit": null, "created": "2019-04-29T14:00:00Z", "duration": null, @@ -37,7 +35,6 @@ "subprojects": "https://readthedocs.org/api/v3/projects/project/subprojects/", "superproject": "https://readthedocs.org/api/v3/projects/project/superproject/", "translations": "https://readthedocs.org/api/v3/projects/project/translations/", - "users": "https://readthedocs.org/api/v3/projects/project/users/", "versions": "https://readthedocs.org/api/v3/projects/project/versions/" }, "modified": "2019-04-29T12:00:00Z", @@ -65,7 +62,13 @@ "urls": { "documentation": "http://readthedocs.org/docs/project/en/latest/", "project_homepage": "http://project.com" - } + }, + "users": [ + { + "created": "2019-04-29T10:00:00Z", + "username": "testuser" + } + ] }, "triggered": true, "version": { @@ -86,7 +89,6 @@ "ref": null, "slug": "v1.0", "type": "tag", - "uploaded": false, "urls": { "documentation": "http://readthedocs.org/docs/project/en/v1.0/", "vcs": "https://github.com/rtfd/project/tree/v1.0/" diff --git a/readthedocs/api/v3/tests/responses/projects-versions-detail.json b/readthedocs/api/v3/tests/responses/projects-versions-detail.json index e7677416097..d519d492d3a 100644 --- a/readthedocs/api/v3/tests/responses/projects-versions-detail.json +++ b/readthedocs/api/v3/tests/responses/projects-versions-detail.json @@ -16,7 +16,6 @@ "ref": null, "slug": "v1.0", "type": "tag", - "uploaded": false, "urls": { "documentation": "http://readthedocs.org/docs/project/en/v1.0/", "vcs": "https://github.com/rtfd/project/tree/v1.0/" diff --git a/readthedocs/api/v3/tests/test_projects.py b/readthedocs/api/v3/tests/test_projects.py index b37f49540e8..a6b00b0a604 100644 --- a/readthedocs/api/v3/tests/test_projects.py +++ b/readthedocs/api/v3/tests/test_projects.py @@ -202,18 +202,6 @@ def test_others_projects_builds_list(self): ) self.assertEqual(response.status_code, 403) - def test_others_projects_users_list(self): - self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') - response = self.client.get( - reverse( - 'projects-users-list', - kwargs={ - 'parent_lookup_projects__slug': self.others_project.slug, - }), - - ) - self.assertEqual(response.status_code, 403) - def test_others_projects_detail(self): self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') response = self.client.get( From 6d3eee5e3c5436a6c3940b8f6c98b18bb8e6b3a9 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 6 May 2019 13:52:50 +0200 Subject: [PATCH 92/96] Use more generic viewset from DRF --- readthedocs/api/v3/views.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 53054ed695b..089eebd9be8 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -7,14 +7,13 @@ from rest_framework.mixins import ( CreateModelMixin, ListModelMixin, - RetrieveModelMixin, UpdateModelMixin, ) from rest_framework.pagination import LimitOffsetPagination from rest_framework.renderers import BrowsableAPIRenderer from rest_framework.response import Response from rest_framework.throttling import AnonRateThrottle, UserRateThrottle -from rest_framework.viewsets import GenericViewSet +from rest_framework.viewsets import GenericViewSet, ReadOnlyModelViewSet from rest_framework_extensions.mixins import NestedViewSetMixin from readthedocs.builds.models import Build, Version @@ -63,8 +62,7 @@ class APIv3Settings: class ProjectsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, - FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, - GenericViewSet): + FlexFieldsMixin, ReadOnlyModelViewSet): # Markdown docstring is automatically rendered by BrowsableAPIRenderer. @@ -222,8 +220,7 @@ class TranslationRelationshipViewSet(APIv3Settings, APIAuthMixin, class VersionsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, - FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, - UpdateModelMixin, GenericViewSet): + FlexFieldsMixin, UpdateModelMixin, ReadOnlyModelViewSet): model = Version lookup_field = 'slug' @@ -266,8 +263,7 @@ def update(self, request, *args, **kwargs): class BuildsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, - FlexFieldsMixin, ListModelMixin, RetrieveModelMixin, - GenericViewSet): + FlexFieldsMixin, ReadOnlyModelViewSet): model = Build lookup_field = 'pk' lookup_url_kwarg = 'build_pk' From 85b393c8425d8b3ac395d57202ef27bb9c9f681f Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 6 May 2019 13:57:20 +0200 Subject: [PATCH 93/96] Rename mixin to match what it does (QuerySet) --- readthedocs/api/v3/mixins.py | 2 +- readthedocs/api/v3/views.py | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py index 6c29bd0efda..2f583d6217f 100644 --- a/readthedocs/api/v3/mixins.py +++ b/readthedocs/api/v3/mixins.py @@ -42,7 +42,7 @@ def _get_parent_version(self): ) -class APIAuthMixin(NestedParentObjectMixin): +class ProjectQuerySetMixin(NestedParentObjectMixin): """ Mixin to define queryset permissions for ViewSet only in one place. diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 089eebd9be8..fcf041e9abd 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -21,7 +21,7 @@ from readthedocs.projects.models import Project from .filters import BuildFilter, ProjectFilter, VersionFilter -from .mixins import APIAuthMixin +from .mixins import ProjectQuerySetMixin from .permissions import PublicDetailPrivateListing from .renderers import AlphabeticalSortedJSONRenderer from .serializers import ( @@ -61,7 +61,7 @@ class APIv3Settings: metadata_class = SimpleMetadata -class ProjectsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, +class ProjectsViewSet(APIv3Settings, ProjectQuerySetMixin, NestedViewSetMixin, FlexFieldsMixin, ReadOnlyModelViewSet): # Markdown docstring is automatically rendered by BrowsableAPIRenderer. @@ -133,7 +133,7 @@ def get_queryset(self): # ``projects`` view. return self.admin_projects(self.request.user) - # This could be a class attribute and managed on the ``APIAuthMixin`` in + # This could be a class attribute and managed on the ``ProjectQuerySetMixin`` in # case we want to extend the ``prefetch_related`` to other views as # well. queryset = super().get_queryset() @@ -175,7 +175,7 @@ def superproject(self, request, project_slug): return Response(status=404) -class SubprojectRelationshipViewSet(APIv3Settings, APIAuthMixin, +class SubprojectRelationshipViewSet(APIv3Settings, ProjectQuerySetMixin, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, GenericViewSet): @@ -197,7 +197,7 @@ class SubprojectRelationshipViewSet(APIv3Settings, APIAuthMixin, queryset = Project.objects.all() -class TranslationRelationshipViewSet(APIv3Settings, APIAuthMixin, +class TranslationRelationshipViewSet(APIv3Settings, ProjectQuerySetMixin, NestedViewSetMixin, FlexFieldsMixin, ListModelMixin, GenericViewSet): @@ -219,7 +219,7 @@ class TranslationRelationshipViewSet(APIv3Settings, APIAuthMixin, queryset = Project.objects.all() -class VersionsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, +class VersionsViewSet(APIv3Settings, ProjectQuerySetMixin, NestedViewSetMixin, FlexFieldsMixin, UpdateModelMixin, ReadOnlyModelViewSet): model = Version @@ -262,7 +262,7 @@ def update(self, request, *args, **kwargs): return Response(status=204) -class BuildsViewSet(APIv3Settings, APIAuthMixin, NestedViewSetMixin, +class BuildsViewSet(APIv3Settings, ProjectQuerySetMixin, NestedViewSetMixin, FlexFieldsMixin, ReadOnlyModelViewSet): model = Build lookup_field = 'pk' From b4d794250f1c1bce79105a82def711cd4eca6fa4 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 6 May 2019 13:59:48 +0200 Subject: [PATCH 94/96] Rename "links" field to "_links" since it's an internal generated field --- readthedocs/api/v3/serializers.py | 12 ++++++------ .../v3/tests/responses/projects-builds-detail.json | 2 +- .../api/v3/tests/responses/projects-detail.json | 6 +++--- .../api/v3/tests/responses/projects-list.json | 2 +- .../tests/responses/projects-subprojects-list.json | 4 ++-- .../v3/tests/responses/projects-superproject.json | 2 +- .../projects-versions-builds-list_POST.json | 6 +++--- .../v3/tests/responses/projects-versions-detail.json | 2 +- 8 files changed, 18 insertions(+), 18 deletions(-) diff --git a/readthedocs/api/v3/serializers.py b/readthedocs/api/v3/serializers.py index 2ddb1ad343c..d0efcf066c1 100644 --- a/readthedocs/api/v3/serializers.py +++ b/readthedocs/api/v3/serializers.py @@ -115,7 +115,7 @@ class BuildSerializer(FlexFieldsModelSerializer): success = serializers.SerializerMethodField() duration = serializers.IntegerField(source='length') state = BuildStateSerializer(source='*') - links = BuildLinksSerializer(source='*') + _links = BuildLinksSerializer(source='*') expandable_fields = dict( config=( @@ -139,7 +139,7 @@ class Meta: 'success', 'error', 'commit', - 'links', + '_links', ] def get_finished(self, obj): @@ -215,7 +215,7 @@ class VersionSerializer(FlexFieldsModelSerializer): ref = serializers.CharField() downloads = serializers.SerializerMethodField() urls = VersionURLsSerializer(source='*') - links = VersionLinksSerializer(source='*') + _links = VersionLinksSerializer(source='*') expandable_fields = dict( last_build=( @@ -240,7 +240,7 @@ class Meta: 'type', 'downloads', 'urls', - 'links', + '_links', ] def get_downloads(self, obj): @@ -390,7 +390,7 @@ class ProjectSerializer(FlexFieldsModelSerializer): description = serializers.SerializerMethodField() - links = ProjectLinksSerializer(source='*') + _links = ProjectLinksSerializer(source='*') # TODO: adapt these fields with the proper names in the db and then remove # them from here @@ -435,7 +435,7 @@ class Meta: # 'users', # 'active_versions', - 'links', + '_links', ] def get_description(self, obj): diff --git a/readthedocs/api/v3/tests/responses/projects-builds-detail.json b/readthedocs/api/v3/tests/responses/projects-builds-detail.json index 95b6f3ff261..9a4462e6986 100644 --- a/readthedocs/api/v3/tests/responses/projects-builds-detail.json +++ b/readthedocs/api/v3/tests/responses/projects-builds-detail.json @@ -5,7 +5,7 @@ "error": "", "finished": "2019-04-29T10:01:00Z", "id": 1, - "links": { + "_links": { "_self": "https://readthedocs.org/api/v3/projects/project/builds/1/", "project": "https://readthedocs.org/api/v3/projects/project/", "version": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/" diff --git a/readthedocs/api/v3/tests/responses/projects-detail.json b/readthedocs/api/v3/tests/responses/projects-detail.json index 93c511ddd00..b0c98fe33e8 100644 --- a/readthedocs/api/v3/tests/responses/projects-detail.json +++ b/readthedocs/api/v3/tests/responses/projects-detail.json @@ -16,7 +16,7 @@ "error": "", "finished": "2019-04-29T10:01:00Z", "id": 1, - "links": { + "_links": { "_self": "https://readthedocs.org/api/v3/projects/project/builds/1/", "project": "https://readthedocs.org/api/v3/projects/project/", "version": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/" @@ -29,7 +29,7 @@ "success": true, "version": "v1.0" }, - "links": { + "_links": { "_self": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/", "builds": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/builds/", "project": "https://readthedocs.org/api/v3/projects/project/" @@ -57,7 +57,7 @@ "code": "en", "name": "English" }, - "links": { + "_links": { "_self": "https://readthedocs.org/api/v3/projects/project/", "builds": "https://readthedocs.org/api/v3/projects/project/builds/", "subprojects": "https://readthedocs.org/api/v3/projects/project/subprojects/", diff --git a/readthedocs/api/v3/tests/responses/projects-list.json b/readthedocs/api/v3/tests/responses/projects-list.json index 3244f976144..bc08292f9f5 100644 --- a/readthedocs/api/v3/tests/responses/projects-list.json +++ b/readthedocs/api/v3/tests/responses/projects-list.json @@ -45,7 +45,7 @@ "username": "testuser" } ], - "links": { + "_links": { "_self": "https://readthedocs.org/api/v3/projects/project/", "versions": "https://readthedocs.org/api/v3/projects/project/versions/", "builds": "https://readthedocs.org/api/v3/projects/project/builds/", diff --git a/readthedocs/api/v3/tests/responses/projects-subprojects-list.json b/readthedocs/api/v3/tests/responses/projects-subprojects-list.json index 30638f84861..2c1cbc50a9b 100644 --- a/readthedocs/api/v3/tests/responses/projects-subprojects-list.json +++ b/readthedocs/api/v3/tests/responses/projects-subprojects-list.json @@ -64,7 +64,7 @@ "project", "test" ], - "links": { + "_links": { "_self": "https://readthedocs.org/api/v3/projects/project/", "versions": "https://readthedocs.org/api/v3/projects/project/versions/", "builds": "https://readthedocs.org/api/v3/projects/project/builds/", @@ -86,7 +86,7 @@ }, "tags": [], "users": [], - "links": { + "_links": { "_self": "https://readthedocs.org/api/v3/projects/subproject/", "versions": "https://readthedocs.org/api/v3/projects/subproject/versions/", "builds": "https://readthedocs.org/api/v3/projects/subproject/builds/", diff --git a/readthedocs/api/v3/tests/responses/projects-superproject.json b/readthedocs/api/v3/tests/responses/projects-superproject.json index 3ccf2f9e13d..abe0c5de862 100644 --- a/readthedocs/api/v3/tests/responses/projects-superproject.json +++ b/readthedocs/api/v3/tests/responses/projects-superproject.json @@ -8,7 +8,7 @@ "code": "en", "name": "English" }, - "links": { + "_links": { "_self": "https://readthedocs.org/api/v3/projects/project/", "builds": "https://readthedocs.org/api/v3/projects/project/builds/", "subprojects": "https://readthedocs.org/api/v3/projects/project/subprojects/", diff --git a/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json b/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json index 6362a3efd4e..c5c65075ae0 100644 --- a/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json +++ b/readthedocs/api/v3/tests/responses/projects-versions-builds-list_POST.json @@ -6,7 +6,7 @@ "error": "", "finished": null, "id": 2, - "links": { + "_links": { "_self": "https://readthedocs.org/api/v3/projects/project/builds/2/", "project": "https://readthedocs.org/api/v3/projects/project/", "version": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/" @@ -29,7 +29,7 @@ "code": "en", "name": "English" }, - "links": { + "_links": { "_self": "https://readthedocs.org/api/v3/projects/project/", "builds": "https://readthedocs.org/api/v3/projects/project/builds/", "subprojects": "https://readthedocs.org/api/v3/projects/project/subprojects/", @@ -77,7 +77,7 @@ "downloads": {}, "id": 3, "identifier": "a1b2c3", - "links": { + "_links": { "_self": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/", "builds": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/builds/", "project": "https://readthedocs.org/api/v3/projects/project/" diff --git a/readthedocs/api/v3/tests/responses/projects-versions-detail.json b/readthedocs/api/v3/tests/responses/projects-versions-detail.json index d519d492d3a..861d951f36e 100644 --- a/readthedocs/api/v3/tests/responses/projects-versions-detail.json +++ b/readthedocs/api/v3/tests/responses/projects-versions-detail.json @@ -4,7 +4,7 @@ "downloads": {}, "id": 3, "identifier": "a1b2c3", - "links": { + "_links": { "_self": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/", "builds": "https://readthedocs.org/api/v3/projects/project/versions/v1.0/builds/", "project": "https://readthedocs.org/api/v3/projects/project/" From 8ebfb1c80d2eb9444c13e92901dbfb41052aa922 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Mon, 6 May 2019 16:21:15 +0200 Subject: [PATCH 95/96] Lint fixes --- readthedocs/api/v3/mixins.py | 1 - readthedocs/api/v3/urls.py | 2 +- readthedocs/api/v3/views.py | 19 ++++++++----------- readthedocs/oauth/services/gitlab.py | 2 -- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/readthedocs/api/v3/mixins.py b/readthedocs/api/v3/mixins.py index 2f583d6217f..a1916fdd74d 100644 --- a/readthedocs/api/v3/mixins.py +++ b/readthedocs/api/v3/mixins.py @@ -1,4 +1,3 @@ -from django.contrib.auth.models import User from django.shortcuts import get_object_or_404 from rest_framework.exceptions import NotFound diff --git a/readthedocs/api/v3/urls.py b/readthedocs/api/v3/urls.py index 71b545a3201..f577b44abe2 100644 --- a/readthedocs/api/v3/urls.py +++ b/readthedocs/api/v3/urls.py @@ -1,7 +1,7 @@ from .routers import DefaultRouterWithNesting from .views import ( - BuildsViewSet, BuildsCreateViewSet, + BuildsViewSet, ProjectsViewSet, SubprojectRelationshipViewSet, TranslationRelationshipViewSet, diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index fcf041e9abd..8657807a0a6 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -163,9 +163,7 @@ def get_view_description(self, *args, **kwargs): # pylint: disable=arguments-di @action(detail=True, methods=['get']) def superproject(self, request, project_slug): - """ - Return the superproject of a ``Project``. - """ + """Return the superproject of a ``Project``.""" project = self.get_object() try: superproject = project.superprojects.first().parent @@ -180,15 +178,15 @@ class SubprojectRelationshipViewSet(APIv3Settings, ProjectQuerySetMixin, ListModelMixin, GenericViewSet): # Markdown docstring exposed at BrowsableAPIRenderer. - """ - List subprojects of a ``Project``. - """ + + """List subprojects of a ``Project``.""" # Private/Internal docstring + """ The main query is done via the ``NestedViewSetMixin`` using the ``parents_query_lookups`` defined when registering the urls. - """ + """ # noqa model = Project lookup_field = 'slug' @@ -202,15 +200,14 @@ class TranslationRelationshipViewSet(APIv3Settings, ProjectQuerySetMixin, ListModelMixin, GenericViewSet): # Markdown docstring exposed at BrowsableAPIRenderer. - """ - List translations of a ``Project``. - """ + + """List translations of a ``Project``.""" # Private/Internal docstring """ The main query is done via the ``NestedViewSetMixin`` using the ``parents_query_lookups`` defined when registering the urls. - """ + """ # noqa model = Project lookup_field = 'slug' diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 19e01ab7b1e..4473f788b4e 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -11,14 +11,12 @@ from readthedocs.builds.utils import get_gitlab_username_repo from readthedocs.integrations.models import Integration -from readthedocs.integrations.utils import get_secret from readthedocs.projects.models import Project from ..models import RemoteOrganization, RemoteRepository from .base import Service, SyncServiceError - try: from urlparse import urljoin, urlparse except ImportError: From e4ac3274eef0dee09dd10189cf867043fad4f25b Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 8 May 2019 17:23:58 +0200 Subject: [PATCH 96/96] Inherit order matters NestedViewSetMixin has to be on the left of ProjectQuerySetMixin to filter nested results properly. --- readthedocs/api/v3/tests/test_projects.py | 32 +++++++++++++++++++++++ readthedocs/api/v3/views.py | 17 +++++++----- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/readthedocs/api/v3/tests/test_projects.py b/readthedocs/api/v3/tests/test_projects.py index a6b00b0a604..d5a79bd8fcf 100644 --- a/readthedocs/api/v3/tests/test_projects.py +++ b/readthedocs/api/v3/tests/test_projects.py @@ -339,3 +339,35 @@ def test_projects_versions_builds_list_post(self): response_json, self._get_response_dict('projects-versions-builds-list_POST'), ) + + def test_projects_versions_detail_unique(self): + second_project = fixture.get( + Project, + name='second project', + slug='second-project', + related_projects=[], + main_language_project=None, + users=[self.me], + versions=[], + ) + second_version = fixture.get( + Version, + slug=self.version.slug, + verbose_name=self.version.verbose_name, + identifier='a1b2c3', + project=second_project, + active=True, + built=True, + type='tag', + ) + self.client.credentials(HTTP_AUTHORIZATION=f'Token {self.token.key}') + response = self.client.get( + reverse( + 'projects-versions-detail', + kwargs={ + 'parent_lookup_project__slug': self.project.slug, + 'version_slug': self.version.slug, + }), + + ) + self.assertEqual(response.status_code, 200) diff --git a/readthedocs/api/v3/views.py b/readthedocs/api/v3/views.py index 8657807a0a6..0e8b2b55a0e 100644 --- a/readthedocs/api/v3/views.py +++ b/readthedocs/api/v3/views.py @@ -61,7 +61,7 @@ class APIv3Settings: metadata_class = SimpleMetadata -class ProjectsViewSet(APIv3Settings, ProjectQuerySetMixin, NestedViewSetMixin, +class ProjectsViewSet(APIv3Settings, NestedViewSetMixin, ProjectQuerySetMixin, FlexFieldsMixin, ReadOnlyModelViewSet): # Markdown docstring is automatically rendered by BrowsableAPIRenderer. @@ -173,8 +173,8 @@ def superproject(self, request, project_slug): return Response(status=404) -class SubprojectRelationshipViewSet(APIv3Settings, ProjectQuerySetMixin, - NestedViewSetMixin, FlexFieldsMixin, +class SubprojectRelationshipViewSet(APIv3Settings, NestedViewSetMixin, + ProjectQuerySetMixin, FlexFieldsMixin, ListModelMixin, GenericViewSet): # Markdown docstring exposed at BrowsableAPIRenderer. @@ -195,8 +195,8 @@ class SubprojectRelationshipViewSet(APIv3Settings, ProjectQuerySetMixin, queryset = Project.objects.all() -class TranslationRelationshipViewSet(APIv3Settings, ProjectQuerySetMixin, - NestedViewSetMixin, FlexFieldsMixin, +class TranslationRelationshipViewSet(APIv3Settings, NestedViewSetMixin, + ProjectQuerySetMixin, FlexFieldsMixin, ListModelMixin, GenericViewSet): # Markdown docstring exposed at BrowsableAPIRenderer. @@ -216,7 +216,10 @@ class TranslationRelationshipViewSet(APIv3Settings, ProjectQuerySetMixin, queryset = Project.objects.all() -class VersionsViewSet(APIv3Settings, ProjectQuerySetMixin, NestedViewSetMixin, +# Inherit order is important here. ``NestedViewSetMixin`` has to be on the left +# of ``ProjectQuerySetMixin`` to make calling ``super().get_queryset()`` work +# properly and filter nested dependencies +class VersionsViewSet(APIv3Settings, NestedViewSetMixin, ProjectQuerySetMixin, FlexFieldsMixin, UpdateModelMixin, ReadOnlyModelViewSet): model = Version @@ -259,7 +262,7 @@ def update(self, request, *args, **kwargs): return Response(status=204) -class BuildsViewSet(APIv3Settings, ProjectQuerySetMixin, NestedViewSetMixin, +class BuildsViewSet(APIv3Settings, NestedViewSetMixin, ProjectQuerySetMixin, FlexFieldsMixin, ReadOnlyModelViewSet): model = Build lookup_field = 'pk'