From 8b7c2797f8ba152d277336146b708f246c1f1ad9 Mon Sep 17 00:00:00 2001 From: Manuel Kaufmann Date: Wed, 5 Sep 2018 17:34:49 +0200 Subject: [PATCH] Upgrade all packages using pur (#4318) * Upgrade all packages using pur for reqs in `ls requirements`; do pur --skip django-tastypie,django,docker,elasticsearch,pyelasticsearch,commonmark,stripe,djangorestframework,mkdocs,django-allauth,django-filter,mercurial --requirement requirements/$reqs; done * Run lint with Python 3.6 * Downgrade gitpython to 2.1.10 * Linting error fixed * Upgrade common Branch: https://github.com/rtfd/common/pull/17 * Remove old note for testing * Add note for django-filter * no-else-return fixed in all the files Some block needed to be re-idented to fulfill this check. Basically, if you have a "if something: return another" the "else" clause is not needed since it will be the common case for the flow to continue if the condition is not met. * Only # noqa comment * Note about gitpython --- readthedocs/builds/models.py | 20 +++++------ readthedocs/core/resolver.py | 4 ++- readthedocs/core/views/hooks.py | 17 +++++---- readthedocs/core/views/serve.py | 42 +++++++++++------------ readthedocs/doc_builder/base.py | 10 +++--- readthedocs/doc_builder/environments.py | 3 +- readthedocs/gold/forms.py | 4 +-- readthedocs/notifications/notification.py | 4 +-- readthedocs/oauth/services/bitbucket.py | 12 ++++--- readthedocs/oauth/services/github.py | 4 +-- readthedocs/oauth/services/gitlab.py | 3 +- readthedocs/projects/forms.py | 6 ++-- readthedocs/projects/models.py | 9 +++-- readthedocs/projects/validators.py | 6 ++-- readthedocs/projects/version_handling.py | 2 +- readthedocs/projects/views/public.py | 34 +++++++++--------- readthedocs/vcs_support/backends/git.py | 2 +- readthedocs/wsgi.py | 2 +- requirements/deploy.txt | 9 ++--- requirements/lint.txt | 8 ++--- requirements/pip.txt | 34 ++++++++++++------ requirements/testing.txt | 21 +++++++----- 22 files changed, 143 insertions(+), 113 deletions(-) diff --git a/readthedocs/builds/models.py b/readthedocs/builds/models.py index 969db804b5a..7698cdfd693 100644 --- a/readthedocs/builds/models.py +++ b/readthedocs/builds/models.py @@ -286,11 +286,11 @@ def get_github_url( if not docroot: return '' - else: - if docroot[0] != '/': - docroot = '/{}'.format(docroot) - if docroot[-1] != '/': - docroot = '{}/'.format(docroot) + + if docroot[0] != '/': + docroot = '/{}'.format(docroot) + if docroot[-1] != '/': + docroot = '{}/'.format(docroot) if action == 'view': action_string = 'blob' @@ -320,11 +320,11 @@ def get_gitlab_url( if not docroot: return '' - else: - if docroot[0] != '/': - docroot = '/{}'.format(docroot) - if docroot[-1] != '/': - docroot = '{}/'.format(docroot) + + if docroot[0] != '/': + docroot = '/{}'.format(docroot) + if docroot[-1] != '/': + docroot = '{}/'.format(docroot) if action == 'view': action_string = 'blob' diff --git a/readthedocs/core/resolver.py b/readthedocs/core/resolver.py index 89a5470baa8..43fce68b3f6 100644 --- a/readthedocs/core/resolver.py +++ b/readthedocs/core/resolver.py @@ -132,8 +132,10 @@ def resolve_domain(self, project, private=None): domain = self._get_project_custom_domain(canonical_project) if domain: return domain.domain - elif self._use_subdomain(): + + if self._use_subdomain(): return self._get_project_subdomain(canonical_project) + return getattr(settings, 'PRODUCTION_DOMAIN') def resolve(self, project, require_https=False, filename='', private=None, diff --git a/readthedocs/core/views/hooks.py b/readthedocs/core/views/hooks.py index 0c2b5c50a8e..253d8848ef3 100644 --- a/readthedocs/core/views/hooks.py +++ b/readthedocs/core/views/hooks.py @@ -52,18 +52,20 @@ def _build_version(project, slug, already_built=()): log.info("(Version build) Building %s:%s", project.slug, slug_version.slug) return LATEST - elif project.versions.exclude(active=True).filter(slug=slug).exists(): + + if project.versions.exclude(active=True).filter(slug=slug).exists(): log.info("(Version build) Not Building %s", slug) return None - elif slug not in already_built: + + if slug not in already_built: version = project.versions.get(slug=slug) trigger_build(project=project, version=version, force=True) log.info("(Version build) Building %s:%s", project.slug, version.slug) return slug - else: - log.info("(Version build) Not Building %s", slug) - return None + + log.info("(Version build) Not Building %s", slug) + return None def build_branches(project, branch_list): @@ -305,16 +307,19 @@ def bitbucket_build(request): branches, ) log.debug('Bitbucket webhook payload:\n\n%s\n\n', data) + projects = get_project_from_url(search_url) if projects and branches: return _build_url(search_url, projects, branches) - elif not branches: + + if not branches: log.error( 'Commit/branch not found url=%s branches=%s', search_url, branches, ) return HttpResponseNotFound('Commit/branch not found') + log.info('Project match not found: url=%s', search_url) return HttpResponseNotFound('Project match not found') return HttpResponse('Method not allowed, POST is required', status=405) diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py index 245b0bba137..36510440ae3 100644 --- a/readthedocs/core/views/serve.py +++ b/readthedocs/core/views/serve.py @@ -125,23 +125,23 @@ def _serve_file(request, filename, basepath): if settings.DEBUG or getattr(settings, 'PYTHON_MEDIA', False): # Serve from Python return serve(request, filename, basepath) - else: - # Serve from Nginx - content_type, encoding = mimetypes.guess_type( - os.path.join(basepath, filename)) - content_type = content_type or 'application/octet-stream' - response = HttpResponse(content_type=content_type) - if encoding: - response['Content-Encoding'] = encoding - try: - response['X-Accel-Redirect'] = os.path.join( - basepath[len(settings.SITE_ROOT):], - filename, - ) - except UnicodeEncodeError: - raise Http404 - - return response + + # Serve from Nginx + content_type, encoding = mimetypes.guess_type( + os.path.join(basepath, filename)) + content_type = content_type or 'application/octet-stream' + response = HttpResponse(content_type=content_type) + if encoding: + response['Content-Encoding'] = encoding + try: + response['X-Accel-Redirect'] = os.path.join( + basepath[len(settings.SITE_ROOT):], + filename, + ) + except UnicodeEncodeError: + raise Http404 + + return response @map_project_slug @@ -200,8 +200,8 @@ def _serve_symlink_docs(request, project, privacy_level, filename=''): basepath = public_symlink.project_root if os.path.exists(os.path.join(basepath, filename)): return _serve_file(request, filename, basepath) - else: - files_tried.append(os.path.join(basepath, filename)) + + files_tried.append(os.path.join(basepath, filename)) if (settings.DEBUG or constants.PRIVATE in serve_docs) and privacy_level == constants.PRIVATE: # yapf: disable # noqa # Handle private @@ -210,8 +210,8 @@ def _serve_symlink_docs(request, project, privacy_level, filename=''): if os.path.exists(os.path.join(basepath, filename)): return _serve_file(request, filename, basepath) - else: - files_tried.append(os.path.join(basepath, filename)) + + files_tried.append(os.path.join(basepath, filename)) raise Http404( 'File not found. Tried these files: %s' % ','.join(files_tried)) diff --git a/readthedocs/doc_builder/base.py b/readthedocs/doc_builder/base.py index fcb70964709..fe94702385a 100644 --- a/readthedocs/doc_builder/base.py +++ b/readthedocs/doc_builder/base.py @@ -104,9 +104,9 @@ def create_index(self, extension='md', **__): docs_dir, 'README.{ext}'.format(ext=extension)) if os.path.exists(readme_filename): return 'README' - else: - index_file = open(index_filename, 'w+') - index_text = """ + + index_file = open(index_filename, 'w+') + index_text = """ Welcome to Read the Docs ------------------------ @@ -122,8 +122,8 @@ def create_index(self, extension='md', **__): familiar with Read the Docs. """ - index_file.write(index_text.format(dir=docs_dir, ext=extension)) - index_file.close() + index_file.write(index_text.format(dir=docs_dir, ext=extension)) + index_file.close() return 'index' def run(self, *args, **kwargs): diff --git a/readthedocs/doc_builder/environments.py b/readthedocs/doc_builder/environments.py index 282572b41b3..5815d6e19f9 100644 --- a/readthedocs/doc_builder/environments.py +++ b/readthedocs/doc_builder/environments.py @@ -872,7 +872,8 @@ def container_id(self): """Return id of container if it is valid.""" if self.container_name: return self.container_name - elif self.container: + + if self.container: return self.container.get('Id') def container_state(self): diff --git a/readthedocs/gold/forms.py b/readthedocs/gold/forms.py index 0e03009d605..9e84fea9c0f 100644 --- a/readthedocs/gold/forms.py +++ b/readthedocs/gold/forms.py @@ -92,5 +92,5 @@ def clean(self): cleaned_data = super(GoldProjectForm, self).clean() if self.projects.count() < self.user.num_supported_projects: return cleaned_data - else: - self.add_error(None, 'You already have the max number of supported projects.') + + self.add_error(None, 'You already have the max number of supported projects.') diff --git a/readthedocs/notifications/notification.py b/readthedocs/notifications/notification.py index cd9a96ed3dd..d2300e1f4f0 100644 --- a/readthedocs/notifications/notification.py +++ b/readthedocs/notifications/notification.py @@ -69,8 +69,8 @@ def get_template_names(self, backend_name, source_format=constants.HTML): source_format=source_format, )) return names - else: - raise AttributeError() + + raise AttributeError() def render(self, backend_name, source_format=constants.HTML): return render_to_string( diff --git a/readthedocs/oauth/services/bitbucket.py b/readthedocs/oauth/services/bitbucket.py index 0e9976669dd..37aa8d97e6d 100644 --- a/readthedocs/oauth/services/bitbucket.py +++ b/readthedocs/oauth/services/bitbucket.py @@ -111,8 +111,8 @@ def create_repository(self, fields, privacy=None, organization=None): log.debug('Not importing %s because mismatched orgs', fields['name']) return None - else: - repo.organization = organization + + repo.organization = organization repo.users.add(self.user) repo.name = fields['name'] repo.description = fields['description'] @@ -141,9 +141,11 @@ def create_repository(self, fields, privacy=None, organization=None): repo.json = json.dumps(fields) repo.save() return repo - else: - log.debug('Not importing %s because mismatched type', - fields['name']) + + log.debug( + 'Not importing %s because mismatched type', + fields['name'], + ) def create_organization(self, fields): """ diff --git a/readthedocs/oauth/services/github.py b/readthedocs/oauth/services/github.py index 58a569c04a5..7f4c9c75786 100644 --- a/readthedocs/oauth/services/github.py +++ b/readthedocs/oauth/services/github.py @@ -96,8 +96,8 @@ def create_repository(self, fields, privacy=None, organization=None): log.debug('Not importing %s because mismatched orgs', fields['name']) return None - else: - repo.organization = organization + + repo.organization = organization repo.name = fields['name'] repo.description = fields['description'] repo.ssh_url = fields['ssh_url'] diff --git a/readthedocs/oauth/services/gitlab.py b/readthedocs/oauth/services/gitlab.py index 240c6bbfd54..360a4a120ee 100644 --- a/readthedocs/oauth/services/gitlab.py +++ b/readthedocs/oauth/services/gitlab.py @@ -161,9 +161,8 @@ def create_repository(self, fields, privacy=None, organization=None): fields['name'], ) return None - else: - repo.organization = organization + repo.organization = organization repo.name = fields['name'] repo.description = fields['description'] repo.ssh_url = fields['ssh_url_to_repo'] diff --git a/readthedocs/projects/forms.py b/readthedocs/projects/forms.py index 6b3201458f4..1c5a2304f87 100644 --- a/readthedocs/projects/forms.py +++ b/readthedocs/projects/forms.py @@ -268,7 +268,7 @@ class ProjectRelationshipBaseForm(forms.ModelForm): class Meta(object): model = ProjectRelationship - exclude = [] + fields = '__all__' def __init__(self, *args, **kwargs): self.project = kwargs.pop('project') @@ -634,7 +634,7 @@ class DomainForm(forms.ModelForm): class Meta(object): model = Domain - exclude = ['machine', 'cname', 'count'] + exclude = ['machine', 'cname', 'count'] # pylint: disable=modelform-uses-exclude def __init__(self, *args, **kwargs): self.project = kwargs.pop('project', None) @@ -674,7 +674,7 @@ class IntegrationForm(forms.ModelForm): class Meta(object): model = Integration - exclude = ['provider_data', 'exchanges'] + exclude = ['provider_data', 'exchanges'] # pylint: disable=modelform-uses-exclude def __init__(self, *args, **kwargs): self.project = kwargs.pop('project', None) diff --git a/readthedocs/projects/models.py b/readthedocs/projects/models.py index f189f7898d1..9cfb0b92c45 100644 --- a/readthedocs/projects/models.py +++ b/readthedocs/projects/models.py @@ -535,7 +535,8 @@ def full_json_path(self, version=LATEST): """The path to the build json docs in the project.""" if 'sphinx' in self.documentation_type: return os.path.join(self.conf_dir(version), '_build', 'json') - elif 'mkdocs' in self.documentation_type: + + if 'mkdocs' in self.documentation_type: return os.path.join(self.checkout_path(version), '_build', 'json') def full_singlehtml_path(self, version=LATEST): @@ -555,11 +556,13 @@ def conf_file(self, version=LATEST): if self.conf_py_file: conf_path = os.path.join( self.checkout_path(version), self.conf_py_file,) + if os.path.exists(conf_path): log.info('Inserting conf.py file path from model') return conf_path - else: - log.warning("Conf file specified on model doesn't exist") + + log.warning("Conf file specified on model doesn't exist") + files = self.find('conf.py', version) if not files: files = self.full_find('conf.py', version) diff --git a/readthedocs/projects/validators.py b/readthedocs/projects/validators.py index 976d9e55e84..0b336450ba2 100644 --- a/readthedocs/projects/validators.py +++ b/readthedocs/projects/validators.py @@ -85,9 +85,9 @@ def __call__(self, value): elif self.re_git_user.search(value) or url.scheme in private_schemes: if allow_private_repos: return value - else: - # Throw a more helpful error message - raise ValidationError('Manual cloning via SSH is not supported') + + # Throw a more helpful error message + raise ValidationError('Manual cloning via SSH is not supported') # No more valid URLs without supported URL schemes raise ValidationError(_('Invalid scheme for URL')) diff --git a/readthedocs/projects/version_handling.py b/readthedocs/projects/version_handling.py index 897bfa1ad97..48fb6584643 100644 --- a/readthedocs/projects/version_handling.py +++ b/readthedocs/projects/version_handling.py @@ -74,7 +74,7 @@ def prune_point(self, num_latest): try: self._state[major][minor] = sorted( set(self._state[major][minor]))[-num_latest:] - except TypeError: + except TypeError: # pylint: disable=try-except-raise # Raise these for now. raise diff --git a/readthedocs/projects/views/public.py b/readthedocs/projects/views/public.py index f50b968902a..e5a3e2bf5ea 100644 --- a/readthedocs/projects/views/public.py +++ b/readthedocs/projects/views/public.py @@ -194,23 +194,23 @@ def project_download_media(request, project_slug, type_, version_slug): settings.MEDIA_URL, type_, project_slug, version_slug, '%s.%s' % (project_slug, type_.replace('htmlzip', 'zip'))) return HttpResponseRedirect(path) - else: - # Get relative media path - path = ( - version.project.get_production_media_path( - type_=type_, version_slug=version_slug) - .replace(settings.PRODUCTION_ROOT, '/prod_artifacts')) - content_type, encoding = mimetypes.guess_type(path) - content_type = content_type or 'application/octet-stream' - response = HttpResponse(content_type=content_type) - if encoding: - response['Content-Encoding'] = encoding - response['X-Accel-Redirect'] = path - # Include version in filename; this fixes a long-standing bug - filename = '%s-%s.%s' % ( - project_slug, version_slug, path.split('.')[-1]) - response['Content-Disposition'] = 'filename=%s' % filename - return response + + # Get relative media path + path = ( + version.project.get_production_media_path( + type_=type_, version_slug=version_slug) + .replace(settings.PRODUCTION_ROOT, '/prod_artifacts')) + content_type, encoding = mimetypes.guess_type(path) + content_type = content_type or 'application/octet-stream' + response = HttpResponse(content_type=content_type) + if encoding: + response['Content-Encoding'] = encoding + response['X-Accel-Redirect'] = path + # Include version in filename; this fixes a long-standing bug + filename = '%s-%s.%s' % ( + project_slug, version_slug, path.split('.')[-1]) + response['Content-Disposition'] = 'filename=%s' % filename + return response def elastic_project_search(request, project_slug): diff --git a/readthedocs/vcs_support/backends/git.py b/readthedocs/vcs_support/backends/git.py index 186fa439f1f..ca382294eba 100644 --- a/readthedocs/vcs_support/backends/git.py +++ b/readthedocs/vcs_support/backends/git.py @@ -171,7 +171,7 @@ def parse_branches(self, data): delimiter = str(' ').encode('utf-8') if PY2 else str(' ') raw_branches = csv.reader(StringIO(data), delimiter=delimiter) for branch in raw_branches: - branch = [f for f in branch if f != '' and f != '*'] + branch = [f for f in branch if f not in ('', '*')] # Handle empty branches if branch: branch = branch[0] diff --git a/readthedocs/wsgi.py b/readthedocs/wsgi.py index 7dccee9133a..e1cd6a8ec80 100644 --- a/readthedocs/wsgi.py +++ b/readthedocs/wsgi.py @@ -8,7 +8,7 @@ # This application object is used by any WSGI server configured to use this # file. This includes Django's development server, if the WSGI_APPLICATION # setting points here. -from django.core.wsgi import get_wsgi_application # pylint: disable=wrong-import-position +from django.core.wsgi import get_wsgi_application # noqa application = get_wsgi_application() # Apply WSGI middleware here. diff --git a/requirements/deploy.txt b/requirements/deploy.txt index 1a73a66da5a..261338c6f5e 100644 --- a/requirements/deploy.txt +++ b/requirements/deploy.txt @@ -1,8 +1,9 @@ -r pip.txt -psycopg2==2.7.3.2 -gunicorn==19.1.0 -pysolr==2.0.13 -django-redis-cache==1.6.3 +# http://initd.org/psycopg/docs/install.html#binary-install-from-pypi +psycopg2==2.7.5 --no-binary psycopg2 +gunicorn==19.9.0 +pysolr==3.7.0 +django-redis-cache==1.7.1 #For resizing images pillow diff --git a/requirements/lint.txt b/requirements/lint.txt index b0877c50185..16d825061de 100644 --- a/requirements/lint.txt +++ b/requirements/lint.txt @@ -1,7 +1,7 @@ -r pip.txt -astroid==1.6.4 -pylint==1.9.1 -pylint-django==0.11.1 +astroid==2.0.4 +pylint==2.1.1 +pylint-django==2.0.2 pylint-celery==0.3 -prospector==0.12.10 +prospector==1.1.2 pyflakes==2.0.0 diff --git a/requirements/pip.txt b/requirements/pip.txt index fe587d962ad..d4f5886fc56 100644 --- a/requirements/pip.txt +++ b/requirements/pip.txt @@ -1,10 +1,10 @@ # Base packages -pip==10.0.1 +pip==18.0 appdirs==1.4.3 virtualenv==16.0.0 docutils==0.14 -Sphinx==1.7.4 -sphinx_rtd_theme==0.3.1 +Sphinx==1.7.8 +sphinx_rtd_theme==0.4.1 Pygments==2.2.0 @@ -18,26 +18,30 @@ future==0.16.0 django-tastypie==0.13.0 django-guardian==1.4.9 -django-extensions==2.0.7 +django-extensions==2.1.2 # djangorestframework 3.7.x drops support for django 1.9.x djangorestframework==3.6.4 # Filtering for the REST API +# django-filter>2.0 require Django 1.11 +# https://django-filter.readthedocs.io/en/master/guide/migration.html#migrating-to-2-0 +# Besides the guide, check this comment for the migration +# https://github.com/rtfd/readthedocs.org/pull/4318#discussion_r214163531 django-filter==1.1.0 django-vanilla-views==1.0.5 jsonfield==2.0.2 -requests==2.18.4 +requests==2.19.1 requests-toolbelt==0.8.0 slumber==0.7.1 -lxml==4.2.1 +lxml==4.2.4 defusedxml==0.5.0 # Basic tools redis==2.10.6 -celery==4.1.1 +celery==4.2.1 # django-allauth 0.33.0 dropped support for Django 1.9 # https://django-allauth.readthedocs.io/en/latest/release-notes.html#backwards-incompatible-changes @@ -47,6 +51,14 @@ dnspython==1.15.0 # VCS httplib2==0.11.3 + +# GitPython 2.1.11 makes TestGitBackend.test_git_tags to fail because +# of an UnicodeError +# This commit, +# https://github.com/gitpython-developers/GitPython/commit/7f08b7730438bde34ae55bc3793fa524047bb804, +# introduces the usage of ``str`` which behaves differently in Py2 and +# Py3 We should check if all the tests pass again when we drop Py2 +# support and we should be able to upgrade GitPython==2.1.10 # Search @@ -56,8 +68,8 @@ pyquery==1.4.0 # Utils django-gravatar2==1.4.2 -pytz==2018.4 -beautifulsoup4==4.6.0 +pytz==2018.5 +beautifulsoup4==4.6.3 Unipath==1.1 django-kombu==0.9.4 mock==2.0.0 @@ -78,11 +90,11 @@ django-textclassifier==1.0 django-annoying==0.10.4 django-messages-extends==0.6.0 djangorestframework-jsonp==1.0.2 -django-taggit==0.22.2 +django-taggit==0.23.0 dj-pagination==2.3.2 # Docs -sphinxcontrib-httpdomain==1.6.1 +sphinxcontrib-httpdomain==1.7.0 # commonmark 0.5.5 is the latest version compatible with our docs, the # newer ones make `tox -e docs` to fail diff --git a/requirements/testing.txt b/requirements/testing.txt index d742a8f4c7f..d40c8a6f74b 100644 --- a/requirements/testing.txt +++ b/requirements/testing.txt @@ -1,16 +1,21 @@ -r pip.txt django-dynamic-fixture==2.0.0 - -# 3.6.1 and >3.2.5 is incompatible -# with pytest-describe 0.11.0 -pytest==3.2.5 -pytest-django==3.1.2 -pytest-describe==0.11.0 -pytest-xdist==1.22.0 -apipkg==1.4 +pytest==3.7.4 +pytest-django==3.4.2 +pytest-describe==0.11.1 +pytest-xdist==1.23.0 +apipkg==1.5 execnet==1.5.0 + +# Mercurial 4.3 and newer require Python 2.7 +# Mercurial is actively being ported to Python 3. As of Mercurial 4.3, +# some commands work on Python 3. However, Python 3 is not yet a +# supported platform. +# mercurial-scm.org/wiki/SupportedPythonVersions +# (Pinned to 4.4.2 since what we need for testing is still useful) Mercurial==4.4.2 + yamale==1.7.0 pytest-mock==1.10.0