Merge branch 'master' into patch-2

.github/FUNDING.yml

@@ -0,0 +1 @@
+custom: https://readthedocs.org/sustainability/

CHANGELOG.rst

@@ -1,3 +1,77 @@
+Version 3.5.0
+-------------
+
+:Date: May 30, 2019
+
+* `@pyup-bot <http://github.com/pyup-bot>`__: pyup:  Scheduled weekly dependency update for week 21 (`#5737 <https://github.com/rtfd/readthedocs.org/pull/5737>`__)
+* `@humitos <http://github.com/humitos>`__: Update feature flags exposed to user in docs (`#5734 <https://github.com/rtfd/readthedocs.org/pull/5734>`__)
+* `@davidfischer <http://github.com/davidfischer>`__: Fix the sidebar ad color (`#5731 <https://github.com/rtfd/readthedocs.org/pull/5731>`__)
+* `@davidfischer <http://github.com/davidfischer>`__: Create a funding file (`#5729 <https://github.com/rtfd/readthedocs.org/pull/5729>`__)
+* `@davidfischer <http://github.com/davidfischer>`__: Small commercial hosting page rework (`#5728 <https://github.com/rtfd/readthedocs.org/pull/5728>`__)
+* `@mattparrilla <http://github.com/mattparrilla>`__: Add note about lack of support for private repos (`#5726 <https://github.com/rtfd/readthedocs.org/pull/5726>`__)
+* `@humitos <http://github.com/humitos>`__: Canonical consistency example (`#5722 <https://github.com/rtfd/readthedocs.org/pull/5722>`__)
+* `@humitos <http://github.com/humitos>`__: Use nonstopmode for latexmk (`#5714 <https://github.com/rtfd/readthedocs.org/pull/5714>`__)
+* `@cclauss <http://github.com/cclauss>`__: Identity is not the same thing as equality in Python (`#5713 <https://github.com/rtfd/readthedocs.org/pull/5713>`__)
+* `@pyup-bot <http://github.com/pyup-bot>`__: pyup:  Scheduled weekly dependency update for week 20 (`#5712 <https://github.com/rtfd/readthedocs.org/pull/5712>`__)
+* `@humitos <http://github.com/humitos>`__: Move version "Clean" button to details page (`#5706 <https://github.com/rtfd/readthedocs.org/pull/5706>`__)
+* `@ericholscher <http://github.com/ericholscher>`__: Explicitly mention a support email (`#5703 <https://github.com/rtfd/readthedocs.org/pull/5703>`__)
+* `@davidfischer <http://github.com/davidfischer>`__: Storage updates (`#5698 <https://github.com/rtfd/readthedocs.org/pull/5698>`__)
+* `@humitos <http://github.com/humitos>`__: Enable auth validate passwords (`#5696 <https://github.com/rtfd/readthedocs.org/pull/5696>`__)
+* `@stsewd <http://github.com/stsewd>`__: Simplify lock acquire (`#5695 <https://github.com/rtfd/readthedocs.org/pull/5695>`__)
+* `@stsewd <http://github.com/stsewd>`__: Simplify update docs task (`#5694 <https://github.com/rtfd/readthedocs.org/pull/5694>`__)
+* `@pyup-bot <http://github.com/pyup-bot>`__: pyup:  Scheduled weekly dependency update for week 19 (`#5692 <https://github.com/rtfd/readthedocs.org/pull/5692>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: Warning about using sqlite 3.26.0 for development (`#5681 <https://github.com/rtfd/readthedocs.org/pull/5681>`__)
+* `@davidfischer <http://github.com/davidfischer>`__: Configure the security middleware (`#5679 <https://github.com/rtfd/readthedocs.org/pull/5679>`__)
+* `@stsewd <http://github.com/stsewd>`__: Fix bug in notifications (`#5678 <https://github.com/rtfd/readthedocs.org/pull/5678>`__)
+* `@pyup-bot <http://github.com/pyup-bot>`__: pyup:  Scheduled weekly dependency update for week 18 (`#5667 <https://github.com/rtfd/readthedocs.org/pull/5667>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: pylint fix for notifications, restapi and config (`#5664 <https://github.com/rtfd/readthedocs.org/pull/5664>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: pylint fix for readthedocs.search (`#5663 <https://github.com/rtfd/readthedocs.org/pull/5663>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: pylint fix for readthedocs.projects (`#5662 <https://github.com/rtfd/readthedocs.org/pull/5662>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: pylint fix for readthedocs.doc_builder (`#5660 <https://github.com/rtfd/readthedocs.org/pull/5660>`__)
+* `@humitos <http://github.com/humitos>`__: Support Docker 5.0 image (`#5657 <https://github.com/rtfd/readthedocs.org/pull/5657>`__)
+* `@humitos <http://github.com/humitos>`__: Use latexmk if Sphinx > 1.6 (`#5656 <https://github.com/rtfd/readthedocs.org/pull/5656>`__)
+* `@humitos <http://github.com/humitos>`__: Upgrade docker python package to latest release (`#5654 <https://github.com/rtfd/readthedocs.org/pull/5654>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: pylint fix for readthedocs.core (`#5650 <https://github.com/rtfd/readthedocs.org/pull/5650>`__)
+* `@pyup-bot <http://github.com/pyup-bot>`__: pyup:  Scheduled weekly dependency update for week 17 (`#5645 <https://github.com/rtfd/readthedocs.org/pull/5645>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: Serve custom 404 pages from Django (`#5644 <https://github.com/rtfd/readthedocs.org/pull/5644>`__)
+* `@yarons <http://github.com/yarons>`__: Typo fix (`#5642 <https://github.com/rtfd/readthedocs.org/pull/5642>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: Sitemap hreflang syntax invalid for regional language variants fix (`#5638 <https://github.com/rtfd/readthedocs.org/pull/5638>`__)
+* `@davidfischer <http://github.com/davidfischer>`__: Optimizations and UX improvements to the dashboard screen (`#5637 <https://github.com/rtfd/readthedocs.org/pull/5637>`__)
+* `@davidfischer <http://github.com/davidfischer>`__: Redirect project slugs with underscores (`#5634 <https://github.com/rtfd/readthedocs.org/pull/5634>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: Standardizing the use of settings directly (`#5632 <https://github.com/rtfd/readthedocs.org/pull/5632>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: Note for Docker image size in Docker instructions (`#5630 <https://github.com/rtfd/readthedocs.org/pull/5630>`__)
+* `@davidfischer <http://github.com/davidfischer>`__: UX improvements around SSL certificates (`#5629 <https://github.com/rtfd/readthedocs.org/pull/5629>`__)
+* `@davidfischer <http://github.com/davidfischer>`__: Gold project sponsorship changes (`#5628 <https://github.com/rtfd/readthedocs.org/pull/5628>`__)
+* `@davidfischer <http://github.com/davidfischer>`__: Make sure there's a contact when opting out of advertising (`#5626 <https://github.com/rtfd/readthedocs.org/pull/5626>`__)
+* `@stsewd <http://github.com/stsewd>`__: Remove unused volume from docker (`#5625 <https://github.com/rtfd/readthedocs.org/pull/5625>`__)
+* `@dojutsu-user <http://github.com/dojutsu-user>`__: hotfix: correct way of getting environment variables (`#5622 <https://github.com/rtfd/readthedocs.org/pull/5622>`__)
+* `@pyup-bot <http://github.com/pyup-bot>`__: pyup:  Scheduled weekly dependency update for week 16 (`#5619 <https://github.com/rtfd/readthedocs.org/pull/5619>`__)
+* `@ericholscher <http://github.com/ericholscher>`__: Release 3.4.2 (`#5613 <https://github.com/rtfd/readthedocs.org/pull/5613>`__)
+* `@ericholscher <http://github.com/ericholscher>`__: Add explicit egg versoin to unicode-slugify (`#5612 <https://github.com/rtfd/readthedocs.org/pull/5612>`__)
+* `@dojutsu-user <http://github.com/dojutsu-user>`__: Remove ProxyMiddleware (`#5607 <https://github.com/rtfd/readthedocs.org/pull/5607>`__)
+* `@dojutsu-user <http://github.com/dojutsu-user>`__: Remove 'Versions' tab from Admin Dashboard. (`#5600 <https://github.com/rtfd/readthedocs.org/pull/5600>`__)
+* `@dojutsu-user <http://github.com/dojutsu-user>`__: Notify the user when deleting a superproject (`#5596 <https://github.com/rtfd/readthedocs.org/pull/5596>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: Handle 401, 403 and 404  when setting up webhooks (`#5589 <https://github.com/rtfd/readthedocs.org/pull/5589>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: Unify usage of settings and remove the usage of getattr for settings (`#5588 <https://github.com/rtfd/readthedocs.org/pull/5588>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: Note about admin page in the docs (`#5585 <https://github.com/rtfd/readthedocs.org/pull/5585>`__)
+* `@humitos <http://github.com/humitos>`__: Remove USE_SETUPTOOLS_LATEST feature flag (`#5578 <https://github.com/rtfd/readthedocs.org/pull/5578>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: Validate docs dir before writing custom js (`#5569 <https://github.com/rtfd/readthedocs.org/pull/5569>`__)
+* `@rshrc <http://github.com/rshrc>`__: Added note in YAML docs (`#5565 <https://github.com/rtfd/readthedocs.org/pull/5565>`__)
+* `@shivanshu1234 <http://github.com/shivanshu1234>`__: Specify python3 in installation instructions. (`#5552 <https://github.com/rtfd/readthedocs.org/pull/5552>`__)
+* `@davidfischer <http://github.com/davidfischer>`__: Write build artifacts to (cloud) storage from build servers (`#5549 <https://github.com/rtfd/readthedocs.org/pull/5549>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: "Default branch: latest" does not exist Fix. (`#5547 <https://github.com/rtfd/readthedocs.org/pull/5547>`__)
+* `@dojutsu-user <http://github.com/dojutsu-user>`__: Update ``readthedocs-environment.json`` file when env vars are added/deleted (`#5540 <https://github.com/rtfd/readthedocs.org/pull/5540>`__)
+* `@humitos <http://github.com/humitos>`__: Update common to its latest version (`#5517 <https://github.com/rtfd/readthedocs.org/pull/5517>`__)
+* `@saadmk11 <http://github.com/saadmk11>`__: Profile page performance issue Fix (`#5472 <https://github.com/rtfd/readthedocs.org/pull/5472>`__)
+* `@stsewd <http://github.com/stsewd>`__: Remove unused form (`#5443 <https://github.com/rtfd/readthedocs.org/pull/5443>`__)
+* `@stsewd <http://github.com/stsewd>`__: Use relative paths in config module (`#5377 <https://github.com/rtfd/readthedocs.org/pull/5377>`__)
+* `@humitos <http://github.com/humitos>`__: Initial structure for APIv3 (`#5356 <https://github.com/rtfd/readthedocs.org/pull/5356>`__)
+* `@stsewd <http://github.com/stsewd>`__: Add models for automation rules (`#5323 <https://github.com/rtfd/readthedocs.org/pull/5323>`__)
+* `@stsewd <http://github.com/stsewd>`__: Downgrade pytest-django (`#5294 <https://github.com/rtfd/readthedocs.org/pull/5294>`__)
+* `@ericholscher <http://github.com/ericholscher>`__: Add search for DomainData objects (`#5290 <https://github.com/rtfd/readthedocs.org/pull/5290>`__)
+* `@gorshunovr <http://github.com/gorshunovr>`__: Change version references to :latest tag (`#5245 <https://github.com/rtfd/readthedocs.org/pull/5245>`__)
+* `@dojutsu-user <http://github.com/dojutsu-user>`__: Fix buttons problems in 'Change Email' section. (`#5219 <https://github.com/rtfd/readthedocs.org/pull/5219>`__)
+
 Version 3.4.2
 -------------
 

docs/canonical.rst

@@ -36,7 +36,7 @@ you should see a bit of HTML like this:
 
 .. code-block:: html
 
-    <link rel="canonical" href="http://pip.readthedocs.io/en/latest/installing.html">
+    <link rel="canonical" href="http://docs.fabfile.org/en/2.4/" />
 
 Links
 -----

docs/commercial/index.rst

@@ -1,19 +1,35 @@
 Read the Docs Commercial Features
 ---------------------------------
 
-.. note:: These features are for our new commercial offering, `readthedocs.com`_.
+Read the Docs has a community solution for open source projects on `readthedocs.org`_
+and we offer commercial documentation building and hosting on `readthedocs.com`_.
+Features in this section are specific to our commercial offering.
 
+Private repositories and private documentation
+    The largest difference between the community solution and our commercial offering
+    is the ability to connect to private repositories,
+    to restrict documentation access to certain users,
+    or to share private documentation via private hyperlinks.
 
-All of the other features outlined in these docs work on both sites.
-Things inside this section are specific to our commercial offering.
+Additional build resources
+    Do you have a complicated build process that uses large amounts
+    of CPU, memory, disk, or networking resources?
+    Our commercial offering has much higher default resources
+    that result in faster documentation build times
+    and we can increase it further for very demanding projects.
 
-The largest feature that is different is that documentation on `readthedocs.com`_ is **private**.
-If you have private code that you want documentation for,
-this is our solution.
+Priority support
+    We have a dedicated support team that responds to support requests during business hours.
+    If you need a quick turnaround, please signup for readthedocs.com.
 
+Advertising-free
+    All commercially hosted documentation is always ad-free.
+
+.. _readthedocs.org: https://readthedocs.org
 .. _readthedocs.com: https://readthedocs.com
 
 .. toctree:: 
+   :caption: Additional commercial features
 
    custom_domains
    organizations

docs/guides/feature-flags.rst

@@ -12,22 +12,14 @@ or disable one or more of these featured flags for a particular project.
 Available Flags
 ---------------
 
-``USE_PDF_LATEXMK``: :featureflags:`USE_PDF_LATEXMK`
-
-``USE_SPHINX_LATEST``: :featureflags:`USE_SPHINX_LATEST`
-
-``ALLOW_DEPRECATED_WEBHOOKS``: :featureflags:`ALLOW_DEPRECATED_WEBHOOKS`
-
 ``PIP_ALWAYS_UPGRADE``: :featureflags:`PIP_ALWAYS_UPGRADE`
 
-``SKIP_SUBMODULES``: :featureflags:`SKIP_SUBMODULES`
-
 ``DONT_OVERWRITE_SPHINX_CONTEXT``: :featureflags:`DONT_OVERWRITE_SPHINX_CONTEXT`
 
-``MKDOCS_THEME_RTD``: :featureflags:`MKDOCS_THEME_RTD`
-
 ``DONT_SHALLOW_CLONE``: :featureflags:`DONT_SHALLOW_CLONE`
 
 The ``DONT_SHALLOW_CLONE`` flag is useful if your code accesses old commits during docs build,
 e.g. python-reno release notes manager is known to do that
 (error message line would probably include one of old Git commit id's).
+
+``USE_TESTING_BUILD_IMAGE``: :featureflags:`USE_TESTING_BUILD_IMAGE`

docs/guides/pdf-non-ascii-languages.rst

@@ -1,13 +1,6 @@
 Build PDF format for non-ASCII languages
 ========================================
 
-
-.. warning::
-
-   To be able to follow this guide and build PDF with this method,
-   you need to ask the Read the Docs core team to enable ``USE_PDF_LATEXMK`` :doc:`feature flag </guides/feature-flags>` in your project.
-   Please, `open an issue`_ in our repository asking for this, and wait for one of the core team to enable it.
-
 .. _open an issue: https://github.com/rtfd/readthedocs.org/issues/new
 
 Sphinx offers different `LaTeX engines`_ that support Unicode characters and non-ASCII languages,

docs/index.rst

@@ -122,6 +122,7 @@ to help you create fantastic documentation for your project.
    architecture
    tests
    docs
+   design/index
    development/standards
    development/buildenvironments
    symlinks

docs/install.rst

@@ -97,7 +97,7 @@ Clone the repository somewhere on your disk and enter to the repository::
 
 Create a virtual environment and activate it::
 
-    virtualenv venv
+    virtualenv --python=python3 venv
     source venv/bin/activate
 
 Next, install the dependencies using ``pip``

docs/intro/import-guide.rst

@@ -5,7 +5,9 @@ Importing Your Documentation
    :description lang=en: Import your existing technical documentation from version control into Read the Docs.
 
 
-To import a documentation repository, visit your `Read the Docs dashboard`_ and click Import_.
+To import a public documentation repository, visit your `Read the Docs dashboard`_ and click Import_. For private
+repositories, use the Read the Docs  :doc:`commercial solution <../commercial/index>`.
+
 
 If you have :doc:`connected your Read the Docs account <../connected-accounts>` to GitHub, Bitbucket, or GitLab,
 you will see a list of your repositories that we are able to import.

docs/support.rst

@@ -50,7 +50,7 @@ This includes:
 Commercial Support
 ------------------
 
-We offer commercial support for Read the Docs, commercial hosting,
+We offer commercial support for Read the Docs, :doc:`commercial hosting </commercial/index>`,
 as well as consulting around all documentation systems.
 You can contact us at hello@readthedocs.com to learn more,
 or read more at https://readthedocs.com/services/#open-source-support.

docs/team.rst

@@ -37,7 +37,7 @@ Members
 * `Manuel Kaufmann <https://github.com/humitos>`__ (Central Time)
 * Your Name Here
 
-Feel free to ask any of us if you have questions or want to join!
+**Please don't email us personally for support on Read the Docs.** You can email support@readthedocs.org for any issues you may have. 
 
 Joining
 ~~~~~~~

media/css/readthedocs-doc-embed.css

@@ -250,7 +250,7 @@ div.ethical-footer {
 
 /* Dark theme */
 .ethical-dark-theme .ethical-sidebar {
-    background-color: rgba(255, 255, 255, 0.1);
+    background-color: #4e4b4b;
     border: 1px solid #a0a0a0;
     color: #c2c2c2 !important;
 }

readthedocs/api/v2/utils.py

@@ -1,5 +1,6 @@
 """Utility functions that are used by both views and celery tasks."""
 
+import itertools
 import logging
 
 from rest_framework.pagination import PageNumberPagination
@@ -158,6 +159,23 @@ def delete_versions(project, version_data):
     return set()
 
 
+def run_automation_rules(project, versions_slug):
+    """
+    Runs the automation rules on each version.
+
+    The rules are sorted by priority.
+
+    .. note::
+
+       Currently the versions aren't sorted in any way,
+       the same order is keeped.
+    """
+    versions = project.versions.filter(slug__in=versions_slug)
+    rules = project.automation_rules.all()
+    for version, rule in itertools.product(versions, rules):
+        rule.run(version)
+
+
 class RemoteOrganizationPagination(PageNumberPagination):
     page_size = 25
 

readthedocs/api/v2/views/model_views.py

@@ -213,6 +213,20 @@ def sync_versions(self, request, **kwargs):  # noqa: D205
                 status=status.HTTP_400_BAD_REQUEST,
             )
 
+        try:
+            # The order of added_versions isn't deterministic.
+            # We don't track the commit time or any other metadata.
+            # We usually have one version added per webhook.
+            api_utils.run_automation_rules(project, added_versions)
+        except Exception:
+            # Don't interrupt the request if something goes wrong
+            # in the automation rules.
+            log.exception(
+                'Failed to execute automation rules for [%s]: %s',
+                project.slug, added_versions
+            )
+
+        # TODO: move this to an automation rule
         promoted_version = project.update_stable_version()
         new_stable = project.get_stable_version()
         if promoted_version and new_stable and new_stable.active:

readthedocs/api/v3/tests/test_projects.py

@@ -7,6 +7,7 @@
 from django.core.cache import cache
 from django.test import TestCase
 from django.urls import reverse
+from django.utils.timezone import make_aware
 from rest_framework.authtoken.models import Token
 from rest_framework.test import APIClient
 
@@ -19,8 +20,8 @@ class APIEndpointTests(TestCase):
     fixtures = []
 
     def setUp(self):
-        created = datetime.datetime(2019, 4, 29, 10, 0, 0)
-        modified = datetime.datetime(2019, 4, 29, 12, 0, 0)
+        created = make_aware(datetime.datetime(2019, 4, 29, 10, 0, 0))
+        modified = make_aware(datetime.datetime(2019, 4, 29, 12, 0, 0))
 
         self.me = fixture.get(
             User,