Build: implment build.jobs config file key

Allow people to use `build.jobs` to execute pre/post steps.

```yaml
build:
  jobs:
    pre_install:
      - echo `date`
      - python path/to/myscript.py
    pre_build:
      - sed -i **/*.rst -e "s|{version}|v3.5.1|g"
```

readthedocs/config/config.py

@@ -15,6 +15,7 @@
 from .find import find_one
 from .models import (
     Build,
+    BuildJobs,
     BuildTool,
     BuildWithTools,
     Conda,
@@ -751,6 +752,10 @@ def validate_conda(self):
             conda['environment'] = validate_path(environment, self.base_path)
         return conda
 
+    # NOTE: I think we should rename `BuildWithTools` to `BuildWithOs` since
+    # `os` is the main and mandatory key that makes the diference
+    #
+    # NOTE: `build.jobs` can't be used without using `build.os`
     def validate_build_config_with_tools(self):
         """
         Validates the build object (new format).
@@ -769,6 +774,22 @@ def validate_build_config_with_tools(self):
             for tool in tools.keys():
                 validate_choice(tool, self.settings['tools'].keys())
 
+        jobs = {}
+        with self.catch_validation_error("build.jobs"):
+            # FIXME: should we use `default={}` or kept the `None` here and
+            # shortcircuit the rest of the logic?
+            jobs = self.pop_config("build.jobs", default={})
+            validate_dict(jobs)
+            # NOTE: besides validating that each key is one of the expected
+            # ones, we could validate the value of each of them is a list of
+            # commands. However, I don't think we should validate the "command"
+            # looks like a real command.
+            for job in jobs.keys():
+                validate_choice(
+                    job,
+                    BuildJobs.__slots__,
+                )
+
         if not tools:
             self.error(
                 key='build.tools',
@@ -780,6 +801,16 @@ def validate_build_config_with_tools(self):
                 code=CONFIG_REQUIRED,
             )
 
+        build["jobs"] = {}
+        for job in BuildJobs.__slots__:
+            build["jobs"][job] = []
+
+        for job, commands in jobs.items():
+            with self.catch_validation_error(f"build.jobs.{job}"):
+                build["jobs"][job] = [
+                    validate_string(command) for command in validate_list(commands)
+                ]
+
         build['tools'] = {}
         for tool, version in tools.items():
             with self.catch_validation_error(f'build.tools.{tool}'):
@@ -1263,7 +1294,10 @@ def build(self):
             return BuildWithTools(
                 os=build['os'],
                 tools=tools,
-                apt_packages=build['apt_packages'],
+                jobs=BuildJobs(
+                    **{job: commands for job, commands in build["jobs"].items()}
+                ),
+                apt_packages=build["apt_packages"],
             )
         return Build(**build)
 

readthedocs/config/models.py

@@ -37,7 +37,7 @@ def __init__(self, **kwargs):
 
 class BuildWithTools(Base):
 
-    __slots__ = ('os', 'tools', 'apt_packages')
+    __slots__ = ("os", "tools", "jobs", "apt_packages")
 
     def __init__(self, **kwargs):
         kwargs.setdefault('apt_packages', [])
@@ -49,6 +49,22 @@ class BuildTool(Base):
     __slots__ = ('version', 'full_version')
 
 
+class BuildJobs(Base):
+
+    __slots__ = (
+        "pre_checkout",
+        "post_checkout",
+        "pre_system_dependencies",
+        "post_system_dependencies",
+        "pre_create_environment",
+        "post_create_environment",
+        "pre_install",
+        "post_install",
+        "pre_build",
+        "post_build",
+    )
+
+
 class Python(Base):
 
     __slots__ = ('version', 'install', 'use_system_site_packages')

readthedocs/doc_builder/director.py

@@ -1,4 +1,5 @@
 import os
+import shlex
 from collections import defaultdict
 
 import structlog
@@ -65,7 +66,7 @@ def setup_vcs(self):
                 ),
             )
 
-        environment = self.data.environment_class(
+        self.vcs_environment = self.data.environment_class(
             project=self.data.project,
             version=self.data.version,
             build=self.data.build,
@@ -74,17 +75,17 @@ def setup_vcs(self):
             # ca-certificate package which is compatible with Lets Encrypt
             container_image=settings.RTD_DOCKER_BUILD_SETTINGS["os"]["ubuntu-20.04"],
         )
-        with environment:
+        with self.vcs_environment:
             before_vcs.send(
                 sender=self.data.version,
-                environment=environment,
+                environment=self.vcs_environment,
             )
 
             # Create the VCS repository where all the commands are going to be
             # executed for a particular VCS type
             self.vcs_repository = self.data.project.vcs_repo(
                 version=self.data.version.slug,
-                environment=environment,
+                environment=self.vcs_environment,
                 verbose_name=self.data.version.verbose_name,
                 version_type=self.data.version.type,
             )
@@ -208,15 +209,17 @@ def checkout(self):
             self.vcs_repository.update_submodules(self.data.config)
 
     def post_checkout(self):
-        commands = []  # self.data.config.build.jobs.post_checkout
+        commands = self.data.config.build.jobs.post_checkout
         for command in commands:
-            self.build_environment.run(command)
+            # TODO: we could make a helper `self.run(environment, command)`
+            # that handles split and escape command
+            self.vcs_environment.run(*shlex.split(command), escape_command=False)
 
     # System dependencies (``build.apt_packages``)
     def pre_system_dependencies(self):
-        commands = []  # self.data.config.build.jobs.pre_system_dependencies
+        commands = self.data.config.build.jobs.pre_system_dependencies
         for command in commands:
-            self.build_environment.run(command)
+            self.build_environment.run(*shlex.split(command), escape_command=False)
 
     # NOTE: `system_dependencies` should not be possible to override by the
     # user because it's executed as ``RTD_DOCKER_USER`` (e.g. ``root``) user.
@@ -254,58 +257,60 @@ def system_dependencies(self):
             )
 
     def post_system_dependencies(self):
-        pass
+        commands = self.data.config.build.jobs.post_system_dependencies
+        for command in commands:
+            self.build_environment.run(*shlex.split(command), escape_command=False)
 
     # Language environment
     def pre_create_environment(self):
-        commands = []  # self.data.config.build.jobs.pre_create_environment
+        commands = self.data.config.build.jobs.pre_create_environment
         for command in commands:
-            self.build_environment.run(command)
+            self.build_environment.run(*shlex.split(command), escape_command=False)
 
     def create_environment(self):
         commands = []  # self.data.config.build.jobs.create_environment
         for command in commands:
-            self.build_environment.run(command)
+            self.build_environment.run(*shlex.split(command), escape_command=False)
 
         if not commands:
             self.language_environment.setup_base()
 
     def post_create_environment(self):
-        commands = []  # self.data.config.build.jobs.post_create_environment
+        commands = self.data.config.build.jobs.post_create_environment
         for command in commands:
-            self.build_environment.run(command)
+            self.build_environment.run(*shlex.split(command), escape_command=False)
 
     # Install
     def pre_install(self):
-        commands = []  # self.data.config.build.jobs.pre_install
+        commands = self.data.config.build.jobs.pre_install
         for command in commands:
-            self.build_environment.run(command)
+            self.build_environment.run(*shlex.split(command), escape_command=False)
 
     def install(self):
         commands = []  # self.data.config.build.jobs.install
         for command in commands:
-            self.build_environment.run(command)
+            self.build_environment.run(*shlex.split(command), escape_command=False)
 
         if not commands:
             self.language_environment.install_core_requirements()
             self.language_environment.install_requirements()
 
     def post_install(self):
-        commands = []  # self.data.config.build.jobs.post_install
+        commands = self.data.config.build.jobs.post_install
         for command in commands:
-            self.build_environment.run(command)
+            self.build_environment.run(*shlex.split(command), escape_command=False)
 
     # Build
     def pre_build(self):
-        commands = []  # self.data.config.build.jobs.pre_build
+        commands = self.data.config.build.jobs.pre_build
         for command in commands:
-            self.build_environment.run(command)
+            self.build_environment.run(*shlex.split(command), escape_command=False)
 
     def build_html(self):
         commands = []  # self.data.config.build.jobs.build.html
         if commands:
             for command in commands:
-                self.build_environment.run(command)
+                self.build_environment.run(*shlex.split(command), escape_command=False)
             return True
 
         return self.build_docs_class(self.data.config.doctype)
@@ -317,7 +322,7 @@ def build_pdf(self):
         commands = []  # self.data.config.build.jobs.build.pdf
         if commands:
             for command in commands:
-                self.build_environment.run(command)
+                self.build_environment.run(*shlex.split(command), escape_command=False)
             return True
 
         # Mkdocs has no pdf generation currently.
@@ -336,7 +341,7 @@ def build_htmlzip(self):
         commands = []  # self.data.config.build.jobs.build.htmlzip
         if commands:
             for command in commands:
-                self.build_environment.run(command)
+                self.build_environment.run(*shlex.split(command), escape_command=False)
             return True
 
         # We don't generate a zip for mkdocs currently.
@@ -351,7 +356,7 @@ def build_epub(self):
         commands = []  # self.data.config.build.jobs.build.epub
         if commands:
             for command in commands:
-                self.build_environment.run(command)
+                self.build_environment.run(*shlex.split(command), escape_command=False)
             return True
 
         # Mkdocs has no epub generation currently.
@@ -360,9 +365,9 @@ def build_epub(self):
         return False
 
     def post_build(self):
-        commands = []  # self.data.config.build.jobs.post_build
+        commands = self.data.config.build.jobs.post_build
         for command in commands:
-            self.build_environment.run(command)
+            self.build_environment.run(*shlex.split(command), escape_command=False)
 
     # Helpers
     #