Support Heroku preboot, speed up production deploys.

.circleci/config.yml

@@ -160,7 +160,18 @@ workflows:
   version: 2
   build_and_deploy:
     jobs:
-      - build
+      - build:
+          filters:
+            branches:
+              ignore:
+                # This is a bit counter-intuitive, but we've been extremely
+                # diligent about only pushing to production once something
+                # has been verified to pass CI on master. Because of this,
+                # it's wasted effort to run the exact same tests on production
+                # when we push to it, particularly since the merge on
+                # production is a fast-forward merge and therefore represents
+                # the exact same code that's already been tested on master.
+                - production
       - deploy:
           requires:
             - build

deploy.py

@@ -5,6 +5,7 @@
 import json
 import re
 import tempfile
+import contextlib
 from dataclasses import dataclass
 from typing import List, Optional, Dict
 from pathlib import Path
@@ -114,6 +115,14 @@ def run(self, *args: str):
         cmdline = self._get_cmdline(*args)
         subprocess.check_call(cmdline, cwd=self.cwd, shell=self.shell)
 
+    def is_preboot_enabled(self, *args: str) -> bool:
+        result = subprocess.check_output(
+            self._get_cmdline('features:info', 'preboot', '--json'),
+            cwd=self.cwd,
+            shell=self.shell
+        )
+        return json.loads(result)['enabled']
+
     def get_full_config(self) -> Dict[str, str]:
         result = subprocess.check_output(
             self._get_cmdline('config', '-j'),
@@ -201,6 +210,34 @@ def build(self, cache_from_docker_registry: bool):
 
         build_worker_container(self.worker_container_tag, dockerfile_web=self.container_tag)
 
+    @contextlib.contextmanager
+    def maintenance_mode_if_preboot_is_disabled(self):
+        '''
+        If Heroku preboot is disabled, wrap the enclosed code in Heroku's
+        maintenance mode. Otherwise, we'll assume this is a zero-downtime
+        deploy, e.g. that any migrations that do need to be run will be ones
+        that the old version of the code is still compatible with.
+
+        Note that if the enclosed code raises an exception, we do _not_
+        disable maintenance mode, since we're assuming that the site
+        is broken and maintainers will still need it to be in maintenance
+        mode in order to fix it.
+        '''
+
+        is_preboot_enabled = self.heroku.is_preboot_enabled()
+
+        if is_preboot_enabled:
+            print("Heroku preboot is enabled, proceeding with zero-downtime deploy.")
+        else:
+            print("Heroku preboot is disabled, turning on maintenance mode.")
+            self.heroku.run('maintenance:on')
+
+        yield
+
+        if not is_preboot_enabled:
+            print("Turning off maintenance mode.")
+            self.heroku.run('maintenance:off')
+
     def deploy(self) -> None:
         print("Pushing containers to Docker registry...")
         self.push_to_docker_registry()
@@ -219,19 +256,16 @@ def deploy(self) -> None:
             # if self.is_using_rollbar:
             #     self.run_in_container(['python', 'manage.py', 'rollbarsourcemaps'])
 
-        self.heroku.run('maintenance:on')
-
-        # We want migrations to run while we're in maintenance mode because
-        # our codebase doesn't make any guarantees about being able to run
-        # on database schemas from previous or future versions.
-        print("Running migrations...")
-        self.run_in_container(['python', 'manage.py', 'migrate'])
-        self.run_in_container(['python', 'manage.py', 'initgroups'])
-
-        print("Initiating Heroku release phase...")
-        self.heroku.run('container:release', self.process_type, self.worker_process_type)
+        with self.maintenance_mode_if_preboot_is_disabled():
+            # If Heroku preboot is disabled, then we want migrations to run while we're in
+            # maintenance mode because we're assuming our codebase doesn't make any guarantees
+            # about being able to run on database schemas from previous or future versions.
+            print("Running migrations...")
+            self.run_in_container(['python', 'manage.py', 'migrate'])
+            self.run_in_container(['python', 'manage.py', 'initgroups'])
 
-        self.heroku.run('maintenance:off')
+            print("Initiating Heroku release phase...")
+            self.heroku.run('container:release', self.process_type, self.worker_process_type)
 
         print("Deploy finished.")
 

project/tests/test_deploy.py

@@ -7,15 +7,23 @@
 from project.util.testing_util import Snapshot
 import deploy
 
+
+def binary_encode_json(x):
+    return json.dumps(x).encode('utf-8')
+
+
 MY_DIR = Path(__file__).parent.resolve()
 
 SNAPSHOT_DIR = MY_DIR / "test_deploy_snapshots"
 
 DEFAULT_SUBPROCESS_CMD_PREFIX_OUTPUTS = {
     "git remote get-url": b"https://git.heroku.com/boop.git",
-    "heroku config": json.dumps({
+    "heroku config -j": binary_encode_json({
         'DATABASE_URL': 'postgres://boop'
-    }).encode('utf-8'),
+    }),
+    "heroku features:info preboot --json": binary_encode_json({
+        "enabled": False
+    }),
     "git rev-parse HEAD": b"e7408710b8d091377041cfbe4c185931a214f280",
     "git status -uno --porcelain": b"M somefile.py",
     "heroku auth:token": b"00112233-aabb-ccdd-eeff-001122334455",
@@ -33,7 +41,16 @@ def fake_temporary_directory():
     monkeypatch.setattr(tempfile, 'TemporaryDirectory', fake_temporary_directory)
 
 
-def create_check_output(cmd_prefix_outputs=DEFAULT_SUBPROCESS_CMD_PREFIX_OUTPUTS):
+def create_cmd_prefix_outputs(overrides=None):
+    return {
+        **DEFAULT_SUBPROCESS_CMD_PREFIX_OUTPUTS,
+        **(overrides or {}),
+    }
+
+
+def create_check_output(cmd_prefix_outputs=None):
+    cmd_prefix_outputs = create_cmd_prefix_outputs(cmd_prefix_outputs)
+
     def check_output(args, **kwargs):
         cmd = ' '.join(args)
         for cmd_prefix, output in cmd_prefix_outputs.items():
@@ -105,3 +122,18 @@ def test_heroku_works(subprocess, capsys):
 
     snapshot = Snapshot(capsys.readouterr().out, SNAPSHOT_DIR / "heroku_works.txt")
     assert snapshot.expected == snapshot.actual
+
+
+def test_heroku_with_preboot(subprocess, capsys):
+    subprocess.check_output.side_effect = create_check_output({
+        "heroku features:info preboot --json": json.dumps({
+            "enabled": True
+        }).encode('utf-8'),
+    })
+    subprocess.call.side_effect = successful_check_call_with_print
+    subprocess.check_call.side_effect = successful_check_call_with_print
+
+    deploy.main(['heroku', '-r', 'myapp'])
+
+    snapshot = Snapshot(capsys.readouterr().out, SNAPSHOT_DIR / "heroku_with_preboot.txt")
+    assert snapshot.expected == snapshot.actual

project/tests/test_deploy_snapshots/heroku_with_preboot.txt

@@ -0,0 +1,13 @@
+Running "docker build -f Dockerfile.web -t registry.heroku.com/boop/web --build-arg GIT_REVISION=e7408710b8d091377041cfbe4c185931a214f280 --build-arg IS_GIT_REPO_PRISTINE=False .".
+Running "docker build -f Dockerfile.worker -t registry.heroku.com/boop/worker --build-arg DOCKERFILE_WEB=registry.heroku.com/boop/web /tmp/somedir".
+Pushing containers to Docker registry...
+Running "docker login --username=_ --password=00112233-aabb-ccdd-eeff-001122334455 registry.heroku.com".
+Running "docker push registry.heroku.com/boop/web".
+Running "docker push registry.heroku.com/boop/worker".
+Heroku preboot is enabled, proceeding with zero-downtime deploy.
+Running migrations...
+Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py migrate".
+Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py initgroups".
+Initiating Heroku release phase...
+Running "heroku container:release web worker -r myapp".
+Deploy finished.

project/tests/test_deploy_snapshots/heroku_works.txt

@@ -4,11 +4,13 @@ Pushing containers to Docker registry...
 Running "docker login --username=_ --password=00112233-aabb-ccdd-eeff-001122334455 registry.heroku.com".
 Running "docker push registry.heroku.com/boop/web".
 Running "docker push registry.heroku.com/boop/worker".
+Heroku preboot is disabled, turning on maintenance mode.
 Running "heroku maintenance:on -r myapp".
 Running migrations...
 Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py migrate".
 Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py initgroups".
 Initiating Heroku release phase...
 Running "heroku container:release web worker -r myapp".
+Turning off maintenance mode.
 Running "heroku maintenance:off -r myapp".
 Deploy finished.