Support Heroku preboot, speed up production deploys. (#1366)

This adds support for Heroku preboot.  If it's enabled, we don't enter maintenance mode at all, resulting in a zero-downtime deploy; otherwise, we behave as normal, entering maintenance mode before doing any migrations.

This basically means that we need to disable preboot if we ever make any migrations that will cause the old version of the code to crash.  Once we've successfully done a deploy with Preboot, I'll update our Deployment wiki page with new instructions.

**It also prevents tests from being run on the `production` branch.** This is a bit of a bold move but I've provided the rationale in a comment in our `circle.yml`.

```
                # This is a bit counter-intuitive, but we've been extremely
                # diligent about only pushing to production once something
                # has been verified to pass CI on master. Because of this,
                # it's wasted effort to run the exact same tests on production
                # when we push to it, particularly since the merge on
                # production is a fast-forward merge and therefore represents
                # the exact same code that's already been tested on master.
```

Taken together, both changes should make deploys to production much faster without introducing risk.

.circleci/config.yml

@@ -160,7 +160,18 @@ workflows:
   version: 2
   build_and_deploy:
     jobs:
-      - build
+      - build:
+          filters:
+            branches:
+              ignore:
+                # This is a bit counter-intuitive, but we've been extremely
+                # diligent about only pushing to production once something
+                # has been verified to pass CI on master. Because of this,
+                # it's wasted effort to run the exact same tests on production
+                # when we push to it, particularly since the merge on
+                # production is a fast-forward merge and therefore represents
+                # the exact same code that's already been tested on master.
+                - production
       - deploy:
           requires:
             - build

deploy.py

@@ -5,6 +5,7 @@
 import json
 import re
 import tempfile
+import contextlib
 from dataclasses import dataclass
 from typing import List, Optional, Dict
 from pathlib import Path
@@ -114,6 +115,14 @@ def run(self, *args: str):
         cmdline = self._get_cmdline(*args)
         subprocess.check_call(cmdline, cwd=self.cwd, shell=self.shell)
 
+    def is_preboot_enabled(self, *args: str) -> bool:
+        result = subprocess.check_output(
+            self._get_cmdline('features:info', 'preboot', '--json'),
+            cwd=self.cwd,
+            shell=self.shell
+        )
+        return json.loads(result)['enabled']
+
     def get_full_config(self) -> Dict[str, str]:
         result = subprocess.check_output(
             self._get_cmdline('config', '-j'),
@@ -201,6 +210,34 @@ def build(self, cache_from_docker_registry: bool):
 
         build_worker_container(self.worker_container_tag, dockerfile_web=self.container_tag)
 
+    @contextlib.contextmanager
+    def maintenance_mode_if_preboot_is_disabled(self):
+        '''
+        If Heroku preboot is disabled, wrap the enclosed code in Heroku's
+        maintenance mode. Otherwise, we'll assume this is a zero-downtime
+        deploy, e.g. that any migrations that do need to be run will be ones
+        that the old version of the code is still compatible with.
+
+        Note that if the enclosed code raises an exception, we do _not_
+        disable maintenance mode, since we're assuming that the site
+        is broken and maintainers will still need it to be in maintenance
+        mode in order to fix it.
+        '''
+
+        is_preboot_enabled = self.heroku.is_preboot_enabled()
+
+        if is_preboot_enabled:
+            print("Heroku preboot is enabled, proceeding with zero-downtime deploy.")
+        else:
+            print("Heroku preboot is disabled, turning on maintenance mode.")
+            self.heroku.run('maintenance:on')
+
+        yield
+
+        if not is_preboot_enabled:
+            print("Turning off maintenance mode.")
+            self.heroku.run('maintenance:off')
+
     def deploy(self) -> None:
         print("Pushing containers to Docker registry...")
         self.push_to_docker_registry()
@@ -219,19 +256,16 @@ def deploy(self) -> None:
             # if self.is_using_rollbar:
             #     self.run_in_container(['python', 'manage.py', 'rollbarsourcemaps'])
 
-        self.heroku.run('maintenance:on')
-
-        # We want migrations to run while we're in maintenance mode because
-        # our codebase doesn't make any guarantees about being able to run
-        # on database schemas from previous or future versions.
-        print("Running migrations...")
-        self.run_in_container(['python', 'manage.py', 'migrate'])
-        self.run_in_container(['python', 'manage.py', 'initgroups'])
-
-        print("Initiating Heroku release phase...")
-        self.heroku.run('container:release', self.process_type, self.worker_process_type)
+        with self.maintenance_mode_if_preboot_is_disabled():
+            # If Heroku preboot is disabled, then we want migrations to run while we're in
+            # maintenance mode because we're assuming our codebase doesn't make any guarantees
+            # about being able to run on database schemas from previous or future versions.
+            print("Running migrations...")
+            self.run_in_container(['python', 'manage.py', 'migrate'])
+            self.run_in_container(['python', 'manage.py', 'initgroups'])
 
-        self.heroku.run('maintenance:off')
+            print("Initiating Heroku release phase...")
+            self.heroku.run('container:release', self.process_type, self.worker_process_type)
 
         print("Deploy finished.")
 

project/tests/test_deploy.py

@@ -7,15 +7,23 @@
 from project.util.testing_util import Snapshot
 import deploy
 
+
+def binary_encode_json(x):
+    return json.dumps(x).encode('utf-8')
+
+
 MY_DIR = Path(__file__).parent.resolve()
 
 SNAPSHOT_DIR = MY_DIR / "test_deploy_snapshots"
 
 DEFAULT_SUBPROCESS_CMD_PREFIX_OUTPUTS = {
     "git remote get-url": b"https://git.heroku.com/boop.git",
-    "heroku config": json.dumps({
+    "heroku config -j": binary_encode_json({
         'DATABASE_URL': 'postgres://boop'
-    }).encode('utf-8'),
+    }),
+    "heroku features:info preboot --json": binary_encode_json({
+        "enabled": False
+    }),
     "git rev-parse HEAD": b"e7408710b8d091377041cfbe4c185931a214f280",
     "git status -uno --porcelain": b"M somefile.py",
     "heroku auth:token": b"00112233-aabb-ccdd-eeff-001122334455",
@@ -33,7 +41,16 @@ def fake_temporary_directory():
     monkeypatch.setattr(tempfile, 'TemporaryDirectory', fake_temporary_directory)
 
 
-def create_check_output(cmd_prefix_outputs=DEFAULT_SUBPROCESS_CMD_PREFIX_OUTPUTS):
+def create_cmd_prefix_outputs(overrides=None):
+    return {
+        **DEFAULT_SUBPROCESS_CMD_PREFIX_OUTPUTS,
+        **(overrides or {}),
+    }
+
+
+def create_check_output(cmd_prefix_outputs=None):
+    cmd_prefix_outputs = create_cmd_prefix_outputs(cmd_prefix_outputs)
+
     def check_output(args, **kwargs):
         cmd = ' '.join(args)
         for cmd_prefix, output in cmd_prefix_outputs.items():
@@ -105,3 +122,18 @@ def test_heroku_works(subprocess, capsys):
 
     snapshot = Snapshot(capsys.readouterr().out, SNAPSHOT_DIR / "heroku_works.txt")
     assert snapshot.expected == snapshot.actual
+
+
+def test_heroku_with_preboot(subprocess, capsys):
+    subprocess.check_output.side_effect = create_check_output({
+        "heroku features:info preboot --json": json.dumps({
+            "enabled": True
+        }).encode('utf-8'),
+    })
+    subprocess.call.side_effect = successful_check_call_with_print
+    subprocess.check_call.side_effect = successful_check_call_with_print
+
+    deploy.main(['heroku', '-r', 'myapp'])
+
+    snapshot = Snapshot(capsys.readouterr().out, SNAPSHOT_DIR / "heroku_with_preboot.txt")
+    assert snapshot.expected == snapshot.actual

project/tests/test_deploy_snapshots/heroku_with_preboot.txt

@@ -0,0 +1,13 @@
+Running "docker build -f Dockerfile.web -t registry.heroku.com/boop/web --build-arg GIT_REVISION=e7408710b8d091377041cfbe4c185931a214f280 --build-arg IS_GIT_REPO_PRISTINE=False .".
+Running "docker build -f Dockerfile.worker -t registry.heroku.com/boop/worker --build-arg DOCKERFILE_WEB=registry.heroku.com/boop/web /tmp/somedir".
+Pushing containers to Docker registry...
+Running "docker login --username=_ --password=00112233-aabb-ccdd-eeff-001122334455 registry.heroku.com".
+Running "docker push registry.heroku.com/boop/web".
+Running "docker push registry.heroku.com/boop/worker".
+Heroku preboot is enabled, proceeding with zero-downtime deploy.
+Running migrations...
+Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py migrate".
+Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py initgroups".
+Initiating Heroku release phase...
+Running "heroku container:release web worker -r myapp".
+Deploy finished.

project/tests/test_deploy_snapshots/heroku_works.txt

@@ -4,11 +4,13 @@ Pushing containers to Docker registry...
 Running "docker login --username=_ --password=00112233-aabb-ccdd-eeff-001122334455 registry.heroku.com".
 Running "docker push registry.heroku.com/boop/web".
 Running "docker push registry.heroku.com/boop/worker".
+Heroku preboot is disabled, turning on maintenance mode.
 Running "heroku maintenance:on -r myapp".
 Running migrations...
 Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py migrate".
 Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py initgroups".
 Initiating Heroku release phase...
 Running "heroku container:release web worker -r myapp".
+Turning off maintenance mode.
 Running "heroku maintenance:off -r myapp".
 Deploy finished.