From be0ae4dae8368d042af8065aea0c6869ded04561 Mon Sep 17 00:00:00 2001 From: Atul Varma Date: Mon, 4 May 2020 16:49:24 -0400 Subject: [PATCH] Support Heroku preboot, speed up production deploys. (#1366) This adds support for Heroku preboot. If it's enabled, we don't enter maintenance mode at all, resulting in a zero-downtime deploy; otherwise, we behave as normal, entering maintenance mode before doing any migrations. This basically means that we need to disable preboot if we ever make any migrations that will cause the old version of the code to crash. Once we've successfully done a deploy with Preboot, I'll update our Deployment wiki page with new instructions. **It also prevents tests from being run on the `production` branch.** This is a bit of a bold move but I've provided the rationale in a comment in our `circle.yml`. ``` # This is a bit counter-intuitive, but we've been extremely # diligent about only pushing to production once something # has been verified to pass CI on master. Because of this, # it's wasted effort to run the exact same tests on production # when we push to it, particularly since the merge on # production is a fast-forward merge and therefore represents # the exact same code that's already been tested on master. ``` Taken together, both changes should make deploys to production much faster without introducing risk. --- .circleci/config.yml | 13 ++++- deploy.py | 58 +++++++++++++++---- project/tests/test_deploy.py | 38 +++++++++++- .../heroku_with_preboot.txt | 13 +++++ .../test_deploy_snapshots/heroku_works.txt | 2 + 5 files changed, 108 insertions(+), 16 deletions(-) create mode 100644 project/tests/test_deploy_snapshots/heroku_with_preboot.txt diff --git a/.circleci/config.yml b/.circleci/config.yml index c1e73387b..df42d118d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -160,7 +160,18 @@ workflows: version: 2 build_and_deploy: jobs: - - build + - build: + filters: + branches: + ignore: + # This is a bit counter-intuitive, but we've been extremely + # diligent about only pushing to production once something + # has been verified to pass CI on master. Because of this, + # it's wasted effort to run the exact same tests on production + # when we push to it, particularly since the merge on + # production is a fast-forward merge and therefore represents + # the exact same code that's already been tested on master. + - production - deploy: requires: - build diff --git a/deploy.py b/deploy.py index f099cb570..c52635787 100644 --- a/deploy.py +++ b/deploy.py @@ -5,6 +5,7 @@ import json import re import tempfile +import contextlib from dataclasses import dataclass from typing import List, Optional, Dict from pathlib import Path @@ -114,6 +115,14 @@ def run(self, *args: str): cmdline = self._get_cmdline(*args) subprocess.check_call(cmdline, cwd=self.cwd, shell=self.shell) + def is_preboot_enabled(self, *args: str) -> bool: + result = subprocess.check_output( + self._get_cmdline('features:info', 'preboot', '--json'), + cwd=self.cwd, + shell=self.shell + ) + return json.loads(result)['enabled'] + def get_full_config(self) -> Dict[str, str]: result = subprocess.check_output( self._get_cmdline('config', '-j'), @@ -201,6 +210,34 @@ def build(self, cache_from_docker_registry: bool): build_worker_container(self.worker_container_tag, dockerfile_web=self.container_tag) + @contextlib.contextmanager + def maintenance_mode_if_preboot_is_disabled(self): + ''' + If Heroku preboot is disabled, wrap the enclosed code in Heroku's + maintenance mode. Otherwise, we'll assume this is a zero-downtime + deploy, e.g. that any migrations that do need to be run will be ones + that the old version of the code is still compatible with. + + Note that if the enclosed code raises an exception, we do _not_ + disable maintenance mode, since we're assuming that the site + is broken and maintainers will still need it to be in maintenance + mode in order to fix it. + ''' + + is_preboot_enabled = self.heroku.is_preboot_enabled() + + if is_preboot_enabled: + print("Heroku preboot is enabled, proceeding with zero-downtime deploy.") + else: + print("Heroku preboot is disabled, turning on maintenance mode.") + self.heroku.run('maintenance:on') + + yield + + if not is_preboot_enabled: + print("Turning off maintenance mode.") + self.heroku.run('maintenance:off') + def deploy(self) -> None: print("Pushing containers to Docker registry...") self.push_to_docker_registry() @@ -219,19 +256,16 @@ def deploy(self) -> None: # if self.is_using_rollbar: # self.run_in_container(['python', 'manage.py', 'rollbarsourcemaps']) - self.heroku.run('maintenance:on') - - # We want migrations to run while we're in maintenance mode because - # our codebase doesn't make any guarantees about being able to run - # on database schemas from previous or future versions. - print("Running migrations...") - self.run_in_container(['python', 'manage.py', 'migrate']) - self.run_in_container(['python', 'manage.py', 'initgroups']) - - print("Initiating Heroku release phase...") - self.heroku.run('container:release', self.process_type, self.worker_process_type) + with self.maintenance_mode_if_preboot_is_disabled(): + # If Heroku preboot is disabled, then we want migrations to run while we're in + # maintenance mode because we're assuming our codebase doesn't make any guarantees + # about being able to run on database schemas from previous or future versions. + print("Running migrations...") + self.run_in_container(['python', 'manage.py', 'migrate']) + self.run_in_container(['python', 'manage.py', 'initgroups']) - self.heroku.run('maintenance:off') + print("Initiating Heroku release phase...") + self.heroku.run('container:release', self.process_type, self.worker_process_type) print("Deploy finished.") diff --git a/project/tests/test_deploy.py b/project/tests/test_deploy.py index e004a1f77..e9d22cb5c 100644 --- a/project/tests/test_deploy.py +++ b/project/tests/test_deploy.py @@ -7,15 +7,23 @@ from project.util.testing_util import Snapshot import deploy + +def binary_encode_json(x): + return json.dumps(x).encode('utf-8') + + MY_DIR = Path(__file__).parent.resolve() SNAPSHOT_DIR = MY_DIR / "test_deploy_snapshots" DEFAULT_SUBPROCESS_CMD_PREFIX_OUTPUTS = { "git remote get-url": b"https://git.heroku.com/boop.git", - "heroku config": json.dumps({ + "heroku config -j": binary_encode_json({ 'DATABASE_URL': 'postgres://boop' - }).encode('utf-8'), + }), + "heroku features:info preboot --json": binary_encode_json({ + "enabled": False + }), "git rev-parse HEAD": b"e7408710b8d091377041cfbe4c185931a214f280", "git status -uno --porcelain": b"M somefile.py", "heroku auth:token": b"00112233-aabb-ccdd-eeff-001122334455", @@ -33,7 +41,16 @@ def fake_temporary_directory(): monkeypatch.setattr(tempfile, 'TemporaryDirectory', fake_temporary_directory) -def create_check_output(cmd_prefix_outputs=DEFAULT_SUBPROCESS_CMD_PREFIX_OUTPUTS): +def create_cmd_prefix_outputs(overrides=None): + return { + **DEFAULT_SUBPROCESS_CMD_PREFIX_OUTPUTS, + **(overrides or {}), + } + + +def create_check_output(cmd_prefix_outputs=None): + cmd_prefix_outputs = create_cmd_prefix_outputs(cmd_prefix_outputs) + def check_output(args, **kwargs): cmd = ' '.join(args) for cmd_prefix, output in cmd_prefix_outputs.items(): @@ -105,3 +122,18 @@ def test_heroku_works(subprocess, capsys): snapshot = Snapshot(capsys.readouterr().out, SNAPSHOT_DIR / "heroku_works.txt") assert snapshot.expected == snapshot.actual + + +def test_heroku_with_preboot(subprocess, capsys): + subprocess.check_output.side_effect = create_check_output({ + "heroku features:info preboot --json": json.dumps({ + "enabled": True + }).encode('utf-8'), + }) + subprocess.call.side_effect = successful_check_call_with_print + subprocess.check_call.side_effect = successful_check_call_with_print + + deploy.main(['heroku', '-r', 'myapp']) + + snapshot = Snapshot(capsys.readouterr().out, SNAPSHOT_DIR / "heroku_with_preboot.txt") + assert snapshot.expected == snapshot.actual diff --git a/project/tests/test_deploy_snapshots/heroku_with_preboot.txt b/project/tests/test_deploy_snapshots/heroku_with_preboot.txt new file mode 100644 index 000000000..fc0d5e7af --- /dev/null +++ b/project/tests/test_deploy_snapshots/heroku_with_preboot.txt @@ -0,0 +1,13 @@ +Running "docker build -f Dockerfile.web -t registry.heroku.com/boop/web --build-arg GIT_REVISION=e7408710b8d091377041cfbe4c185931a214f280 --build-arg IS_GIT_REPO_PRISTINE=False .". +Running "docker build -f Dockerfile.worker -t registry.heroku.com/boop/worker --build-arg DOCKERFILE_WEB=registry.heroku.com/boop/web /tmp/somedir". +Pushing containers to Docker registry... +Running "docker login --username=_ --password=00112233-aabb-ccdd-eeff-001122334455 registry.heroku.com". +Running "docker push registry.heroku.com/boop/web". +Running "docker push registry.heroku.com/boop/worker". +Heroku preboot is enabled, proceeding with zero-downtime deploy. +Running migrations... +Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py migrate". +Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py initgroups". +Initiating Heroku release phase... +Running "heroku container:release web worker -r myapp". +Deploy finished. diff --git a/project/tests/test_deploy_snapshots/heroku_works.txt b/project/tests/test_deploy_snapshots/heroku_works.txt index 854bce465..eadb26502 100644 --- a/project/tests/test_deploy_snapshots/heroku_works.txt +++ b/project/tests/test_deploy_snapshots/heroku_works.txt @@ -4,11 +4,13 @@ Pushing containers to Docker registry... Running "docker login --username=_ --password=00112233-aabb-ccdd-eeff-001122334455 registry.heroku.com". Running "docker push registry.heroku.com/boop/web". Running "docker push registry.heroku.com/boop/worker". +Heroku preboot is disabled, turning on maintenance mode. Running "heroku maintenance:on -r myapp". Running migrations... Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py migrate". Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py initgroups". Initiating Heroku release phase... Running "heroku container:release web worker -r myapp". +Turning off maintenance mode. Running "heroku maintenance:off -r myapp". Deploy finished.