diff --git a/.circleci/config.yml b/.circleci/config.yml index c1e73387b..df42d118d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -160,7 +160,18 @@ workflows: version: 2 build_and_deploy: jobs: - - build + - build: + filters: + branches: + ignore: + # This is a bit counter-intuitive, but we've been extremely + # diligent about only pushing to production once something + # has been verified to pass CI on master. Because of this, + # it's wasted effort to run the exact same tests on production + # when we push to it, particularly since the merge on + # production is a fast-forward merge and therefore represents + # the exact same code that's already been tested on master. + - production - deploy: requires: - build diff --git a/deploy.py b/deploy.py index f099cb570..c52635787 100644 --- a/deploy.py +++ b/deploy.py @@ -5,6 +5,7 @@ import json import re import tempfile +import contextlib from dataclasses import dataclass from typing import List, Optional, Dict from pathlib import Path @@ -114,6 +115,14 @@ def run(self, *args: str): cmdline = self._get_cmdline(*args) subprocess.check_call(cmdline, cwd=self.cwd, shell=self.shell) + def is_preboot_enabled(self, *args: str) -> bool: + result = subprocess.check_output( + self._get_cmdline('features:info', 'preboot', '--json'), + cwd=self.cwd, + shell=self.shell + ) + return json.loads(result)['enabled'] + def get_full_config(self) -> Dict[str, str]: result = subprocess.check_output( self._get_cmdline('config', '-j'), @@ -201,6 +210,34 @@ def build(self, cache_from_docker_registry: bool): build_worker_container(self.worker_container_tag, dockerfile_web=self.container_tag) + @contextlib.contextmanager + def maintenance_mode_if_preboot_is_disabled(self): + ''' + If Heroku preboot is disabled, wrap the enclosed code in Heroku's + maintenance mode. Otherwise, we'll assume this is a zero-downtime + deploy, e.g. that any migrations that do need to be run will be ones + that the old version of the code is still compatible with. + + Note that if the enclosed code raises an exception, we do _not_ + disable maintenance mode, since we're assuming that the site + is broken and maintainers will still need it to be in maintenance + mode in order to fix it. + ''' + + is_preboot_enabled = self.heroku.is_preboot_enabled() + + if is_preboot_enabled: + print("Heroku preboot is enabled, proceeding with zero-downtime deploy.") + else: + print("Heroku preboot is disabled, turning on maintenance mode.") + self.heroku.run('maintenance:on') + + yield + + if not is_preboot_enabled: + print("Turning off maintenance mode.") + self.heroku.run('maintenance:off') + def deploy(self) -> None: print("Pushing containers to Docker registry...") self.push_to_docker_registry() @@ -219,19 +256,16 @@ def deploy(self) -> None: # if self.is_using_rollbar: # self.run_in_container(['python', 'manage.py', 'rollbarsourcemaps']) - self.heroku.run('maintenance:on') - - # We want migrations to run while we're in maintenance mode because - # our codebase doesn't make any guarantees about being able to run - # on database schemas from previous or future versions. - print("Running migrations...") - self.run_in_container(['python', 'manage.py', 'migrate']) - self.run_in_container(['python', 'manage.py', 'initgroups']) - - print("Initiating Heroku release phase...") - self.heroku.run('container:release', self.process_type, self.worker_process_type) + with self.maintenance_mode_if_preboot_is_disabled(): + # If Heroku preboot is disabled, then we want migrations to run while we're in + # maintenance mode because we're assuming our codebase doesn't make any guarantees + # about being able to run on database schemas from previous or future versions. + print("Running migrations...") + self.run_in_container(['python', 'manage.py', 'migrate']) + self.run_in_container(['python', 'manage.py', 'initgroups']) - self.heroku.run('maintenance:off') + print("Initiating Heroku release phase...") + self.heroku.run('container:release', self.process_type, self.worker_process_type) print("Deploy finished.") diff --git a/project/tests/test_deploy.py b/project/tests/test_deploy.py index e004a1f77..e9d22cb5c 100644 --- a/project/tests/test_deploy.py +++ b/project/tests/test_deploy.py @@ -7,15 +7,23 @@ from project.util.testing_util import Snapshot import deploy + +def binary_encode_json(x): + return json.dumps(x).encode('utf-8') + + MY_DIR = Path(__file__).parent.resolve() SNAPSHOT_DIR = MY_DIR / "test_deploy_snapshots" DEFAULT_SUBPROCESS_CMD_PREFIX_OUTPUTS = { "git remote get-url": b"https://git.heroku.com/boop.git", - "heroku config": json.dumps({ + "heroku config -j": binary_encode_json({ 'DATABASE_URL': 'postgres://boop' - }).encode('utf-8'), + }), + "heroku features:info preboot --json": binary_encode_json({ + "enabled": False + }), "git rev-parse HEAD": b"e7408710b8d091377041cfbe4c185931a214f280", "git status -uno --porcelain": b"M somefile.py", "heroku auth:token": b"00112233-aabb-ccdd-eeff-001122334455", @@ -33,7 +41,16 @@ def fake_temporary_directory(): monkeypatch.setattr(tempfile, 'TemporaryDirectory', fake_temporary_directory) -def create_check_output(cmd_prefix_outputs=DEFAULT_SUBPROCESS_CMD_PREFIX_OUTPUTS): +def create_cmd_prefix_outputs(overrides=None): + return { + **DEFAULT_SUBPROCESS_CMD_PREFIX_OUTPUTS, + **(overrides or {}), + } + + +def create_check_output(cmd_prefix_outputs=None): + cmd_prefix_outputs = create_cmd_prefix_outputs(cmd_prefix_outputs) + def check_output(args, **kwargs): cmd = ' '.join(args) for cmd_prefix, output in cmd_prefix_outputs.items(): @@ -105,3 +122,18 @@ def test_heroku_works(subprocess, capsys): snapshot = Snapshot(capsys.readouterr().out, SNAPSHOT_DIR / "heroku_works.txt") assert snapshot.expected == snapshot.actual + + +def test_heroku_with_preboot(subprocess, capsys): + subprocess.check_output.side_effect = create_check_output({ + "heroku features:info preboot --json": json.dumps({ + "enabled": True + }).encode('utf-8'), + }) + subprocess.call.side_effect = successful_check_call_with_print + subprocess.check_call.side_effect = successful_check_call_with_print + + deploy.main(['heroku', '-r', 'myapp']) + + snapshot = Snapshot(capsys.readouterr().out, SNAPSHOT_DIR / "heroku_with_preboot.txt") + assert snapshot.expected == snapshot.actual diff --git a/project/tests/test_deploy_snapshots/heroku_with_preboot.txt b/project/tests/test_deploy_snapshots/heroku_with_preboot.txt new file mode 100644 index 000000000..fc0d5e7af --- /dev/null +++ b/project/tests/test_deploy_snapshots/heroku_with_preboot.txt @@ -0,0 +1,13 @@ +Running "docker build -f Dockerfile.web -t registry.heroku.com/boop/web --build-arg GIT_REVISION=e7408710b8d091377041cfbe4c185931a214f280 --build-arg IS_GIT_REPO_PRISTINE=False .". +Running "docker build -f Dockerfile.worker -t registry.heroku.com/boop/worker --build-arg DOCKERFILE_WEB=registry.heroku.com/boop/web /tmp/somedir". +Pushing containers to Docker registry... +Running "docker login --username=_ --password=00112233-aabb-ccdd-eeff-001122334455 registry.heroku.com". +Running "docker push registry.heroku.com/boop/web". +Running "docker push registry.heroku.com/boop/worker". +Heroku preboot is enabled, proceeding with zero-downtime deploy. +Running migrations... +Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py migrate". +Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py initgroups". +Initiating Heroku release phase... +Running "heroku container:release web worker -r myapp". +Deploy finished. diff --git a/project/tests/test_deploy_snapshots/heroku_works.txt b/project/tests/test_deploy_snapshots/heroku_works.txt index 854bce465..eadb26502 100644 --- a/project/tests/test_deploy_snapshots/heroku_works.txt +++ b/project/tests/test_deploy_snapshots/heroku_works.txt @@ -4,11 +4,13 @@ Pushing containers to Docker registry... Running "docker login --username=_ --password=00112233-aabb-ccdd-eeff-001122334455 registry.heroku.com". Running "docker push registry.heroku.com/boop/web". Running "docker push registry.heroku.com/boop/worker". +Heroku preboot is disabled, turning on maintenance mode. Running "heroku maintenance:on -r myapp". Running migrations... Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py migrate". Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py initgroups". Initiating Heroku release phase... Running "heroku container:release web worker -r myapp". +Turning off maintenance mode. Running "heroku maintenance:off -r myapp". Deploy finished.