Add tests for deploy.py. (#1363)

I'd like to start making some improvements to our deployment script to support things like Heroku preboot, but that introduces enough complexity that I'd like to add proper unit tests for our deploy script first.

This also adds a new `Snapshot` helper class that makes it easier to do Jest-like snapshot testing using pytest.

This also removes `deploy.py` from our `.coveragerc` so we're now tracking coverage of it.  The coverage for the whole project is going down right now because we don't have enough tests to cover the whole thing, and I need to move on to I18n for now so I don't have time to add full test coverage, so we'll just have to do with what we've got for now.

.coveragerc

@@ -3,4 +3,3 @@ omit =
   .venv/*
   docker_django_management.py
   manage.py
-  deploy.py

deploy.py

@@ -233,6 +233,8 @@ def deploy(self) -> None:
 
         self.heroku.run('maintenance:off')
 
+        print("Deploy finished.")
+
 
 def deploy_heroku(args):
     deployer = HerokuDeployer(args.remote)
@@ -266,7 +268,7 @@ def selfcheck(args):
     print("Deployment prerequisites satisfied!")
 
 
-def main():
+def main(args: Optional[List[str]] = None):
     parser = argparse.ArgumentParser()
     subparsers = parser.add_subparsers(
         title='subcommands',
@@ -326,12 +328,12 @@ def main():
     )
     parser_heroku_run.set_defaults(func=heroku_run)
 
-    args = parser.parse_args()
-    if not hasattr(args, 'func'):
+    parsed_args = parser.parse_args(args)
+    if not hasattr(parsed_args, 'func'):
         parser.print_help()
         sys.exit(1)
 
-    args.func(args)
+    parsed_args.func(parsed_args)
 
 
 if __name__ == '__main__':

project/tests/test_deploy.py

@@ -0,0 +1,107 @@
+from pathlib import Path
+import contextlib
+from unittest.mock import MagicMock
+import json
+import pytest
+
+from project.util.testing_util import Snapshot
+import deploy
+
+MY_DIR = Path(__file__).parent.resolve()
+
+SNAPSHOT_DIR = MY_DIR / "test_deploy_snapshots"
+
+DEFAULT_SUBPROCESS_CMD_PREFIX_OUTPUTS = {
+    "git remote get-url": b"https://git.heroku.com/boop.git",
+    "heroku config": json.dumps({
+        'DATABASE_URL': 'postgres://boop'
+    }).encode('utf-8'),
+    "git rev-parse HEAD": b"e7408710b8d091377041cfbe4c185931a214f280",
+    "git status -uno --porcelain": b"M somefile.py",
+    "heroku auth:token": b"00112233-aabb-ccdd-eeff-001122334455",
+}
+
+
+@pytest.fixture(autouse=True)
+def fake_tempfile(monkeypatch):
+    import tempfile
+
+    @contextlib.contextmanager
+    def fake_temporary_directory():
+        yield "/tmp/somedir"
+
+    monkeypatch.setattr(tempfile, 'TemporaryDirectory', fake_temporary_directory)
+
+
+def create_check_output(cmd_prefix_outputs=DEFAULT_SUBPROCESS_CMD_PREFIX_OUTPUTS):
+    def check_output(args, **kwargs):
+        cmd = ' '.join(args)
+        for cmd_prefix, output in cmd_prefix_outputs.items():
+            assert isinstance(output, bytes), f"Output for '{cmd_prefix}' must be bytes"
+            if cmd.startswith(cmd_prefix):
+                return output
+        raise AssertionError(f"Unexpected check_output args: {args}")
+
+    return check_output
+
+
+def successful_check_call_with_print(args, **kwargs):
+    cmd = ' '.join(args)
+    print(f'Running "{cmd}".')
+    return 0
+
+
+@pytest.fixture
+def subprocess(monkeypatch):
+    import subprocess
+    monkeypatch.setattr(subprocess, 'check_call', MagicMock())
+    monkeypatch.setattr(subprocess, 'check_output', MagicMock())
+    monkeypatch.setattr(subprocess, 'call', MagicMock())
+    yield subprocess
+
+
+@contextlib.contextmanager
+def expect_normal_exit():
+    with pytest.raises(SystemExit) as excinfo:
+        yield
+    assert excinfo.value.code == 0
+
+
+@contextlib.contextmanager
+def expect_abnormal_exit():
+    with pytest.raises(SystemExit) as excinfo:
+        yield
+    assert excinfo.value.code != 0
+
+
+def test_it_shows_help_when_asked_for_help(capsys):
+    with expect_normal_exit():
+        deploy.main(["--help"])
+    assert "usage: " in capsys.readouterr().out
+
+
+def test_it_shows_help_when_given_no_args(capsys):
+    with expect_abnormal_exit():
+        deploy.main([])
+    assert "usage: " in capsys.readouterr().out
+
+
+def test_selfcheck_works(capsys):
+    deploy.main(['selfcheck'])
+    assert "Deployment prerequisites satisfied" in capsys.readouterr().out
+
+
+def test_heroku_raises_err_with_no_remote(capsys):
+    with pytest.raises(ValueError, match="Please specify a git remote"):
+        deploy.main(['heroku'])
+
+
+def test_heroku_works(subprocess, capsys):
+    subprocess.check_output.side_effect = create_check_output()
+    subprocess.call.side_effect = successful_check_call_with_print
+    subprocess.check_call.side_effect = successful_check_call_with_print
+
+    deploy.main(['heroku', '-r', 'myapp'])
+
+    snapshot = Snapshot(capsys.readouterr().out, SNAPSHOT_DIR / "heroku_works.txt")
+    assert snapshot.expected == snapshot.actual

project/tests/test_deploy_snapshots/heroku_works.txt

@@ -0,0 +1,14 @@
+Running "docker build -f Dockerfile.web -t registry.heroku.com/boop/web --build-arg GIT_REVISION=e7408710b8d091377041cfbe4c185931a214f280 --build-arg IS_GIT_REPO_PRISTINE=False .".
+Running "docker build -f Dockerfile.worker -t registry.heroku.com/boop/worker --build-arg DOCKERFILE_WEB=registry.heroku.com/boop/web /tmp/somedir".
+Pushing containers to Docker registry...
+Running "docker login --username=_ --password=00112233-aabb-ccdd-eeff-001122334455 registry.heroku.com".
+Running "docker push registry.heroku.com/boop/web".
+Running "docker push registry.heroku.com/boop/worker".
+Running "heroku maintenance:on -r myapp".
+Running migrations...
+Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py migrate".
+Running "docker run --rm -it -e DATABASE_URL registry.heroku.com/boop/web python manage.py initgroups".
+Initiating Heroku release phase...
+Running "heroku container:release web worker -r myapp".
+Running "heroku maintenance:off -r myapp".
+Deploy finished.

project/util/testing_util.py

@@ -1,3 +1,5 @@
+import sys
+from pathlib import Path
 from typing import Dict, Any, Optional
 import pytest
 
@@ -124,3 +126,42 @@ class Blob:
 
     def __init__(self, **kwargs):
         self.__dict__ = kwargs
+
+
+class Snapshot:
+    '''
+    A helper class to make snapshot/golden file testing easier in pytest.
+
+    To use it, pass in the actual output that you want to snapshot test
+    against, and a Path to a file that stores the snapshot's expected output.
+
+    If the snapshot file doesn't already exist, it will be created with
+    the passed-in actual output.
+
+    The content of the snapshot file will be set to the 'expected'
+    attribute, while the passed-in actual output will be set to the
+    'actual' attribute.
+
+    You'll want to `assert snapshot.actual == snapshot.expected` in
+    your actual test, as this will make sure that helpful diff output
+    is displayed if the snapshot test fails.
+
+    To update the snapshot to match the current actual output, you
+    will need to manually delete the snapshot file.
+    '''
+
+    def __init__(self, actual: str, path: Path):
+        self.actual = actual
+        self.path = path
+
+        if not path.exists():
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(actual)
+
+        self.expected = path.read_text()
+
+        if self.actual != self.expected:
+            sys.stderr.write(
+                f"Warning: snapshot does not match! To update the snapshot, "
+                f"delete '{self.path}'."
+            )