From d3740e1ff3a69004415d25ba3eb31fafe355d255 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 22 Oct 2019 22:49:15 +0200 Subject: [PATCH 01/25] Add gitpython as dev dependency --- .travis.yml | 3 ++- README.md | 2 +- dev-requirements.txt | 3 +++ 3 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 dev-requirements.txt diff --git a/.travis.yml b/.travis.yml index 998bf51..5c69939 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,8 @@ cache: pip install: - set -e - pip install --upgrade pip - - pip install pyflakes pytest . + - pip install . + - pip install -r dev-requirements.txt script: - chartpress --version - chartpress --help diff --git a/README.md b/README.md index 210f7e7..75d5139 100644 --- a/README.md +++ b/README.md @@ -169,7 +169,7 @@ Testing of this python package can be done using [`pyflakes`](https://github.com pip install -e . # install dev dependencies -pip install pyflakes pytest +pip install -r dev-requirements.txt # run tests pyflakes . diff --git a/dev-requirements.txt b/dev-requirements.txt new file mode 100644 index 0000000..df93b93 --- /dev/null +++ b/dev-requirements.txt @@ -0,0 +1,3 @@ +gitpython +pyflakes +pytest From fe68e51f70e918c8fa9069edd83460b20aabb64c Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 22 Oct 2019 23:20:18 +0200 Subject: [PATCH 02/25] Add git_repo fixture for repo interactions With this git_repo fixture, we can have a clean slate with fresh data to itneract with. --- tests/conftest.py | 17 +++++++++++++++++ tests/data/chartpress.yaml | 0 tests/data/testchart/Chart.yaml | 0 tests/data/testchart/values.yaml | 0 tests/test_repo_interactions.py | 7 +++++++ 5 files changed, 24 insertions(+) create mode 100644 tests/conftest.py create mode 100644 tests/data/chartpress.yaml create mode 100644 tests/data/testchart/Chart.yaml create mode 100644 tests/data/testchart/values.yaml create mode 100644 tests/test_repo_interactions.py diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..e32d5cc --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,17 @@ +import os +import tempfile +from distutils.dir_util import copy_tree + +import git +import pytest + +@pytest.fixture(scope="function") +def git_repo(): + """A temporary git repo with """ + with tempfile.TemporaryDirectory() as temp_dir: + copy_tree("tests/data", os.path.join(temp_dir)) + + stashed_cwd = os.getcwd() + os.chdir(temp_dir) + yield git.Repo.init(temp_dir) + os.chdir(stashed_cwd) diff --git a/tests/data/chartpress.yaml b/tests/data/chartpress.yaml new file mode 100644 index 0000000..e69de29 diff --git a/tests/data/testchart/Chart.yaml b/tests/data/testchart/Chart.yaml new file mode 100644 index 0000000..e69de29 diff --git a/tests/data/testchart/values.yaml b/tests/data/testchart/values.yaml new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_repo_interactions.py b/tests/test_repo_interactions.py new file mode 100644 index 0000000..4c75dd8 --- /dev/null +++ b/tests/test_repo_interactions.py @@ -0,0 +1,7 @@ +import os + +def test_git_repo_fixture(git_repo): + assert git_repo.working_dir == os.getcwd() + assert os.path.isfile("chartpress.yaml") + assert os.path.isfile("testchart/Chart.yaml") + assert os.path.isfile("testchart/values.yaml") From 712c5da03bd841d02ce151fac44f82ae33a21ae8 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 23 Oct 2019 00:56:50 +0200 Subject: [PATCH 03/25] Initial chartpress run test --- tests/conftest.py | 6 ++++-- tests/data/chartpress.yaml | 23 +++++++++++++++++++++++ tests/data/image/Dockerfile | 7 +++++++ tests/data/testchart/Chart.yaml | 3 +++ tests/data/testchart/values.yaml | 4 ++++ tests/test_repo_interactions.py | 6 ++++++ 6 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 tests/data/image/Dockerfile diff --git a/tests/conftest.py b/tests/conftest.py index e32d5cc..1578323 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,8 +10,10 @@ def git_repo(): """A temporary git repo with """ with tempfile.TemporaryDirectory() as temp_dir: copy_tree("tests/data", os.path.join(temp_dir)) - stashed_cwd = os.getcwd() os.chdir(temp_dir) - yield git.Repo.init(temp_dir) + repo = git.Repo.init(temp_dir) + repo.index.add(".") + repo.index.commit("initial commit") + yield repo os.chdir(stashed_cwd) diff --git a/tests/data/chartpress.yaml b/tests/data/chartpress.yaml index e69de29..92f28c0 100644 --- a/tests/data/chartpress.yaml +++ b/tests/data/chartpress.yaml @@ -0,0 +1,23 @@ +charts: + - name: testchart + imagePrefix: testchart/ + resetTag: test-reset-tag + resetVersion: 0.0.1-test.reset.version + repo: + git: jupyterhub/we-are-not-ready-for-this-yet + published: https://test.local + paths: + - extra-path.txt + images: + testimage: + buildArgs: + TEST_STATIC_BUILD_ARG: "test" + TEST_DYNAMIC_BUILD_ARG: "{TAG}-{LAST_COMMIT}" + contextPath: . + dockerfilePath: image/Dockerfile + valuesPath: + - image + - list.0 + - list.1.image + paths: + - extra-path.txt diff --git a/tests/data/image/Dockerfile b/tests/data/image/Dockerfile new file mode 100644 index 0000000..4457ead --- /dev/null +++ b/tests/data/image/Dockerfile @@ -0,0 +1,7 @@ +FROM scratch + +ARG TEST_STATIC_BUILD_ARG +ARG TEST_DYNAMIC_BUILD_ARG + +LABEL test_static_build_arg="$TEST_STATIC_BUILD_ARG" +LABEL test_dynamic_build_arg="$TEST_DYNAMIC_BUILD_ARG" diff --git a/tests/data/testchart/Chart.yaml b/tests/data/testchart/Chart.yaml index e69de29..b0ca6a4 100644 --- a/tests/data/testchart/Chart.yaml +++ b/tests/data/testchart/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v1 +name: testchart +version: 0.0.1-test.reset.version diff --git a/tests/data/testchart/values.yaml b/tests/data/testchart/values.yaml index e69de29..beea351 100644 --- a/tests/data/testchart/values.yaml +++ b/tests/data/testchart/values.yaml @@ -0,0 +1,4 @@ +image: testchart/testimage:test-reset-tag +list: + - "testchart/testimage:test-reset-tag" + - image: "testchart/testimage:test-reset-tag" diff --git a/tests/test_repo_interactions.py b/tests/test_repo_interactions.py index 4c75dd8..c71abc4 100644 --- a/tests/test_repo_interactions.py +++ b/tests/test_repo_interactions.py @@ -1,7 +1,13 @@ import os +import chartpress + def test_git_repo_fixture(git_repo): assert git_repo.working_dir == os.getcwd() assert os.path.isfile("chartpress.yaml") assert os.path.isfile("testchart/Chart.yaml") assert os.path.isfile("testchart/values.yaml") + + +def test_chartpress_run(git_repo): + chartpress.main() From 2d2573f01dbb0a217f4acbd44c1414ec18c29c88 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 23 Oct 2019 01:24:30 +0200 Subject: [PATCH 04/25] Allow for python passed arguments to main() --- chartpress.py | 4 ++-- tests/test_repo_interactions.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/chartpress.py b/chartpress.py index 3af8a05..1fa4664 100755 --- a/chartpress.py +++ b/chartpress.py @@ -519,7 +519,7 @@ def __call__(self, parser, namespace, values, option_string=None): -def main(): +def main(args=None): """Run chartpress""" argparser = argparse.ArgumentParser(description=__doc__) @@ -576,7 +576,7 @@ def main(): help='Deprecated: this flag will be ignored. The new logic to determine if an image needs to be rebuilt does not require this. It will find the time in git history where the image was last in need of a rebuild due to changes, and check if that build exists locally or remotely already.', ) - args = argparser.parse_args() + args = argparser.parse_args(args) if args.version: print(f"chartpress version {__version__}") diff --git a/tests/test_repo_interactions.py b/tests/test_repo_interactions.py index c71abc4..6d2e757 100644 --- a/tests/test_repo_interactions.py +++ b/tests/test_repo_interactions.py @@ -10,4 +10,4 @@ def test_git_repo_fixture(git_repo): def test_chartpress_run(git_repo): - chartpress.main() + chartpress.main([]) From 8f251b356681c1932bd05cd377b11bfaefddd913 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 23 Oct 2019 01:38:45 +0200 Subject: [PATCH 05/25] Use pytest-flakes --- .travis.yml | 3 +-- README.md | 8 +++++--- dev-requirements.txt | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5c69939..18f9458 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,8 +10,7 @@ install: script: - chartpress --version - chartpress --help - - pyflakes . - - pytest -v ./tests + - pytest --verbose --flakes # This is a workaround to an issue caused by the existence of a docker # registrymirror in our CI environment. Without this fix that removes the diff --git a/README.md b/README.md index 75d5139..fa952dd 100644 --- a/README.md +++ b/README.md @@ -162,7 +162,10 @@ git: ## Development -Testing of this python package can be done using [`pyflakes`](https://github.com/PyCQA/pyflakes) and [`pytest`](https://github.com/pytest-dev/pytest). There is also some additional testing that is only run as part of TravisCI, as declared in [`.travis.yml`](.travis.yml). +Testing of this python package can be done using +[`pytest`](https://github.com/pytest-dev/pytest). There is also some additional +testing that is only run as part of TravisCI, as declared in +[`.travis.yml`](.travis.yml). ``` # install chartpress locally @@ -172,6 +175,5 @@ pip install -e . pip install -r dev-requirements.txt # run tests -pyflakes . -pytest -v +pytest --verbose --flakes --exitfirst ``` diff --git a/dev-requirements.txt b/dev-requirements.txt index df93b93..03007b5 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,3 +1,3 @@ gitpython -pyflakes pytest +pytest-flakes From 37cd973399cd303c4d54e37d994f1abb9dabbc53 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 23 Oct 2019 01:46:10 +0200 Subject: [PATCH 06/25] Ability to work with a untagged branch --- chartpress.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/chartpress.py b/chartpress.py index 1fa4664..5bc2e58 100755 --- a/chartpress.py +++ b/chartpress.py @@ -72,12 +72,17 @@ def latest_tag_or_mod_commit(*paths, **kwargs): **kwargs, ).decode('utf-8').strip() - git_describe_head = check_output( - [ - 'git', 'describe', '--tags', '--long', - ], - **kwargs, - ).decode('utf-8').strip().rsplit("-", maxsplit=2) + try: + git_describe_head = check_output( + [ + 'git', 'describe', '--tags', '--long', + ], + **kwargs, + ).decode('utf-8').strip().rsplit("-", maxsplit=2) + except subprocess.CalledProcessError: + # no tags on branch + return latest_modification_commit + latest_tag = git_describe_head[0] latest_tagged_commit = check_output( [ From 7f80b23ba8bf98133922562277ba4de4438bec2d Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 23 Oct 2019 16:44:51 +0200 Subject: [PATCH 07/25] Use pytest's monkeypatch and test_git_remote --- chartpress.py | 1 + tests/README.md | 11 +++++++++++ tests/conftest.py | 12 ++++++++---- tests/{test_regexp.py => test_helpers.py} | 10 ++++++++++ 4 files changed, 30 insertions(+), 4 deletions(-) create mode 100644 tests/README.md rename tests/{test_regexp.py => test_helpers.py} (71%) diff --git a/chartpress.py b/chartpress.py index 5bc2e58..77ea2ec 100755 --- a/chartpress.py +++ b/chartpress.py @@ -470,6 +470,7 @@ def publish_pages(chart_name, chart_version, chart_repo_github_path, chart_repo_ git_remote(chart_repo_github_path), checkout_dir, ], + # warning: if echoed, this call could reveal the github token echo=False, ) check_call(['git', 'checkout', 'gh-pages'], cwd=checkout_dir) diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..975cd70 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,11 @@ +# Notes about the tests + +These tests are [pytest](https://docs.pytest.org) tests. + +## References +- [pytest](https://docs.pytest.org) +- [pytest-flakes](https://github.com/fschulze/pytest-flakes) and + [pyflakes](https://github.com/PyCQA/pyflakes) for passive code check tests + added to the other pytest tests. +- [monkeypatching](https://docs.pytest.org/en/latest/monkeypatch.html) +- [GitPython](https://gitpython.readthedocs.io/en/stable/) diff --git a/tests/conftest.py b/tests/conftest.py index 1578323..22f36f0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,14 +6,18 @@ import pytest @pytest.fixture(scope="function") -def git_repo(): +def git_repo(monkeypatch): """A temporary git repo with """ with tempfile.TemporaryDirectory() as temp_dir: + # copy content of tests/data folder to the temp dir copy_tree("tests/data", os.path.join(temp_dir)) - stashed_cwd = os.getcwd() - os.chdir(temp_dir) + + # enter the directory + monkeypatch.chdir(temp_dir) + + # initialize the repo and make one initial commit repo = git.Repo.init(temp_dir) repo.index.add(".") repo.index.commit("initial commit") + yield repo - os.chdir(stashed_cwd) diff --git a/tests/test_regexp.py b/tests/test_helpers.py similarity index 71% rename from tests/test_regexp.py rename to tests/test_helpers.py index b597655..89c053c 100644 --- a/tests/test_regexp.py +++ b/tests/test_helpers.py @@ -1,3 +1,6 @@ +from chartpress import GITHUB_TOKEN_KEY + +from chartpress import git_remote from chartpress import _strip_identifiers_build_suffix from chartpress import _get_identifier @@ -12,3 +15,10 @@ def test__get_identifier(): assert _get_identifier(tag="0.1.2-alpha.1", n_commits="0", commit="asdf1234", long=True) == "0.1.2-alpha.1.000.asdf1234" assert _get_identifier(tag="0.1.2-alpha.1", n_commits="0", commit="asdf1234", long=False) == "0.1.2-alpha.1" assert _get_identifier(tag="0.1.2-alpha.1", n_commits="5", commit="asdf1234", long=False) == "0.1.2-alpha.1.005.asdf1234" + +def test_git_remote(monkeypatch): + monkeypatch.setenv(GITHUB_TOKEN_KEY, "test-github-token") + assert git_remote("jupyterhub/helm-chart") == "https://test-github-token@github.com/jupyterhub/helm-chart" + + monkeypatch.delenv(GITHUB_TOKEN_KEY) + assert git_remote("jupyterhub/helm-chart") == "git@github.com:jupyterhub/helm-chart" From ca1c5f769e6a8c4cb788a0470b94c14220409ee9 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 23 Oct 2019 17:59:23 +0200 Subject: [PATCH 08/25] Add test_latest_tag_or_mod_commit --- tests/conftest.py | 8 ++++---- tests/test_helpers.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 22f36f0..656558a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,8 +16,8 @@ def git_repo(monkeypatch): monkeypatch.chdir(temp_dir) # initialize the repo and make one initial commit - repo = git.Repo.init(temp_dir) - repo.index.add(".") - repo.index.commit("initial commit") + r = git.Repo.init(temp_dir) + r.git.add(all=True) + r.index.commit("initial commit") - yield repo + yield r diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 89c053c..d2955d2 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,6 +1,7 @@ from chartpress import GITHUB_TOKEN_KEY from chartpress import git_remote +from chartpress import latest_tag_or_mod_commit from chartpress import _strip_identifiers_build_suffix from chartpress import _get_identifier @@ -22,3 +23,17 @@ def test_git_remote(monkeypatch): monkeypatch.delenv(GITHUB_TOKEN_KEY) assert git_remote("jupyterhub/helm-chart") == "git@github.com:jupyterhub/helm-chart" + +def test_latest_tag_or_mod_commit(git_repo): + open('tag-mod.txt', "w").close() + git_repo.index.add("tag-mod.txt") + tag_commit = git_repo.index.commit("tag commit") + git_repo.create_tag("1.0.0", message="1.0.0") + + open('post-tag-mod.txt', "w").close() + git_repo.index.add("post-tag-mod.txt") + post_tag_commit = git_repo.index.commit("post tag commit") + + assert tag_commit.hexsha.startswith(latest_tag_or_mod_commit("chartpress.yaml")) + assert tag_commit.hexsha.startswith(latest_tag_or_mod_commit("tag-mod.txt")) + assert post_tag_commit.hexsha.startswith(latest_tag_or_mod_commit("post-tag-mod.txt")) From 1214e36996adf002751a62aaf8b766d0f83dafd2 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 23 Oct 2019 20:03:57 +0200 Subject: [PATCH 09/25] pytest everything previously in .travis.yml --- .travis.yml | 39 ------------- README.md | 7 +-- chartpress.py | 7 ++- tests/README.md | 5 +- tests/data/chartpress.yaml | 1 + tests/test_helpers.py | 6 +- tests/test_repo_interactions.py | 99 ++++++++++++++++++++++++++++++++- 7 files changed, 113 insertions(+), 51 deletions(-) diff --git a/.travis.yml b/.travis.yml index 18f9458..0294e5b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,42 +11,3 @@ script: - chartpress --version - chartpress --help - pytest --verbose --flakes - - # This is a workaround to an issue caused by the existence of a docker - # registrymirror in our CI environment. Without this fix that removes the - # mirror, chartpress fails to realize the existence of already built images - # and rebuilds them. - # - # ref: https://github.com/moby/moby/issues/39120 - - |- - echo '{"mtu": 1460}' | sudo dd of=/etc/docker/daemon.json && - sudo systemctl restart docker && - echo "Travis docker registry mirroring disabled.\n\n" - - - |- - git clone https://github.com/jupyterhub/zero-to-jupyterhub-k8s && - cd zero-to-jupyterhub-k8s && - chartpress && - echo -e "\n\n### realistic run: expecting no rebuilds" && - git --no-pager diff --unified=1 - - - |- - chartpress --skip-build --tag 1.2.3-test.tag && - echo -e "\n\n### --tag --skip-build: expecting no rebuilds and version 1.2.3-test.tag" && - git --no-pager diff --unified=1 - - - |- - git tag -a 1.2.3-test.tag -m 1.2.3-test.tag HEAD && - chartpress --skip-build && - echo -e "\n\n### git tag: expecting version 1.2.3-test.tag" && - git --no-pager diff --unified=1 - - - |- - chartpress --skip-build --long && - echo -e "\n\n### --long: expecting version 1.2.3-test.tag+000.asdf1234" && - git --no-pager diff --unified=1 - - - |- - chartpress --reset && - echo -e '\n\n### --reset: expecting version "0.0.1+set.by.chartpress"' && - git --no-pager diff --unified=1 diff --git a/README.md b/README.md index fa952dd..26873e6 100644 --- a/README.md +++ b/README.md @@ -163,11 +163,10 @@ git: ## Development Testing of this python package can be done using -[`pytest`](https://github.com/pytest-dev/pytest). There is also some additional -testing that is only run as part of TravisCI, as declared in -[`.travis.yml`](.travis.yml). +[`pytest`](https://github.com/pytest-dev/pytest). For more details on the +testing, see [tests/README.md](tests/README.md). -``` +```bash # install chartpress locally pip install -e . diff --git a/chartpress.py b/chartpress.py index 77ea2ec..1dea0c2 100755 --- a/chartpress.py +++ b/chartpress.py @@ -172,7 +172,6 @@ def image_needs_pushing(image): return False -@lru_cache() def image_needs_building(image): """Return whether an image needs building @@ -192,9 +191,11 @@ def image_needs_building(image): # first, check for locally built image try: + print(image) d.images.get(image) except docker.errors.ImageNotFound: # image not found, check registry + print("not found") pass else: # it exists locally, no need to check remote @@ -451,7 +452,9 @@ def build_chart(name, version=None, paths=None, long=False): version = _get_identifier(latest_tag_in_branch, n_commits, chart_commit, long) - chart['version'] = version + if chart['version'] != version: + print(f"Updating {chart_file}: version: {version}") + chart['version'] = version with open(chart_file, 'w') as f: yaml.dump(chart, f) diff --git a/tests/README.md b/tests/README.md index 975cd70..6146844 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,8 +4,9 @@ These tests are [pytest](https://docs.pytest.org) tests. ## References - [pytest](https://docs.pytest.org) + - Fixture: [capfd](https://docs.pytest.org/en/latest/reference.html#_pytest.capture.capfd) + - Fixture: [monkeypatching](https://docs.pytest.org/en/latest/capfd.html) - [pytest-flakes](https://github.com/fschulze/pytest-flakes) and [pyflakes](https://github.com/PyCQA/pyflakes) for passive code check tests - added to the other pytest tests. -- [monkeypatching](https://docs.pytest.org/en/latest/monkeypatch.html) + added to the other pytest tests when using the `--flakes` flag with `pytest`. - [GitPython](https://gitpython.readthedocs.io/en/stable/) diff --git a/tests/data/chartpress.yaml b/tests/data/chartpress.yaml index 92f28c0..b2bf80f 100644 --- a/tests/data/chartpress.yaml +++ b/tests/data/chartpress.yaml @@ -7,6 +7,7 @@ charts: git: jupyterhub/we-are-not-ready-for-this-yet published: https://test.local paths: + - testchart/ - extra-path.txt images: testimage: diff --git a/tests/test_helpers.py b/tests/test_helpers.py index d2955d2..6e1ba12 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -34,6 +34,6 @@ def test_latest_tag_or_mod_commit(git_repo): git_repo.index.add("post-tag-mod.txt") post_tag_commit = git_repo.index.commit("post tag commit") - assert tag_commit.hexsha.startswith(latest_tag_or_mod_commit("chartpress.yaml")) - assert tag_commit.hexsha.startswith(latest_tag_or_mod_commit("tag-mod.txt")) - assert post_tag_commit.hexsha.startswith(latest_tag_or_mod_commit("post-tag-mod.txt")) + assert latest_tag_or_mod_commit("chartpress.yaml") == tag_commit.hexsha[:7] + assert latest_tag_or_mod_commit("tag-mod.txt") == tag_commit.hexsha[:7] + assert latest_tag_or_mod_commit("post-tag-mod.txt") == post_tag_commit.hexsha[:7] diff --git a/tests/test_repo_interactions.py b/tests/test_repo_interactions.py index 6d2e757..18d26b6 100644 --- a/tests/test_repo_interactions.py +++ b/tests/test_repo_interactions.py @@ -9,5 +9,102 @@ def test_git_repo_fixture(git_repo): assert os.path.isfile("testchart/values.yaml") -def test_chartpress_run(git_repo): +def test_chartpress_run(git_repo, capfd): + """Run chartpress and inspect the output.""" + + # run chartpress on initial commit + _, _ = capfd.readouterr() chartpress.main([]) + out, err = capfd.readouterr() + print() + print("--- chartpress ---") + print(out) + print("------------------") + + sha = git_repo.commit("HEAD").hexsha[:7] + tag = f"0.0.1-001.{sha}" + + # verify image was built + # verify the fallback tag of "0.0.1" when a tag is missing + assert f"Successfully tagged testchart/testimage:{tag}" + + # verify the passing of static and dynamic --build-args + assert f"--build-arg TEST_STATIC_BUILD_ARG=test" in out + assert f"--build-arg TEST_DYNAMIC_BUILD_ARG={tag}-{sha}" in out + + # verify updates of Chart.yaml and values.yaml + assert f"Updating testchart/Chart.yaml: version: {tag}" in out + assert f"Updating testchart/values.yaml: image: testchart/testimage:{tag}" in out + assert f"Updating testchart/values.yaml: list.0: testchart/testimage:{tag}" in out + assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:{tag}" in out + + _, _ = capfd.readouterr() + chartpress.main(["--reset"]) + out, err = capfd.readouterr() + print() + print('--- chartpress --reset ---') + print(out) + print("--------------------------") + + # verify usage of chartpress.yaml's resetVersion and resetTag + assert f"Updating testchart/Chart.yaml: version: 0.0.1-test.reset.version" in out + assert f"Updating testchart/values.yaml: image: testchart/testimage:test-reset-tag" in out + assert f"Updating testchart/values.yaml: list.0: testchart/testimage:test-reset-tag" in out + assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:test-reset-tag" in out + + _, _ = capfd.readouterr() + chartpress.main([]) + out, err = capfd.readouterr() + print() + print('--- chartpress ---') + print(out) + print("------------------") + + # verify that we don't need to rebuild the image + assert f"Skipping build" in out + + _, _ = capfd.readouterr() + chartpress.main(["--skip-build", "--tag", "1.2.3-test.tag"]) + out, err = capfd.readouterr() + print() + print('--- chartpress --skip-build --tag 1.2.3-test.tag ---') + print(out) + print("----------------------------------------------------") + + # verify usage --skip-build (this string from docker's output) + assert f"Successfully tagged" not in out + + # verify usage of --tag + assert f"Updating testchart/Chart.yaml: version: 1.2.3-test.tag" in out + assert f"Updating testchart/values.yaml: image: testchart/testimage:1.2.3-test.tag" in out + assert f"Updating testchart/values.yaml: list.0: testchart/testimage:1.2.3-test.tag" in out + assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:1.2.3-test.tag" in out + + # make a git tag and verify it is detected + git_repo.create_tag("1.2.3-test.tag", message="1.2.3-test.tag") + + _, _ = capfd.readouterr() + chartpress.main(["--skip-build"]) + out, err = capfd.readouterr() + print() + print('--- chartpress --skip-build ---') + print(out) + print("-------------------------------") + + # verify detection of git tag by the fact that nothing required updating as + # the previous write to --tag updated Chart.yaml and values.yaml already + assert f"Updating" not in out + + _, _ = capfd.readouterr() + chartpress.main(["--skip-build", "--long"]) + out, err = capfd.readouterr() + print() + print('--- chartpress --skip-build --long ---') + print(out) + print("--------------------------------------") + + # verify usage of --long + assert f"Updating testchart/Chart.yaml: version: 1.2.3-test.tag.000.{sha}" in out + assert f"Updating testchart/values.yaml: image: testchart/testimage:1.2.3-test.tag.000.{sha}" in out + assert f"Updating testchart/values.yaml: list.0: testchart/testimage:1.2.3-test.tag.000.{sha}" in out + assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:1.2.3-test.tag.000.{sha}" in out From b77c849d97bd89195724c856848f5cd6831bfe3e Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 23 Oct 2019 20:30:59 +0200 Subject: [PATCH 10/25] Test --image-prefix and image_needs_pushing --- .travis.yml | 14 +++++++++++++- tests/README.md | 5 +++++ tests/test_helpers.py | 5 +++++ tests/test_repo_interactions.py | 14 ++++++++++++++ 4 files changed, 37 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 0294e5b..16c6894 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,4 +10,16 @@ install: script: - chartpress --version - chartpress --help - - pytest --verbose --flakes + + # This is a workaround to an issue caused by the existence of a docker + # registrymirror in our CI environment. Without this fix that removes the + # mirror, chartpress fails to realize the existence of already built images + # and rebuilds them. + # + # ref: https://github.com/moby/moby/issues/39120 + - |- + echo '{"mtu": 1460}' | sudo dd of=/etc/docker/daemon.json && + sudo systemctl restart docker && + echo "Travis docker registry mirroring disabled.\n\n" + + - pytest --verbose --flakes \ No newline at end of file diff --git a/tests/README.md b/tests/README.md index 6146844..8a6f994 100644 --- a/tests/README.md +++ b/tests/README.md @@ -2,6 +2,11 @@ These tests are [pytest](https://docs.pytest.org) tests. +## Not yet tested +- `--push` +- `--publish-chart` +- `--extra-message` + ## References - [pytest](https://docs.pytest.org) - Fixture: [capfd](https://docs.pytest.org/en/latest/reference.html#_pytest.capture.capfd) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 6e1ba12..a07847c 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,6 +1,7 @@ from chartpress import GITHUB_TOKEN_KEY from chartpress import git_remote +from chartpress import image_needs_pushing from chartpress import latest_tag_or_mod_commit from chartpress import _strip_identifiers_build_suffix from chartpress import _get_identifier @@ -24,6 +25,10 @@ def test_git_remote(monkeypatch): monkeypatch.delenv(GITHUB_TOKEN_KEY) assert git_remote("jupyterhub/helm-chart") == "git@github.com:jupyterhub/helm-chart" +def test_image_needs_pushing(): + assert image_needs_pushing("jupyterhub/image-not-to-be-found:latest") + assert not image_needs_pushing("jupyterhub/k8s-hub:0.8.2") + def test_latest_tag_or_mod_commit(git_repo): open('tag-mod.txt', "w").close() git_repo.index.add("tag-mod.txt") diff --git a/tests/test_repo_interactions.py b/tests/test_repo_interactions.py index 18d26b6..dca75c1 100644 --- a/tests/test_repo_interactions.py +++ b/tests/test_repo_interactions.py @@ -108,3 +108,17 @@ def test_chartpress_run(git_repo, capfd): assert f"Updating testchart/values.yaml: image: testchart/testimage:1.2.3-test.tag.000.{sha}" in out assert f"Updating testchart/values.yaml: list.0: testchart/testimage:1.2.3-test.tag.000.{sha}" in out assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:1.2.3-test.tag.000.{sha}" in out + + _, _ = capfd.readouterr() + chartpress.main(["--skip-build", "--image-prefix", "test-prefix/"]) + out, err = capfd.readouterr() + print() + print('--- chartpress --skip-build --image-prefix test-prefix ---') + print(out) + print("----------------------------------------------------------") + + # verify usage of --image-prefix + assert f"Updating testchart/Chart.yaml: version: 1.2.3-test.tag" in out + assert f"Updating testchart/values.yaml: image: test-prefix/testimage:1.2.3-test.tag" in out + assert f"Updating testchart/values.yaml: list.0: test-prefix/testimage:1.2.3-test.tag" in out + assert f"Updating testchart/values.yaml: list.1.image: test-prefix/testimage:1.2.3-test.tag" in out From d21e6dc716e91bc375e6c375e20a5d8ff21381ad Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 24 Oct 2019 13:34:50 +0200 Subject: [PATCH 11/25] Add back lru_cache --- chartpress.py | 3 ++- tests/test_repo_interactions.py | 5 +++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/chartpress.py b/chartpress.py index 1dea0c2..af5bf84 100755 --- a/chartpress.py +++ b/chartpress.py @@ -56,6 +56,7 @@ def git_remote(git_repo): return 'git@github.com:{0}'.format(git_repo) +@lru_cache() def latest_tag_or_mod_commit(*paths, **kwargs): """ Get the latest of a) the latest tagged commit, or b) the latest modification @@ -171,7 +172,7 @@ def image_needs_pushing(image): else: return False - +@lru_cache() def image_needs_building(image): """Return whether an image needs building diff --git a/tests/test_repo_interactions.py b/tests/test_repo_interactions.py index dca75c1..ee20045 100644 --- a/tests/test_repo_interactions.py +++ b/tests/test_repo_interactions.py @@ -52,6 +52,11 @@ def test_chartpress_run(git_repo, capfd): assert f"Updating testchart/values.yaml: list.0: testchart/testimage:test-reset-tag" in out assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:test-reset-tag" in out + # clear cache of image_needs_building + chartpress.image_needs_building.cache_clear() + # The cache_clear function was provided by the lru_cache decorator + # ref: https://docs.python.org/3/library/functools.html#functools.lru_cache + _, _ = capfd.readouterr() chartpress.main([]) out, err = capfd.readouterr() From f770169dc3cc5a0de41b07ef348670496c7a8675 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Thu, 24 Oct 2019 19:19:31 +0200 Subject: [PATCH 12/25] Apply suggestions from code review Co-Authored-By: Simon Li --- tests/test_repo_interactions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_repo_interactions.py b/tests/test_repo_interactions.py index ee20045..3d7448a 100644 --- a/tests/test_repo_interactions.py +++ b/tests/test_repo_interactions.py @@ -26,7 +26,7 @@ def test_chartpress_run(git_repo, capfd): # verify image was built # verify the fallback tag of "0.0.1" when a tag is missing - assert f"Successfully tagged testchart/testimage:{tag}" + assert f"Successfully tagged testchart/testimage:{tag}" in out # verify the passing of static and dynamic --build-args assert f"--build-arg TEST_STATIC_BUILD_ARG=test" in out From cf7a028114aa30f606e91a68ff1e71e1eb18946e Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Fri, 25 Oct 2019 15:20:14 +0200 Subject: [PATCH 13/25] Add gh-pages branch --- tests/conftest.py | 25 ++++++++++++++++--- .../{data => test_helm_chart}/chartpress.yaml | 0 .../image/Dockerfile | 0 .../testchart/Chart.yaml | 0 .../testchart/templates/configmap.yaml | 7 ++++++ .../testchart/values.yaml | 0 tests/test_helm_chart_repo/index.yaml | 0 7 files changed, 28 insertions(+), 4 deletions(-) rename tests/{data => test_helm_chart}/chartpress.yaml (100%) rename tests/{data => test_helm_chart}/image/Dockerfile (100%) rename tests/{data => test_helm_chart}/testchart/Chart.yaml (100%) create mode 100644 tests/test_helm_chart/testchart/templates/configmap.yaml rename tests/{data => test_helm_chart}/testchart/values.yaml (100%) create mode 100644 tests/test_helm_chart_repo/index.yaml diff --git a/tests/conftest.py b/tests/conftest.py index 656558a..42159a6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,16 +7,33 @@ @pytest.fixture(scope="function") def git_repo(monkeypatch): - """A temporary git repo with """ + """ + This fixture provides a temporary git repo with two branches initialized. + master contains a test helm chart copied from tests/test_helm_chart, and + gh-pages that contains the content of tests/test_helm_chart_repo. + """ with tempfile.TemporaryDirectory() as temp_dir: - # copy content of tests/data folder to the temp dir - copy_tree("tests/data", os.path.join(temp_dir)) + chartpress_dir = os.getcwd() + test_helm_chart_dir = os.path.join(chartpress_dir, "tests/test_helm_chart") + test_helm_chart_repo_dir = os.path.join(chartpress_dir, "tests/test_helm_chart_repo") # enter the directory monkeypatch.chdir(temp_dir) - # initialize the repo and make one initial commit + # initialize the repo r = git.Repo.init(temp_dir) + + # enter blank branch gh-pages + # copy content of tests/test_helm_chart_repo and commit it + r.git.checkout("--orphan", "gh-pages") + copy_tree(test_helm_chart_repo_dir, temp_dir) + r.git.add(all=True) + r.index.commit("initial commit") + + # enter blank branch master + # copy content of tests/test_helm_chart and commit it + r.git.checkout("--orphan", "master") + copy_tree(test_helm_chart_dir, temp_dir) r.git.add(all=True) r.index.commit("initial commit") diff --git a/tests/data/chartpress.yaml b/tests/test_helm_chart/chartpress.yaml similarity index 100% rename from tests/data/chartpress.yaml rename to tests/test_helm_chart/chartpress.yaml diff --git a/tests/data/image/Dockerfile b/tests/test_helm_chart/image/Dockerfile similarity index 100% rename from tests/data/image/Dockerfile rename to tests/test_helm_chart/image/Dockerfile diff --git a/tests/data/testchart/Chart.yaml b/tests/test_helm_chart/testchart/Chart.yaml similarity index 100% rename from tests/data/testchart/Chart.yaml rename to tests/test_helm_chart/testchart/Chart.yaml diff --git a/tests/test_helm_chart/testchart/templates/configmap.yaml b/tests/test_helm_chart/testchart/templates/configmap.yaml new file mode 100644 index 0000000..3f956ac --- /dev/null +++ b/tests/test_helm_chart/testchart/templates/configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +data: + test.json: | + { + "key": "value" + } diff --git a/tests/data/testchart/values.yaml b/tests/test_helm_chart/testchart/values.yaml similarity index 100% rename from tests/data/testchart/values.yaml rename to tests/test_helm_chart/testchart/values.yaml diff --git a/tests/test_helm_chart_repo/index.yaml b/tests/test_helm_chart_repo/index.yaml new file mode 100644 index 0000000..e69de29 From d14c6b8e8bc704b2e2e28019337305035dccac47 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Fri, 25 Oct 2019 15:43:43 +0200 Subject: [PATCH 14/25] Refactor for readability and DRY principle --- tests/test_repo_interactions.py | 115 +++++++++++--------------------- 1 file changed, 40 insertions(+), 75 deletions(-) diff --git a/tests/test_repo_interactions.py b/tests/test_repo_interactions.py index 3d7448a..0fdfc23 100644 --- a/tests/test_repo_interactions.py +++ b/tests/test_repo_interactions.py @@ -12,17 +12,12 @@ def test_git_repo_fixture(git_repo): def test_chartpress_run(git_repo, capfd): """Run chartpress and inspect the output.""" - # run chartpress on initial commit - _, _ = capfd.readouterr() - chartpress.main([]) - out, err = capfd.readouterr() - print() - print("--- chartpress ---") - print(out) - print("------------------") - + # summarize information from git_repo sha = git_repo.commit("HEAD").hexsha[:7] tag = f"0.0.1-001.{sha}" + + # run chartpress + out = _capture_output([], capfd) # verify image was built # verify the fallback tag of "0.0.1" when a tag is missing @@ -38,13 +33,7 @@ def test_chartpress_run(git_repo, capfd): assert f"Updating testchart/values.yaml: list.0: testchart/testimage:{tag}" in out assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:{tag}" in out - _, _ = capfd.readouterr() - chartpress.main(["--reset"]) - out, err = capfd.readouterr() - print() - print('--- chartpress --reset ---') - print(out) - print("--------------------------") + out = _capture_output(["--reset"], capfd) # verify usage of chartpress.yaml's resetVersion and resetTag assert f"Updating testchart/Chart.yaml: version: 0.0.1-test.reset.version" in out @@ -53,77 +42,53 @@ def test_chartpress_run(git_repo, capfd): assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:test-reset-tag" in out # clear cache of image_needs_building - chartpress.image_needs_building.cache_clear() - # The cache_clear function was provided by the lru_cache decorator # ref: https://docs.python.org/3/library/functools.html#functools.lru_cache - - _, _ = capfd.readouterr() - chartpress.main([]) - out, err = capfd.readouterr() - print() - print('--- chartpress ---') - print(out) - print("------------------") + chartpress.image_needs_building.cache_clear() # verify that we don't need to rebuild the image + out = _capture_output([], capfd) assert f"Skipping build" in out - _, _ = capfd.readouterr() - chartpress.main(["--skip-build", "--tag", "1.2.3-test.tag"]) - out, err = capfd.readouterr() - print() - print('--- chartpress --skip-build --tag 1.2.3-test.tag ---') - print(out) - print("----------------------------------------------------") - - # verify usage --skip-build (this string from docker's output) + # verify usage --skip-build and --tag + tag = "1.2.3-test.tag" + out = _capture_output(["--skip-build", "--tag", tag], capfd) assert f"Successfully tagged" not in out + assert f"Updating testchart/Chart.yaml: version: {tag}" in out + assert f"Updating testchart/values.yaml: image: testchart/testimage:{tag}" in out + assert f"Updating testchart/values.yaml: list.0: testchart/testimage:{tag}" in out + assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:{tag}" in out - # verify usage of --tag - assert f"Updating testchart/Chart.yaml: version: 1.2.3-test.tag" in out - assert f"Updating testchart/values.yaml: image: testchart/testimage:1.2.3-test.tag" in out - assert f"Updating testchart/values.yaml: list.0: testchart/testimage:1.2.3-test.tag" in out - assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:1.2.3-test.tag" in out - - # make a git tag and verify it is detected - git_repo.create_tag("1.2.3-test.tag", message="1.2.3-test.tag") - - _, _ = capfd.readouterr() - chartpress.main(["--skip-build"]) - out, err = capfd.readouterr() - print() - print('--- chartpress --skip-build ---') - print(out) - print("-------------------------------") - - # verify detection of git tag by the fact that nothing required updating as - # the previous write to --tag updated Chart.yaml and values.yaml already + # verify a real git tag is detected + git_repo.create_tag(tag, message=tag) + out = _capture_output(["--skip-build"], capfd) + # This assertion verifies chartpress has considered the git tag by the fact + # that no values required to be updated. No values should be updated as the + # previous call with a provided --tag updated Chart.yaml and values.yaml + # already. assert f"Updating" not in out - _, _ = capfd.readouterr() - chartpress.main(["--skip-build", "--long"]) - out, err = capfd.readouterr() - print() - print('--- chartpress --skip-build --long ---') - print(out) - print("--------------------------------------") - # verify usage of --long - assert f"Updating testchart/Chart.yaml: version: 1.2.3-test.tag.000.{sha}" in out - assert f"Updating testchart/values.yaml: image: testchart/testimage:1.2.3-test.tag.000.{sha}" in out - assert f"Updating testchart/values.yaml: list.0: testchart/testimage:1.2.3-test.tag.000.{sha}" in out - assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:1.2.3-test.tag.000.{sha}" in out + out = _capture_output(["--skip-build", "--long"], capfd) + assert f"Updating testchart/Chart.yaml: version: {tag}.000.{sha}" in out + assert f"Updating testchart/values.yaml: image: testchart/testimage:{tag}.000.{sha}" in out + assert f"Updating testchart/values.yaml: list.0: testchart/testimage:{tag}.000.{sha}" in out + assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:{tag}.000.{sha}" in out + # verify usage of --image-prefix + out = _capture_output(["--skip-build", "--image-prefix", "test-prefix/"], capfd) + assert f"Updating testchart/Chart.yaml: version: {tag}" in out + assert f"Updating testchart/values.yaml: image: test-prefix/testimage:{tag}" in out + assert f"Updating testchart/values.yaml: list.0: test-prefix/testimage:{tag}" in out + assert f"Updating testchart/values.yaml: list.1.image: test-prefix/testimage:{tag}" in out + +def _capture_output(args, capfd): _, _ = capfd.readouterr() - chartpress.main(["--skip-build", "--image-prefix", "test-prefix/"]) + chartpress.main(args) out, err = capfd.readouterr() + header = f'--- chartpress {" ".join(args)} ---' + footer = "-" * len(header) print() - print('--- chartpress --skip-build --image-prefix test-prefix ---') + print(header) print(out) - print("----------------------------------------------------------") - - # verify usage of --image-prefix - assert f"Updating testchart/Chart.yaml: version: 1.2.3-test.tag" in out - assert f"Updating testchart/values.yaml: image: test-prefix/testimage:1.2.3-test.tag" in out - assert f"Updating testchart/values.yaml: list.0: test-prefix/testimage:1.2.3-test.tag" in out - assert f"Updating testchart/values.yaml: list.1.image: test-prefix/testimage:1.2.3-test.tag" in out + print(footer) + return out From 56e8dd075007b99d58123f75a5804ed29e0383f7 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Fri, 25 Oct 2019 15:48:25 +0200 Subject: [PATCH 15/25] Test successful gh-pages branch setup --- tests/test_repo_interactions.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_repo_interactions.py b/tests/test_repo_interactions.py index 0fdfc23..439a41d 100644 --- a/tests/test_repo_interactions.py +++ b/tests/test_repo_interactions.py @@ -3,11 +3,18 @@ import chartpress def test_git_repo_fixture(git_repo): + # assert we use the git repo as our current working directory assert git_repo.working_dir == os.getcwd() + + # assert we have copied files assert os.path.isfile("chartpress.yaml") assert os.path.isfile("testchart/Chart.yaml") assert os.path.isfile("testchart/values.yaml") + # assert there is another branch to contain published content as well + git_repo.git.checkout("gh-pages") + assert os.path.isfile("index.yaml") + def test_chartpress_run(git_repo, capfd): """Run chartpress and inspect the output.""" @@ -81,6 +88,7 @@ def test_chartpress_run(git_repo, capfd): assert f"Updating testchart/values.yaml: list.0: test-prefix/testimage:{tag}" in out assert f"Updating testchart/values.yaml: list.1.image: test-prefix/testimage:{tag}" in out + def _capture_output(args, capfd): _, _ = capfd.readouterr() chartpress.main(args) From 40cb6f2c2476c7ffe4eef1d2cc4efe75cb5a8f58 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Fri, 25 Oct 2019 18:08:58 +0200 Subject: [PATCH 16/25] Test --publish-chart and --extra-message --- chartpress.py | 10 +++- tests/README.md | 2 - tests/test_helm_chart/chartpress.yaml | 2 +- tests/test_helm_chart_repo/index.yaml | 19 +++++++ tests/test_repo_interactions.py | 75 ++++++++++++++++++++++++++- 5 files changed, 102 insertions(+), 6 deletions(-) diff --git a/chartpress.py b/chartpress.py index af5bf84..f61479a 100755 --- a/chartpress.py +++ b/chartpress.py @@ -49,6 +49,12 @@ def git_remote(git_repo): Depending on the system setup it returns ssh or https remote. """ + # if not matching something/something + # such as a local directory ".", then + # simply return this unmodified. + if not re.match(r'^[^/]+/[^/]+$', git_repo): + return git_repo + github_token = os.getenv(GITHUB_TOKEN_KEY) if github_token: return 'https://{0}@github.com/{1}'.format( @@ -475,9 +481,9 @@ def publish_pages(chart_name, chart_version, chart_repo_github_path, chart_repo_ checkout_dir, ], # warning: if echoed, this call could reveal the github token - echo=False, + echo=True, ) - check_call(['git', 'checkout', 'gh-pages'], cwd=checkout_dir) + check_call(['git', 'checkout', 'gh-pages'], cwd=checkout_dir, echo=True) # package the latest version into a temporary directory # and run helm repo index with --merge to update index.yaml diff --git a/tests/README.md b/tests/README.md index 8a6f994..0ebbe71 100644 --- a/tests/README.md +++ b/tests/README.md @@ -4,8 +4,6 @@ These tests are [pytest](https://docs.pytest.org) tests. ## Not yet tested - `--push` -- `--publish-chart` -- `--extra-message` ## References - [pytest](https://docs.pytest.org) diff --git a/tests/test_helm_chart/chartpress.yaml b/tests/test_helm_chart/chartpress.yaml index b2bf80f..00b1d6b 100644 --- a/tests/test_helm_chart/chartpress.yaml +++ b/tests/test_helm_chart/chartpress.yaml @@ -4,7 +4,7 @@ charts: resetTag: test-reset-tag resetVersion: 0.0.1-test.reset.version repo: - git: jupyterhub/we-are-not-ready-for-this-yet + git: "." published: https://test.local paths: - testchart/ diff --git a/tests/test_helm_chart_repo/index.yaml b/tests/test_helm_chart_repo/index.yaml index e69de29..fdba4d4 100644 --- a/tests/test_helm_chart_repo/index.yaml +++ b/tests/test_helm_chart_repo/index.yaml @@ -0,0 +1,19 @@ +# https://helm.sh/docs/chart_repository/#the-index-file +apiVersion: v1 +entries: + testchart: + - apiVersion: v1 + created: 2019-01-01T00:00:00.000000000Z + digest: bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb + name: testchart + urls: + - https://test.local/testchart-1.2.2.tgz + version: 1.2.2 + - apiVersion: v1 + created: 2019-02-01T00:00:00.000000000Z + digest: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + name: testchart + urls: + - https://test.local/testchart-1.2.1.tgz + version: 1.2.1 +generated: 2019-02-01T00:00:00.000000000Z diff --git a/tests/test_repo_interactions.py b/tests/test_repo_interactions.py index 439a41d..976131a 100644 --- a/tests/test_repo_interactions.py +++ b/tests/test_repo_interactions.py @@ -40,22 +40,25 @@ def test_chartpress_run(git_repo, capfd): assert f"Updating testchart/values.yaml: list.0: testchart/testimage:{tag}" in out assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:{tag}" in out - out = _capture_output(["--reset"], capfd) # verify usage of chartpress.yaml's resetVersion and resetTag + out = _capture_output(["--reset"], capfd) assert f"Updating testchart/Chart.yaml: version: 0.0.1-test.reset.version" in out assert f"Updating testchart/values.yaml: image: testchart/testimage:test-reset-tag" in out assert f"Updating testchart/values.yaml: list.0: testchart/testimage:test-reset-tag" in out assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:test-reset-tag" in out + # clear cache of image_needs_building # ref: https://docs.python.org/3/library/functools.html#functools.lru_cache chartpress.image_needs_building.cache_clear() + # verify that we don't need to rebuild the image out = _capture_output([], capfd) assert f"Skipping build" in out + # verify usage --skip-build and --tag tag = "1.2.3-test.tag" out = _capture_output(["--skip-build", "--tag", tag], capfd) @@ -65,6 +68,7 @@ def test_chartpress_run(git_repo, capfd): assert f"Updating testchart/values.yaml: list.0: testchart/testimage:{tag}" in out assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:{tag}" in out + # verify a real git tag is detected git_repo.create_tag(tag, message=tag) out = _capture_output(["--skip-build"], capfd) @@ -74,6 +78,7 @@ def test_chartpress_run(git_repo, capfd): # already. assert f"Updating" not in out + # verify usage of --long out = _capture_output(["--skip-build", "--long"], capfd) assert f"Updating testchart/Chart.yaml: version: {tag}.000.{sha}" in out @@ -81,6 +86,7 @@ def test_chartpress_run(git_repo, capfd): assert f"Updating testchart/values.yaml: list.0: testchart/testimage:{tag}.000.{sha}" in out assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:{tag}.000.{sha}" in out + # verify usage of --image-prefix out = _capture_output(["--skip-build", "--image-prefix", "test-prefix/"], capfd) assert f"Updating testchart/Chart.yaml: version: {tag}" in out @@ -89,6 +95,73 @@ def test_chartpress_run(git_repo, capfd): assert f"Updating testchart/values.yaml: list.1.image: test-prefix/testimage:{tag}" in out + # verify usage of --publish-chart and --extra-message + out = _capture_output( + [ + "--skip-build", + "--publish-chart", + "--extra-message", "test added --extra-message", + ], + capfd, + ) + # verify output of --publish-chart + assert "Branch 'gh-pages' set up to track remote branch 'gh-pages' from 'origin'." in out + assert "Successfully packaged chart and saved it to:" in out + assert f"/testchart-{tag}.tgz" in out + + git_repo.git.stash() + git_repo.git.checkout("gh-pages") + + # verify result of --publish-chart + with open("index.yaml", "r") as f: + index_yaml = f.read() + print(index_yaml) + assert f"version: 1.2.1" in index_yaml + assert f"version: 1.2.2" in index_yaml + assert f"version: {tag}" in index_yaml + + # verify result of --extra-message + automatic_helm_chart_repo_commit = git_repo.commit("HEAD") + assert "test added --extra-message" in automatic_helm_chart_repo_commit.message + + git_repo.git.checkout("master") + git_repo.git.stash("pop") + + + # verify we don't overwrite the previous version when we make dev commits + # and use --publish-chart + open("extra-path.txt", "w").close() + git_repo.git.add(all=True) + sha = git_repo.index.commit("Added extra-path.txt").hexsha[:7] + chartpress.latest_tag_or_mod_commit.cache_clear() + out = _capture_output( + [ + "--skip-build", + "--publish-chart", + ], + capfd, + ) + # verify output of --publish-chart + assert "Branch 'gh-pages' set up to track remote branch 'gh-pages' from 'origin'." in out + assert "Successfully packaged chart and saved it to:" in out + assert f"/testchart-{tag}.001.{sha}.tgz" in out + + git_repo.git.stash() + git_repo.git.checkout("gh-pages") + + # verify result of --publish-chart + with open("index.yaml", "r") as f: + index_yaml = f.read() + print(index_yaml) + assert f"version: 1.2.1" in index_yaml + assert f"version: 1.2.2" in index_yaml + assert f"version: {tag}" in index_yaml + assert f"version: {tag}.001.{sha}" in index_yaml + + git_repo.git.checkout("master") + git_repo.git.stash("pop") + + def _capture_output(args, capfd): _, _ = capfd.readouterr() chartpress.main(args) From 284e53bac1f38c1aaa27a80a76c3edacbe500f96 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Fri, 25 Oct 2019 18:18:50 +0200 Subject: [PATCH 17/25] Install and initialize helm on TravisCI --- .travis.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 16c6894..463367d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,10 +7,10 @@ install: - pip install --upgrade pip - pip install . - pip install -r dev-requirements.txt -script: - - chartpress --version - - chartpress --help + - curl -L https://git.io/get_helm.sh | bash + - helm init --client-only +script: # This is a workaround to an issue caused by the existence of a docker # registrymirror in our CI environment. Without this fix that removes the # mirror, chartpress fails to realize the existence of already built images @@ -21,5 +21,6 @@ script: echo '{"mtu": 1460}' | sudo dd of=/etc/docker/daemon.json && sudo systemctl restart docker && echo "Travis docker registry mirroring disabled.\n\n" - - - pytest --verbose --flakes \ No newline at end of file + - chartpress --version + - chartpress --help + - pytest --verbose --flakes From 6ccf4303442bb071e3c7c93fee731a1fb749fd65 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Fri, 25 Oct 2019 20:11:37 +0200 Subject: [PATCH 18/25] Cleanup print statements --- chartpress.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/chartpress.py b/chartpress.py index f61479a..174c21b 100755 --- a/chartpress.py +++ b/chartpress.py @@ -198,11 +198,9 @@ def image_needs_building(image): # first, check for locally built image try: - print(image) d.images.get(image) except docker.errors.ImageNotFound: # image not found, check registry - print("not found") pass else: # it exists locally, no need to check remote From ad34f144b561f7b5ee09948a70117fe467c8306e Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 26 Oct 2019 12:48:30 +0200 Subject: [PATCH 19/25] Add tests for paths configuration --- chartpress.py | 2 + tests/test_helm_chart/chartpress.yaml | 5 +-- tests/test_repo_interactions.py | 56 ++++++++++++++++++++++++++- 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/chartpress.py b/chartpress.py index 174c21b..d90a6e4 100755 --- a/chartpress.py +++ b/chartpress.py @@ -607,6 +607,8 @@ def main(args=None): # - build values.yaml (--reset) # - push chart (--publish-chart, --extra-message) for chart in config['charts']: + # FIXME: Ensure that '.' means the chart path itself. I think it referrs to anything. + # FIXME: Ensure that a changed image leads to a changed chart chart_paths = ['.'] + list(chart.get('paths', [])) chart_version = args.tag diff --git a/tests/test_helm_chart/chartpress.yaml b/tests/test_helm_chart/chartpress.yaml index 00b1d6b..403aa7b 100644 --- a/tests/test_helm_chart/chartpress.yaml +++ b/tests/test_helm_chart/chartpress.yaml @@ -7,8 +7,7 @@ charts: git: "." published: https://test.local paths: - - testchart/ - - extra-path.txt + - extra-chart-path.txt images: testimage: buildArgs: @@ -21,4 +20,4 @@ charts: - list.0 - list.1.image paths: - - extra-path.txt + - extra-image-path.txt diff --git a/tests/test_repo_interactions.py b/tests/test_repo_interactions.py index 976131a..b231559 100644 --- a/tests/test_repo_interactions.py +++ b/tests/test_repo_interactions.py @@ -130,9 +130,9 @@ def test_chartpress_run(git_repo, capfd): # verify we don't overwrite the previous version when we make dev commits # and use --publish-chart - open("extra-path.txt", "w").close() + open("extra-chart-path.txt", "w").close() git_repo.git.add(all=True) - sha = git_repo.index.commit("Added extra-path.txt").hexsha[:7] + sha = git_repo.index.commit("Added extra-chart-path.txt").hexsha[:7] chartpress.latest_tag_or_mod_commit.cache_clear() out = _capture_output( [ @@ -162,6 +162,58 @@ def test_chartpress_run(git_repo, capfd): git_repo.git.stash("pop") +def test_chartpress_paths_configuration(git_repo, capfd): + """ + Background: + - A helm chart resides in a directory with the name of the chart. This kind + of directory will contain Chart.yaml, values.yaml, and the templates + folder for example. + - chartpress.yaml is meant to reside in a parent directory to helm chart + directories. + + By default, the Chart.yaml's version field should only be updated if + anything within its folder has changed, but also if any of the chart's + images are being updated when running chartpress. + + This default can be augmented by paths configuration in chartpress.yaml. + These additional paths are relative to the chartpress.yaml location, which + must be in the parent folder of the charts. + + This test verifies the statements above. + """ + + # Add a file outside the chart repo and verify chartpress didn't update the + # Chart.yaml version or the image tags in values.yaml. + open("not-in-paths.txt", "w").close() + git_repo.git.add(all=True) + sha = git_repo.index.commit("Added not-in-paths.txt").hexsha[:7] + tag = f"0.0.1-002.{sha}" + out = _capture_output(["--skip-build"], capfd) + assert f"Updating testchart/Chart.yaml: version: {tag}" not in out + assert f"Updating testchart/values.yaml: image: testchart/testimage: {tag}" not in out + + # Add a file specified in the chart's paths configuration and verify + # chartpress updated the Chart.yaml version, but not the image tags in + # values.yaml. + open("extra-chart-path.txt", "w").close() + git_repo.git.add(all=True) + sha = git_repo.index.commit("Added extra-chart-path.txt").hexsha[:7] + tag = f"0.0.1-003.{sha}" + out = _capture_output(["--skip-build"], capfd) + assert f"Updating testchart/Chart.yaml: version: {tag}" in out + assert f"Updating testchart/values.yaml: image: testchart/testimage: {tag}" not in out + + # Add a file specified in a chart image's paths configuration and verify + # updates to Chart.yaml version as well as the image tags in values.yaml. + open("extra-image-path.txt", "w").close() + git_repo.git.add(all=True) + sha = git_repo.index.commit("Added extra-image-path.txt").hexsha[:7] + tag = f"0.0.1-004.{sha}" + out = _capture_output(["--skip-build"], capfd) + assert f"Updating testchart/Chart.yaml: version: {tag}" in out + assert f"Updating testchart/values.yaml: image: testchart/testimage: {tag}" in out + + def _capture_output(args, capfd): _, _ = capfd.readouterr() chartpress.main(args) From bcb4890187f8607dd0a2e120d26cc949cc451e9e Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Sat, 26 Oct 2019 15:08:59 +0200 Subject: [PATCH 20/25] Reworked documentation about paths --- README.md | 31 +++++++++++++++++++------------ chartpress.py | 25 +++++++++++++++---------- 2 files changed, 34 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 26873e6..e417bf3 100644 --- a/README.md +++ b/README.md @@ -99,12 +99,16 @@ charts: # --reset flag. It defaults to "0.0.1-set.by.chartpress". This is a valid # SemVer 2 version, which is required for a helm lint command to succeed. resetVersion: 1.2.3 - # the git repo whose gh-pages contains the charts + # The git repo whose gh-pages contains the charts. This can be a local + # path such as "." as well but if matching / will be + # assumed to be a separate GitHub repository. repo: git: jupyterhub/helm-chart published: https://jupyterhub.github.io/helm-chart - # additional paths (if any) relevant to the chart version - # outside the chart directory itself + # Additional paths that when modified should lead to an updated Chart.yaml + # version, other than the chart directory in or any path that + # influence the images of the chart. These paths should be set relative to + # chartpress.yaml's directory. paths: - ../setup.py - ../binderhub @@ -117,21 +121,24 @@ charts: buildArgs: MY_STATIC_BUILD_ARG: "hello world" MY_DYNAMIC_BUILD_ARG: "{TAG}-{LAST_COMMIT}" - # Context to send to docker build for use by the Dockerfile - # (if different from the current directory) + # contextPath is the path to the directory that is to be considered the + # current working directory during the build process of the Dockerfile. + # This is by default the folder of the Dockerfile. This path should be + # set relative to chartpress.yaml. contextPath: .. - # Dockerfile path, if different from the default - # (may be needed if contextPath is set) + # Path to the Dockerfile, relative to chartpress.yaml. Defaults to + # "images//Dockerfile". dockerfilePath: images/binderhub/Dockerfile - # path(s) in values.yaml to be updated with image name and tag + # Path(s) in /values.yaml to be updated with image name and + # tag. valuesPath: - singleuser.image - singleuser.profileList.0.kubespawner_override.image - # additional paths (if any) relevant to the image - # outside the image directory itself + # Additional paths, relative to chartpress.yaml's directory, that should + # be used to indicate that a new tag of the image is required, aside + # from the contextPath and dockerfilePath for building the image itself. paths: - - ../setup.py - - ../binderhub + - assets ``` ## Caveats diff --git a/chartpress.py b/chartpress.py index d90a6e4..2a855c4 100755 --- a/chartpress.py +++ b/chartpress.py @@ -131,18 +131,23 @@ def render_build_args(image_options, ns): return build_args -def build_image(image_path, image_name, build_args=None, dockerfile_path=None): +def build_image(image_name, context_path, dockerfile_path=None, build_args=None): """Build an image Args: - image_path (str): the path to the image directory - image_name (str): image 'name:tag' to build - build_args (dict, optional): dict of docker build arguments + image_name (str): + The image name formatted as 'name:tag'. + context_path (str): + The path to the directory that is to be considered the current working + directory during the build process of the Dockerfile. This is typically + the same folder as the Dockerfile resides in. dockerfile_path (str, optional): - path to dockerfile relative to image_path - if not `image_path/Dockerfile`. + Path to dockerfile relative to context_path if not + `context_path/Dockerfile`. + build_args (dict, optional): + Dictionary of docker build arguments. """ - cmd = ['docker', 'build', '-t', image_name, image_path] + cmd = ['docker', 'build', '-t', image_name, context_path] if dockerfile_path: cmd.extend(['-f', dockerfile_path]) @@ -291,12 +296,12 @@ def build_images(prefix, images, tag=None, push=False, chart_version=None, skip_ """ value_modifications = {} for name, options in images.items(): - image_path = options.get('contextPath', os.path.join('images', name)) + context_path = options.get('contextPath', os.path.join('images', name)) image_tag = tag chart_version = _strip_identifiers_build_suffix(chart_version) # include chartpress.yaml itself as it can contain build args and # similar that influence the image that would be built - paths = list(options.get('paths', [])) + [image_path, 'chartpress.yaml'] + paths = list(options.get('paths', [])) + [context_path, 'chartpress.yaml'] image_commit = latest_tag_or_mod_commit(*paths, echo=False) if image_tag is None: n_commits = check_output( @@ -337,7 +342,7 @@ def build_images(prefix, images, tag=None, push=False, chart_version=None, skip_ 'TAG': image_tag, }, ) - build_image(image_path, image_spec, build_args, options.get('dockerfilePath')) + build_image(context_path, image_spec, build_args, options.get('dockerfilePath')) else: print(f"Skipping build for {image_spec}, it already exists") From 116e5bf8c7479df1c6dc7623a6cc4473e4fc8350 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 28 Oct 2019 11:20:14 +0100 Subject: [PATCH 21/25] Test render_build_args --- tests/test_helpers.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index a07847c..47fb9e9 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -42,3 +42,20 @@ def test_latest_tag_or_mod_commit(git_repo): assert latest_tag_or_mod_commit("chartpress.yaml") == tag_commit.hexsha[:7] assert latest_tag_or_mod_commit("tag-mod.txt") == tag_commit.hexsha[:7] assert latest_tag_or_mod_commit("post-tag-mod.txt") == post_tag_commit.hexsha[:7] + +def test_render_build_args(git_repo): + with open('chartpress.yaml') as f: + config = yaml.load(f) + for chart in config["charts"]: + for name, options in chart["images"].items(): + build_args = render_build_args( + options, + { + 'LAST_COMMIT': "sha", + 'TAG': "tag", + }, + ) + assert build_args == { + 'TEST_STATIC_BUILD_ARG': 'test', + 'TEST_DYNAMIC_BUILD_ARG': 'tag-sha', + } From 655223ec95ae841cc3eed56e9e5cdc2a5a2508ff Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 28 Oct 2019 11:20:18 +0100 Subject: [PATCH 22/25] Bugfixes relating to usage of chart/image paths - Bumping chart version on changes to chartpress.yaml, the helm chart folder, its dependent images, and optionally additional paths. Previous behavior was to bump chart version on any change to the folder where chartpress.yaml resided and optionally additional paths. - Bumping image version on change to Dockerfile residing outside of image build context should now work. --- chartpress.py | 92 ++++++++++++++++++--------- tests/README.md | 36 +++++++++++ tests/test_helm_chart/chartpress.yaml | 2 +- tests/test_helpers.py | 7 ++ tests/test_repo_interactions.py | 20 +++--- 5 files changed, 118 insertions(+), 39 deletions(-) diff --git a/chartpress.py b/chartpress.py index 2a855c4..63d9403 100755 --- a/chartpress.py +++ b/chartpress.py @@ -131,26 +131,52 @@ def render_build_args(image_options, ns): return build_args -def build_image(image_name, context_path, dockerfile_path=None, build_args=None): +def get_image_paths(name, options): + """ + Returns the paths that when changed should trigger a rebuild of a chart's + image. This includes the Dockerfile itself and the context of the Dockerfile + during the build process. + + The first element will always be the context path, the second always the + Dockerfile path, and the optional others for extra paths. + """ + r = [] + if options.get("contextPath"): + r.append(options["contextPath"]) + else: + r.append(os.path.join("images", name)) + if options.get("dockerfilePath"): + r.append(options["dockerfilePath"]) + else: + r.append(os.path.join(r[0], "Dockerfile")) + r.extend(options.get("paths", [])) + return r + +def build_image(image_spec, context_path, dockerfile_path=None, build_args=None): """Build an image Args: - image_name (str): - The image name formatted as 'name:tag'. + image_spec (str): + The image name, including tag, and optionally a registry prefix for + other image registries than DockerHub. + + Examples: + - jupyterhub/k8s-hub:0.9.0 + - index.docker.io/library/ubuntu:latest + - eu.gcr.io/my-gcp-project/my-image:0.1.0 context_path (str): The path to the directory that is to be considered the current working directory during the build process of the Dockerfile. This is typically the same folder as the Dockerfile resides in. dockerfile_path (str, optional): - Path to dockerfile relative to context_path if not - `context_path/Dockerfile`. + Path to Dockerfile relative to chartpress.yaml's directory if not + "/Dockerfile". build_args (dict, optional): Dictionary of docker build arguments. """ - cmd = ['docker', 'build', '-t', image_name, context_path] + cmd = ['docker', 'build', '-t', image_spec, context_path] if dockerfile_path: cmd.extend(['-f', dockerfile_path]) - for k, v in (build_args or {}).items(): cmd += ['--build-arg', '{}={}'.format(k, v)] check_call(cmd) @@ -163,16 +189,19 @@ def docker_client(): @lru_cache() def image_needs_pushing(image): - """Return whether an image needs pushing + """ + Returns a boolean whether an image needs pushing by checking if the image + exists on the image registry. Args: - image (str): the `repository:tag` image to be build. - - Returns: + image (str): + The name of an image to be passed to the docker push command. - True: if image needs to be pushed (not on registry) - False: if not (already present on registry) + Examples: + - jupyterhub/k8s-hub:0.9.0 + - index.docker.io/library/ubuntu:latest + - eu.gcr.io/my-gcp-project/my-image:0.1.0 """ d = docker_client() try: @@ -185,19 +214,19 @@ def image_needs_pushing(image): @lru_cache() def image_needs_building(image): - """Return whether an image needs building - - Checks if the image exists (ignores commit range), - either locally or on the registry. + """ + Returns a boolean whether an image needs building by checking if the image + exists either locally or on the registry. Args: - image (str): the `repository:tag` image to be build. - - Returns: + image (str): + The name of an image to be passed to the docker build command. - True: if image needs to be built - False: if not (image already exists) + Examples: + - jupyterhub/k8s-hub:0.9.0 + - index.docker.io/library/ubuntu:latest + - eu.gcr.io/my-gcp-project/my-image:0.1.0 """ d = docker_client() @@ -296,13 +325,14 @@ def build_images(prefix, images, tag=None, push=False, chart_version=None, skip_ """ value_modifications = {} for name, options in images.items(): - context_path = options.get('contextPath', os.path.join('images', name)) image_tag = tag chart_version = _strip_identifiers_build_suffix(chart_version) # include chartpress.yaml itself as it can contain build args and # similar that influence the image that would be built - paths = list(options.get('paths', [])) + [context_path, 'chartpress.yaml'] - image_commit = latest_tag_or_mod_commit(*paths, echo=False) + image_paths = get_image_paths(name, options) + ["chartpress.yaml"] + context_path = image_paths[0] + dockerfile_path = image_paths[1] + image_commit = latest_tag_or_mod_commit(*image_paths, echo=False) if image_tag is None: n_commits = check_output( [ @@ -342,7 +372,7 @@ def build_images(prefix, images, tag=None, push=False, chart_version=None, skip_ 'TAG': image_tag, }, ) - build_image(context_path, image_spec, build_args, options.get('dockerfilePath')) + build_image(image_spec, context_path, dockerfile_path=dockerfile_path, build_args=build_args) else: print(f"Skipping build for {image_spec}, it already exists") @@ -612,9 +642,13 @@ def main(args=None): # - build values.yaml (--reset) # - push chart (--publish-chart, --extra-message) for chart in config['charts']: - # FIXME: Ensure that '.' means the chart path itself. I think it referrs to anything. - # FIXME: Ensure that a changed image leads to a changed chart - chart_paths = ['.'] + list(chart.get('paths', [])) + chart_paths = [] + chart_paths.append("chartpress.yaml") + chart_paths.append(chart['name']) + chart_paths.extend(chart.get('paths', [])) + for image_name, image_config in chart['images'].items(): + chart_paths.extend(get_image_paths(image_name, image_config)) + chart_paths = list(set(chart_paths)) chart_version = args.tag if chart_version: diff --git a/tests/README.md b/tests/README.md index 0ebbe71..b9278ca 100644 --- a/tests/README.md +++ b/tests/README.md @@ -2,6 +2,42 @@ These tests are [pytest](https://docs.pytest.org) tests. +useful flags: + + +```bash +pytest -vx --flakes + +# -v / --verbose: +# allows you to see the names etc of individual tests etc. +# -x / --exitfirst: +# to stop running tests of first error +# --flakes: +# use pyflakes to add some static code analysis tests +# -k: +# only run tests which match the given substring +``` + +## Developing tests +Keep your eye out for test issues caused by caching! The +[`@lru_cache()`](https://docs.python.org/3/library/functools.html#functools.lru_cache) +decorator will do in memory cache that affects us when we run multiple commands +to chartpress in the same process. This decorator also adds the `cache_clear()` +function to the function object. + +It is very possible to run into issues with caching on `image_needs_building`, +`image_needs_pushing` and `latest_tag_or_mod_commit`. + +```python +# clearing the cache of these functions is important if you run chartpress +# without --skip-build +chartpress.image_needs_building.cache_clear() +chartpress.image_needs_pushing.cache_clear() + +# clearing the cache of this function is important very often +chartpress.latest_tag_or_mod_commit.cache_clear() +``` + ## Not yet tested - `--push` diff --git a/tests/test_helm_chart/chartpress.yaml b/tests/test_helm_chart/chartpress.yaml index 403aa7b..468dc51 100644 --- a/tests/test_helm_chart/chartpress.yaml +++ b/tests/test_helm_chart/chartpress.yaml @@ -13,7 +13,7 @@ charts: buildArgs: TEST_STATIC_BUILD_ARG: "test" TEST_DYNAMIC_BUILD_ARG: "{TAG}-{LAST_COMMIT}" - contextPath: . + contextPath: image dockerfilePath: image/Dockerfile valuesPath: - image diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 47fb9e9..2549f41 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -3,9 +3,16 @@ from chartpress import git_remote from chartpress import image_needs_pushing from chartpress import latest_tag_or_mod_commit +from chartpress import render_build_args from chartpress import _strip_identifiers_build_suffix from chartpress import _get_identifier +from ruamel.yaml import YAML +# use safe roundtrip yaml loader +yaml = YAML(typ='rt') +yaml.preserve_quotes = True ## avoid mangling of quotes +yaml.indent(mapping=2, offset=2, sequence=4) + def test__strip_identifiers_build_suffix(): assert _strip_identifiers_build_suffix(identifier="0.1.2-005.asdf1234") == "0.1.2" assert _strip_identifiers_build_suffix(identifier="0.1.2-alpha.1.005.asdf1234") == "0.1.2-alpha.1" diff --git a/tests/test_repo_interactions.py b/tests/test_repo_interactions.py index b231559..80f1ebd 100644 --- a/tests/test_repo_interactions.py +++ b/tests/test_repo_interactions.py @@ -24,6 +24,7 @@ def test_chartpress_run(git_repo, capfd): tag = f"0.0.1-001.{sha}" # run chartpress + chartpress.image_needs_building.cache_clear() out = _capture_output([], capfd) # verify image was built @@ -48,13 +49,8 @@ def test_chartpress_run(git_repo, capfd): assert f"Updating testchart/values.yaml: list.0: testchart/testimage:test-reset-tag" in out assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:test-reset-tag" in out - - # clear cache of image_needs_building - # ref: https://docs.python.org/3/library/functools.html#functools.lru_cache - chartpress.image_needs_building.cache_clear() - - # verify that we don't need to rebuild the image + chartpress.image_needs_building.cache_clear() out = _capture_output([], capfd) assert f"Skipping build" in out @@ -152,7 +148,9 @@ def test_chartpress_run(git_repo, capfd): # verify result of --publish-chart with open("index.yaml", "r") as f: index_yaml = f.read() + print("index_yaml follows:") print(index_yaml) + print("-------------------") assert f"version: 1.2.1" in index_yaml assert f"version: 1.2.2" in index_yaml assert f"version: {tag}" in index_yaml @@ -184,34 +182,38 @@ def test_chartpress_paths_configuration(git_repo, capfd): # Add a file outside the chart repo and verify chartpress didn't update the # Chart.yaml version or the image tags in values.yaml. + chartpress.latest_tag_or_mod_commit.cache_clear() open("not-in-paths.txt", "w").close() git_repo.git.add(all=True) sha = git_repo.index.commit("Added not-in-paths.txt").hexsha[:7] tag = f"0.0.1-002.{sha}" out = _capture_output(["--skip-build"], capfd) assert f"Updating testchart/Chart.yaml: version: {tag}" not in out - assert f"Updating testchart/values.yaml: image: testchart/testimage: {tag}" not in out + assert f"Updating testchart/values.yaml: image: testchart/testimage:{tag}" not in out + # Add a file specified in the chart's paths configuration and verify # chartpress updated the Chart.yaml version, but not the image tags in # values.yaml. + chartpress.latest_tag_or_mod_commit.cache_clear() open("extra-chart-path.txt", "w").close() git_repo.git.add(all=True) sha = git_repo.index.commit("Added extra-chart-path.txt").hexsha[:7] tag = f"0.0.1-003.{sha}" out = _capture_output(["--skip-build"], capfd) assert f"Updating testchart/Chart.yaml: version: {tag}" in out - assert f"Updating testchart/values.yaml: image: testchart/testimage: {tag}" not in out + assert f"Updating testchart/values.yaml: image: testchart/testimage:{tag}" not in out # Add a file specified in a chart image's paths configuration and verify # updates to Chart.yaml version as well as the image tags in values.yaml. + chartpress.latest_tag_or_mod_commit.cache_clear() open("extra-image-path.txt", "w").close() git_repo.git.add(all=True) sha = git_repo.index.commit("Added extra-image-path.txt").hexsha[:7] tag = f"0.0.1-004.{sha}" out = _capture_output(["--skip-build"], capfd) assert f"Updating testchart/Chart.yaml: version: {tag}" in out - assert f"Updating testchart/values.yaml: image: testchart/testimage: {tag}" in out + assert f"Updating testchart/values.yaml: image: testchart/testimage:{tag}" in out def _capture_output(args, capfd): From ac85a6e5beae55a667abc413a6c4278e9ed415a0 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Tue, 29 Oct 2019 07:38:59 +0100 Subject: [PATCH 23/25] Clear test cache between CLI runs --- tests/README.md | 22 ---------------------- tests/test_repo_interactions.py | 21 +++++++++++++++------ 2 files changed, 15 insertions(+), 28 deletions(-) diff --git a/tests/README.md b/tests/README.md index b9278ca..a655b2f 100644 --- a/tests/README.md +++ b/tests/README.md @@ -2,8 +2,6 @@ These tests are [pytest](https://docs.pytest.org) tests. -useful flags: - ```bash pytest -vx --flakes @@ -18,26 +16,6 @@ pytest -vx --flakes # only run tests which match the given substring ``` -## Developing tests -Keep your eye out for test issues caused by caching! The -[`@lru_cache()`](https://docs.python.org/3/library/functools.html#functools.lru_cache) -decorator will do in memory cache that affects us when we run multiple commands -to chartpress in the same process. This decorator also adds the `cache_clear()` -function to the function object. - -It is very possible to run into issues with caching on `image_needs_building`, -`image_needs_pushing` and `latest_tag_or_mod_commit`. - -```python -# clearing the cache of these functions is important if you run chartpress -# without --skip-build -chartpress.image_needs_building.cache_clear() -chartpress.image_needs_pushing.cache_clear() - -# clearing the cache of this function is important very often -chartpress.latest_tag_or_mod_commit.cache_clear() -``` - ## Not yet tested - `--push` diff --git a/tests/test_repo_interactions.py b/tests/test_repo_interactions.py index 80f1ebd..bb293ea 100644 --- a/tests/test_repo_interactions.py +++ b/tests/test_repo_interactions.py @@ -24,7 +24,6 @@ def test_chartpress_run(git_repo, capfd): tag = f"0.0.1-001.{sha}" # run chartpress - chartpress.image_needs_building.cache_clear() out = _capture_output([], capfd) # verify image was built @@ -50,7 +49,6 @@ def test_chartpress_run(git_repo, capfd): assert f"Updating testchart/values.yaml: list.1.image: testchart/testimage:test-reset-tag" in out # verify that we don't need to rebuild the image - chartpress.image_needs_building.cache_clear() out = _capture_output([], capfd) assert f"Skipping build" in out @@ -129,7 +127,6 @@ def test_chartpress_run(git_repo, capfd): open("extra-chart-path.txt", "w").close() git_repo.git.add(all=True) sha = git_repo.index.commit("Added extra-chart-path.txt").hexsha[:7] - chartpress.latest_tag_or_mod_commit.cache_clear() out = _capture_output( [ "--skip-build", @@ -182,7 +179,6 @@ def test_chartpress_paths_configuration(git_repo, capfd): # Add a file outside the chart repo and verify chartpress didn't update the # Chart.yaml version or the image tags in values.yaml. - chartpress.latest_tag_or_mod_commit.cache_clear() open("not-in-paths.txt", "w").close() git_repo.git.add(all=True) sha = git_repo.index.commit("Added not-in-paths.txt").hexsha[:7] @@ -195,7 +191,6 @@ def test_chartpress_paths_configuration(git_repo, capfd): # Add a file specified in the chart's paths configuration and verify # chartpress updated the Chart.yaml version, but not the image tags in # values.yaml. - chartpress.latest_tag_or_mod_commit.cache_clear() open("extra-chart-path.txt", "w").close() git_repo.git.add(all=True) sha = git_repo.index.commit("Added extra-chart-path.txt").hexsha[:7] @@ -206,7 +201,6 @@ def test_chartpress_paths_configuration(git_repo, capfd): # Add a file specified in a chart image's paths configuration and verify # updates to Chart.yaml version as well as the image tags in values.yaml. - chartpress.latest_tag_or_mod_commit.cache_clear() open("extra-image-path.txt", "w").close() git_repo.git.add(all=True) sha = git_repo.index.commit("Added extra-image-path.txt").hexsha[:7] @@ -217,9 +211,24 @@ def test_chartpress_paths_configuration(git_repo, capfd): def _capture_output(args, capfd): + """ + Calls chartpress given provided arguments and captures the output during the + call. + """ + # clear cache of in memory cached functions + # this allows us to better mimic the chartpress CLI behavior + chartpress.image_needs_building.cache_clear() + chartpress.image_needs_pushing.cache_clear() + chartpress.latest_tag_or_mod_commit.cache_clear() + + # first flush past captured output, then run chartpress, and finally read + # and save all output that came of it _, _ = capfd.readouterr() chartpress.main(args) out, err = capfd.readouterr() + + # since the output was captured, print it back out again for debugging + # purposes if a test fails for example header = f'--- chartpress {" ".join(args)} ---' footer = "-" * len(header) print() From ccbbba71d4419a2da14de6bc0287deca63de5110 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 30 Oct 2019 21:38:38 +0100 Subject: [PATCH 24/25] Docstrings about helm chart registries --- README.md | 2 +- chartpress.py | 29 +++++++++++++++++++++++++++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e417bf3..073fd86 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ Chartpress can do the following with the help of some configuration. - Build docker images and tag them appropriately - Push built images to a docker iamge repository - Update values.yaml to reference the built images -- Publish a chart to a Helm chart repository based on GitHub pages +- Publish a chart to a Helm chart registry based on GitHub pages - Reset changes to Chart.yaml and values.yaml ## Requirements diff --git a/chartpress.py b/chartpress.py index 63d9403..0445924 100755 --- a/chartpress.py +++ b/chartpress.py @@ -503,7 +503,32 @@ def build_chart(name, version=None, paths=None, long=False): def publish_pages(chart_name, chart_version, chart_repo_github_path, chart_repo_url, extra_message=''): - """Publish Helm chart index to github pages""" + """ + Update a Helm chart registry hosted in the gh-pages branch of a GitHub git + repository. + + The strategy adopted to do this is: + + 1. Clone the Helm chart registry as found in the gh-pages branch of a git + reposistory. + 2. Create a temporary directory and `helm package` the chart into a file + within this temporary directory now only containing the chart .tar file. + 3. Generate a index.yaml with `helm repo index` based on charts found in the + temporary directory folder (a single one), and then merge in the bigger + and existing index.yaml from the cloned Helm chart registry using the + --merge flag. + + Note that if we would add the new chart .tar file next to the other .tar + files and use `helm repo index` we would recreate `index.yaml` and update + all the timestamps etc. which is something we don't want. Using `helm repo + index` on a directory with only the new chart .tar file allows us to avoid + this issue. + + Also note that the --merge flag will not override existing entries to the + fresh index.yaml file with the index.yaml from the --merge flag. Due to + this, it is as we would have a --force-publish-chart by default. + """ + # clone the Helm chart repo and checkout its gh-pages branch, # note the use of cwd (current working directory) checkout_dir = '{}-{}'.format(chart_name, chart_version) @@ -580,7 +605,7 @@ def main(args=None): argparser.add_argument( '--publish-chart', action='store_true', - help='Package a Helm chart and publish it to a Helm chart repository contructed with a GitHub git repository and GitHub pages.', + help='Package a Helm chart and publish it to a Helm chart registry contructed with a GitHub git repository and GitHub pages.', ) argparser.add_argument( '--extra-message', From 68c7e7085eb3dff2ff3be1fd284b4797f46140f3 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Wed, 30 Oct 2019 22:01:33 +0100 Subject: [PATCH 25/25] Complement docstring --- chartpress.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/chartpress.py b/chartpress.py index 0445924..995c579 100755 --- a/chartpress.py +++ b/chartpress.py @@ -517,6 +517,8 @@ def publish_pages(chart_name, chart_version, chart_repo_github_path, chart_repo_ temporary directory folder (a single one), and then merge in the bigger and existing index.yaml from the cloned Helm chart registry using the --merge flag. + 4. Copy the new index.yaml and packaged Helm chart .tar into the gh-pages + branch, commit it, and push it back to the origin remote. Note that if we would add the new chart .tar file next to the other .tar files and use `helm repo index` we would recreate `index.yaml` and update