From 831149fe15a293a54a1f2bcd0ec349947fadf7b2 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Mon, 25 Sep 2023 16:31:47 +1300 Subject: [PATCH 01/12] Adjust pyproject.toml if it exists and requirements.txt does not. --- .github/workflows/db-charm-tests.yaml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/db-charm-tests.yaml b/.github/workflows/db-charm-tests.yaml index edb991415..23688c274 100644 --- a/.github/workflows/db-charm-tests.yaml +++ b/.github/workflows/db-charm-tests.yaml @@ -11,9 +11,8 @@ jobs: charm-repo: - "canonical/postgresql-operator" - "canonical/postgresql-k8s-operator" -# TODO: uncomment once we've fixed https://github.com/canonical/operator/issues/987 -# - "canonical/mysql-operator" -# - "canonical/mysql-k8s-operator" + - "canonical/mysql-operator" + - "canonical/mysql-k8s-operator" steps: - name: Checkout the ${{ matrix.charm-repo }} repository @@ -23,8 +22,14 @@ jobs: - name: Update 'ops' dependency in test charm to latest run: | - sed -i -e "/^ops[ ><=]/d" -e "/canonical\/operator/d" -e "/#egg=ops/d" requirements.txt - echo -e "\ngit+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_SHA#egg=ops" >> requirements.txt + if [ -e "requirements.txt" ]; then + sed -i -e "/^ops[ ><=]/d" -e "/canonical\/operator/d" -e "/#egg=ops/d" requirements.txt + echo -e "\ngit+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_SHA#egg=ops" >> requirements.txt + else + GITHUB_REPOSITORY_ESC=`echo $GITHUB_REPOSITORY | sed -e 's/\//\\\\\\//'` + GITHUB_SERVER_URL_ESC=`echo $GITHUB_SERVER_URL | sed -e 's/\//\\\\\\//g'` + sed -i -e "s/^ops[ ><=].*/git\+$GITHUB_SERVER_URL_ESC\/$GITHUB_REPOSITORY_ESC@$GITHUB_SHA#egg=ops/g" pyproject.toml + fi - name: Install dependencies run: pip install tox==4.2.8 From 60e12e3dca8995e25afcd95ee1964fb57782bdf1 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Mon, 25 Sep 2023 16:51:37 +1300 Subject: [PATCH 02/12] Adjust the format of the pyproject.toml replacement. --- .github/workflows/db-charm-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/db-charm-tests.yaml b/.github/workflows/db-charm-tests.yaml index 23688c274..42c5142b9 100644 --- a/.github/workflows/db-charm-tests.yaml +++ b/.github/workflows/db-charm-tests.yaml @@ -28,7 +28,7 @@ jobs: else GITHUB_REPOSITORY_ESC=`echo $GITHUB_REPOSITORY | sed -e 's/\//\\\\\\//'` GITHUB_SERVER_URL_ESC=`echo $GITHUB_SERVER_URL | sed -e 's/\//\\\\\\//g'` - sed -i -e "s/^ops[ ><=].*/git\+$GITHUB_SERVER_URL_ESC\/$GITHUB_REPOSITORY_ESC@$GITHUB_SHA#egg=ops/g" pyproject.toml + sed -i -e "s/^ops[ ><=].*/ops = { git = \"$GITHUB_SERVER_URL_ESC\/$GITHUB_REPOSITORY_ESC.git\", rev=\"$GITHUB_SHA\" }/g" pyproject.toml fi - name: Install dependencies From 4b76843f426bc9ab9d830e0841b703a697b5085b Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Mon, 25 Sep 2023 16:56:48 +1300 Subject: [PATCH 03/12] Double-check what the Github CI is using to reproduce locally. --- .github/workflows/db-charm-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/db-charm-tests.yaml b/.github/workflows/db-charm-tests.yaml index 42c5142b9..471e9418f 100644 --- a/.github/workflows/db-charm-tests.yaml +++ b/.github/workflows/db-charm-tests.yaml @@ -30,6 +30,7 @@ jobs: GITHUB_SERVER_URL_ESC=`echo $GITHUB_SERVER_URL | sed -e 's/\//\\\\\\//g'` sed -i -e "s/^ops[ ><=].*/ops = { git = \"$GITHUB_SERVER_URL_ESC\/$GITHUB_REPOSITORY_ESC.git\", rev=\"$GITHUB_SHA\" }/g" pyproject.toml fi + cat pyproject.toml - name: Install dependencies run: pip install tox==4.2.8 From 1515dbe12d36060009788b229989162af325435f Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 26 Sep 2023 09:47:55 +1300 Subject: [PATCH 04/12] Remove debugging output. --- .github/workflows/db-charm-tests.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/db-charm-tests.yaml b/.github/workflows/db-charm-tests.yaml index 471e9418f..42c5142b9 100644 --- a/.github/workflows/db-charm-tests.yaml +++ b/.github/workflows/db-charm-tests.yaml @@ -30,7 +30,6 @@ jobs: GITHUB_SERVER_URL_ESC=`echo $GITHUB_SERVER_URL | sed -e 's/\//\\\\\\//g'` sed -i -e "s/^ops[ ><=].*/ops = { git = \"$GITHUB_SERVER_URL_ESC\/$GITHUB_REPOSITORY_ESC.git\", rev=\"$GITHUB_SHA\" }/g" pyproject.toml fi - cat pyproject.toml - name: Install dependencies run: pip install tox==4.2.8 From cdfd1d29145c3b1a8d94540312e098d3f7c80b5b Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 26 Sep 2023 16:50:18 +1300 Subject: [PATCH 05/12] Check out the ops repo locally and do an editable install. --- .github/workflows/db-charm-tests.yaml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/db-charm-tests.yaml b/.github/workflows/db-charm-tests.yaml index 42c5142b9..178cc182f 100644 --- a/.github/workflows/db-charm-tests.yaml +++ b/.github/workflows/db-charm-tests.yaml @@ -20,15 +20,19 @@ jobs: with: repository: ${{ matrix.charm-repo }} + - name: Checkout the operator repository + uses: actions/checkout@v3 + with: + path: myops + - name: Update 'ops' dependency in test charm to latest run: | if [ -e "requirements.txt" ]; then sed -i -e "/^ops[ ><=]/d" -e "/canonical\/operator/d" -e "/#egg=ops/d" requirements.txt echo -e "\ngit+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_SHA#egg=ops" >> requirements.txt else - GITHUB_REPOSITORY_ESC=`echo $GITHUB_REPOSITORY | sed -e 's/\//\\\\\\//'` - GITHUB_SERVER_URL_ESC=`echo $GITHUB_SERVER_URL | sed -e 's/\//\\\\\\//g'` - sed -i -e "s/^ops[ ><=].*/ops = { git = \"$GITHUB_SERVER_URL_ESC\/$GITHUB_REPOSITORY_ESC.git\", rev=\"$GITHUB_SHA\" }/g" pyproject.toml + sed -i -e "s/^ops[ ><=].*/d" pyproject.toml + sed -i -e "s/poetry install --only main,charm-libs,unit/poetry install --only main,charm-libs,unit\n pip install -e myops" tox.ini fi - name: Install dependencies From f67c0d99c740a9d535690ce3c095737931a22902 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 26 Sep 2023 16:51:31 +1300 Subject: [PATCH 06/12] Typo in sed. --- .github/workflows/db-charm-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/db-charm-tests.yaml b/.github/workflows/db-charm-tests.yaml index 178cc182f..3b6fc66ab 100644 --- a/.github/workflows/db-charm-tests.yaml +++ b/.github/workflows/db-charm-tests.yaml @@ -31,7 +31,7 @@ jobs: sed -i -e "/^ops[ ><=]/d" -e "/canonical\/operator/d" -e "/#egg=ops/d" requirements.txt echo -e "\ngit+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_SHA#egg=ops" >> requirements.txt else - sed -i -e "s/^ops[ ><=].*/d" pyproject.toml + sed -i -e "/^ops[ ><=].*/d" pyproject.toml sed -i -e "s/poetry install --only main,charm-libs,unit/poetry install --only main,charm-libs,unit\n pip install -e myops" tox.ini fi From 0b575733d90f9fb18b0fa44ff562a50af0d0a791 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 26 Sep 2023 16:53:16 +1300 Subject: [PATCH 07/12] Typo in sed. --- .github/workflows/db-charm-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/db-charm-tests.yaml b/.github/workflows/db-charm-tests.yaml index 3b6fc66ab..de337ed0f 100644 --- a/.github/workflows/db-charm-tests.yaml +++ b/.github/workflows/db-charm-tests.yaml @@ -32,7 +32,7 @@ jobs: echo -e "\ngit+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_SHA#egg=ops" >> requirements.txt else sed -i -e "/^ops[ ><=].*/d" pyproject.toml - sed -i -e "s/poetry install --only main,charm-libs,unit/poetry install --only main,charm-libs,unit\n pip install -e myops" tox.ini + sed -i -e "s/poetry install --only main,charm-libs,unit/poetry install --only main,charm-libs,unit\n pip install -e myops/" tox.ini fi - name: Install dependencies From 7a39a6e13f738de39b2f9e97246a772936f84e88 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 26 Sep 2023 16:58:07 +1300 Subject: [PATCH 08/12] Try forcing a poetry lock. --- .github/workflows/db-charm-tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/db-charm-tests.yaml b/.github/workflows/db-charm-tests.yaml index de337ed0f..0e707a659 100644 --- a/.github/workflows/db-charm-tests.yaml +++ b/.github/workflows/db-charm-tests.yaml @@ -32,7 +32,7 @@ jobs: echo -e "\ngit+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_SHA#egg=ops" >> requirements.txt else sed -i -e "/^ops[ ><=].*/d" pyproject.toml - sed -i -e "s/poetry install --only main,charm-libs,unit/poetry install --only main,charm-libs,unit\n pip install -e myops/" tox.ini + sed -i -e "s/poetry install --only main,charm-libs,unit/poetry lock\n poetry install --only main,charm-libs,unit\n pip install -e myops/" tox.ini fi - name: Install dependencies From 5adedb78de171101fd76ecc7ce3ed8aab59c2422 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 26 Sep 2023 17:09:28 +1300 Subject: [PATCH 09/12] Does the tox file get changed? --- .github/workflows/db-charm-tests.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/db-charm-tests.yaml b/.github/workflows/db-charm-tests.yaml index 0e707a659..857553352 100644 --- a/.github/workflows/db-charm-tests.yaml +++ b/.github/workflows/db-charm-tests.yaml @@ -34,6 +34,7 @@ jobs: sed -i -e "/^ops[ ><=].*/d" pyproject.toml sed -i -e "s/poetry install --only main,charm-libs,unit/poetry lock\n poetry install --only main,charm-libs,unit\n pip install -e myops/" tox.ini fi + cat tox.ini - name: Install dependencies run: pip install tox==4.2.8 From 05656a52d182ffdd8b7ae62fd40d0fb8151b452b Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Tue, 26 Sep 2023 17:28:25 +1300 Subject: [PATCH 10/12] Combine the two techniques. Manipulate pyproject.toml, but refer to the local checkout not git. --- .github/workflows/db-charm-tests.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/db-charm-tests.yaml b/.github/workflows/db-charm-tests.yaml index 857553352..d5ebd035b 100644 --- a/.github/workflows/db-charm-tests.yaml +++ b/.github/workflows/db-charm-tests.yaml @@ -25,16 +25,18 @@ jobs: with: path: myops + - name: Install patch dependencies + run: pip install poetry==1.6.1 + - name: Update 'ops' dependency in test charm to latest run: | if [ -e "requirements.txt" ]; then sed -i -e "/^ops[ ><=]/d" -e "/canonical\/operator/d" -e "/#egg=ops/d" requirements.txt echo -e "\ngit+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_SHA#egg=ops" >> requirements.txt else - sed -i -e "/^ops[ ><=].*/d" pyproject.toml - sed -i -e "s/poetry install --only main,charm-libs,unit/poetry lock\n poetry install --only main,charm-libs,unit\n pip install -e myops/" tox.ini + sed -i -e "s/^ops[ ><=].*/ops = {path = \"myops\"}/" pyproject.toml + poetry lock fi - cat tox.ini - name: Install dependencies run: pip install tox==4.2.8 From 0c183eeb30c4d74515018b65b1936682ed6687e6 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Wed, 27 Sep 2023 13:29:41 +1300 Subject: [PATCH 11/12] Allow patch upgrades to tox and poetry. --- .github/workflows/db-charm-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/db-charm-tests.yaml b/.github/workflows/db-charm-tests.yaml index d5ebd035b..f8f64fb97 100644 --- a/.github/workflows/db-charm-tests.yaml +++ b/.github/workflows/db-charm-tests.yaml @@ -26,7 +26,7 @@ jobs: path: myops - name: Install patch dependencies - run: pip install poetry==1.6.1 + run: pip install poetry~=1.6.1 - name: Update 'ops' dependency in test charm to latest run: | @@ -39,7 +39,7 @@ jobs: fi - name: Install dependencies - run: pip install tox==4.2.8 + run: pip install tox~=4.2 - name: Run the charm's unit tests run: tox -vve unit From f49c9a1f66bb1e583abcec96ec50938d04bf4956 Mon Sep 17 00:00:00 2001 From: Tony Meyer Date: Wed, 27 Sep 2023 14:34:26 +1300 Subject: [PATCH 12/12] Fix pinning. * Pin all tox's to anything compatible with 4.2 * Pin poetry to anything compatible with 1.6 * Remove the 'static-charm' tox run from the DB workflow, as none of the repositories have that target, and more recent versions of tox will fail if the target (env) doesn't exist. --- .github/workflows/db-charm-tests.yaml | 5 +---- .github/workflows/framework-tests.yaml | 11 ++++++----- .github/workflows/observability-charm-tests.yaml | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/db-charm-tests.yaml b/.github/workflows/db-charm-tests.yaml index f8f64fb97..46cef2420 100644 --- a/.github/workflows/db-charm-tests.yaml +++ b/.github/workflows/db-charm-tests.yaml @@ -26,7 +26,7 @@ jobs: path: myops - name: Install patch dependencies - run: pip install poetry~=1.6.1 + run: pip install poetry~=1.6 - name: Update 'ops' dependency in test charm to latest run: | @@ -43,6 +43,3 @@ jobs: - name: Run the charm's unit tests run: tox -vve unit - - - name: Run the charm's static analysis checks - run: tox -vve static-charm diff --git a/.github/workflows/framework-tests.yaml b/.github/workflows/framework-tests.yaml index b04ce64a0..082635c89 100644 --- a/.github/workflows/framework-tests.yaml +++ b/.github/workflows/framework-tests.yaml @@ -12,7 +12,7 @@ jobs: uses: actions/setup-python@v2 - name: Install tox - run: pip install tox + run: pip install tox~=4.2 - name: Run linting run: tox -e lint @@ -26,7 +26,7 @@ jobs: uses: actions/setup-python@v2 - name: Install tox - run: pip install tox + run: pip install tox~=4.2 - name: Run static type checks run: tox -e static @@ -47,7 +47,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install tox - run: pip install tox + run: pip install tox~=4.2 - name: Run unit tests run: tox -e unit @@ -73,7 +73,7 @@ jobs: go-version: "1.20" - name: Install tox - run: pip install tox + run: pip install tox~=4.2 - name: Install Pebble run: go install github.com/canonical/pebble/cmd/pebble@latest @@ -88,7 +88,7 @@ jobs: - name: Run Real pebble tests run: tox -e unit -- -k RealPebble env: - RUN_REAL_PEBBLE_TESTS: 1 + RUN_REAL_PEBBLE_TESTS: 1 PEBBLE: /tmp/pebble pip-install: @@ -106,3 +106,4 @@ jobs: - name: Test 'pip install' # Shouldn't happen, but pip install will fail if ls returns multiple lines run: pip install $(ls dist/ops*.gz) + diff --git a/.github/workflows/observability-charm-tests.yaml b/.github/workflows/observability-charm-tests.yaml index 7a66f6b6f..98a96372f 100644 --- a/.github/workflows/observability-charm-tests.yaml +++ b/.github/workflows/observability-charm-tests.yaml @@ -26,7 +26,7 @@ jobs: echo -e "\ngit+$GITHUB_SERVER_URL/$GITHUB_REPOSITORY@$GITHUB_SHA#egg=ops" >> requirements.txt - name: Install dependencies - run: pip install tox + run: pip install tox~=4.2 - name: Run the charm's unit tests run: tox -vve unit