ci, tests: pass charm artefacts to deploy and test charms

This commit enables the "--charm-path" option to pass .charm artefacts
to be deployed and tested at an individual level.
This change enables the option to pass pre-built charms to the tests
to avoid building them on the same test. It also ensures compatibility
with the build_charm.py reusable workflow (from canonical/data-platform-workflows).

Fixes: #639

.github/workflows/integrate.yaml

@@ -122,12 +122,29 @@ jobs:
           # Pinned to 3.x/stable due to https://github.com/canonical/charmcraft/issues/1845
           charmcraft-channel: 3.x/stable
 
+      - name: Download packed charm(s)
+        id: download-charms
+        timeout-minutes: 5
+        uses: actions/download-artifact@v4
+        with:
+          pattern: packed-charm-cache-true-.-charms-${{ matrix.charm }}-*
+          merge-multiple: true
+
+      - name: Debug
+        if: steps.download-charms.outcome == 'success'
+        run: |
+          ls -lh ${{ github.workspace }}/charms/${{ matrix.charm }}/
+
       - name: Integration tests
+        if: steps.download-charms.outcome == 'success'
         run: |
           # Requires the model to be called kubeflow due to
           # https://github.com/canonical/kfp-operators/issues/389
           juju add-model kubeflow
-          sg snap_microk8s -c "tox -e ${{ matrix.charm }}-integration -- --model kubeflow"
+          # Pass the path where the charm artefact is downloaded to the tox command
+          # FIXME: Right now the complete path is half hardcoded to <charm name>_ubuntu-20.04-amd64.charm
+          # We need to find a better way to dynamically get this value
+          sg snap_microk8s -c "tox -e ${{ matrix.charm }}-integration -- --model kubeflow --charm-path=${{ github.workspace }}/charms/${{ matrix.charm }}/${{ matrix.charm }}_ubuntu-20.04-amd64.charm"
 
       - name: Collect charm debug artifacts
         uses: canonical/kubeflow-ci/actions/dump-charm-debug-artifacts@main

.github/workflows/on_pull_request.yaml

@@ -3,19 +3,25 @@ name: On Pull Request
 # On pull_request, we:
 # * always publish to charmhub at latest/edge/branchname
 # * always run tests
-
+# * always run builds with cache
 on:
   pull_request:
 
 jobs:
+  get-paths-and-build:
+    name: Get charm paths and build with cache
+    # FIXME: change to main before merging
+    uses: canonical/charmed-kubeflow-workflows/.github/workflows/get_charms_build_with_cache.yaml@KF-6690-build-with-cache
 
   tests:
     name: Run Tests
+    needs: [get-paths-and-build]
     uses: ./.github/workflows/integrate.yaml
     secrets: inherit
 
   # publish runs in parallel with tests, as we always publish in this situation
   publish-charm:
     name: Publish Charm
+    needs: [get-paths-and-build]
     uses: ./.github/workflows/publish.yaml
     secrets: inherit

charms/kfp-api/tests/integration/conftest.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+from _pytest.config.argparsing import Parser
+
+
+def pytest_addoption(parser: Parser):
+    parser.addoption(
+        "--charm-path",
+        help="Path to charm file when downloaded as artefact as a result of build_charm.yaml",
+    )

charms/kfp-api/tests/integration/test_charm.py

@@ -43,16 +43,21 @@ class TestCharm:
     """Integration test charm"""
 
     @pytest.mark.abort_on_fail
-    async def test_build_and_deploy(self, ops_test: OpsTest):
+    async def test_build_and_deploy(self, ops_test: OpsTest, request):
         """Deploy kfp-api with required charms and relations."""
-        built_charm_path = await ops_test.build_charm("./")
-        logger.info(f"Built charm {built_charm_path}")
 
         image_path = METADATA["resources"]["oci-image"]["upstream-source"]
         resources = {"oci-image": image_path}
+        # Keep the option to run the integration tests locally
+        # by building the charm and then deploying
+        entity_url = (
+            await ops_test.build_charm("./")
+            if not (entity_url := request.config.getoption("--charm-path"))
+            else entity_url
+        )
 
         await ops_test.model.deploy(
-            entity_url=built_charm_path,
+            entity_url=entity_url,
             application_name=APP_NAME,
             resources=resources,
             trust=True,

charms/kfp-metadata-writer/tests/integration/conftest.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+from _pytest.config.argparsing import Parser
+
+
+def pytest_addoption(parser: Parser):
+    parser.addoption(
+        "--charm-path",
+        help="Path to charm file when downloaded as artefact as a result of build_charm.yaml",
+    )

charms/kfp-metadata-writer/tests/integration/test_charm.py

@@ -26,16 +26,24 @@
 
 
 @pytest.mark.abort_on_fail
-async def test_build_and_deploy_with_relations(ops_test: OpsTest):
-    built_charm_path = await ops_test.build_charm(CHARM_ROOT)
-    log.info(f"Built charm {built_charm_path}")
-
+async def test_build_and_deploy_with_relations(ops_test: OpsTest, request):
     image_path = METADATA["resources"]["oci-image"]["upstream-source"]
     resources = {"oci-image": image_path}
+    # Keep the option to run the integration tests locally
+    # by building the charm and then deploying
+    entity_url = (
+        await ops_test.build_charm("./")
+        if not (entity_url := request.config.getoption("--charm-path"))
+        else entity_url
+    )
 
     await ops_test.model.deploy(
-        entity_url=built_charm_path, application_name=APP_NAME, resources=resources, trust=True
+        entity_url=entity_url,
+        application_name=APP_NAME,
+        resources=resources,
+        trust=True,
     )
+
     await ops_test.model.deploy(entity_url=MLMD, channel=MLMD_CHANNEL, trust=True)
     await ops_test.model.integrate(f"{MLMD}:grpc", f"{APP_NAME}:grpc")
     await ops_test.model.wait_for_idle(apps=[APP_NAME, MLMD], status="active", timeout=10 * 60)

charms/kfp-persistence/tests/integration/conftest.py

@@ -0,0 +1,12 @@
+#!/usr/bin/env python3
+# Copyright 2024 Canonical Ltd.
+# See LICENSE file for licensing details.
+
+from _pytest.config.argparsing import Parser
+
+
+def pytest_addoption(parser: Parser):
+    parser.addoption(
+        "--charm-path",
+        help="Path to charm file when downloaded as artefact as a result of build_charm.yaml",
+    )

charms/kfp-persistence/tests/integration/test_charm.py

@@ -39,16 +39,20 @@ class TestCharm:
     """Integration test charm"""
 
     @pytest.mark.abort_on_fail
-    async def test_build_and_deploy(self, ops_test: OpsTest):
+    async def test_build_and_deploy(self, ops_test: OpsTest, request):
         """Deploy kfp-persistence with required charms and relations."""
-        built_charm_path = await ops_test.build_charm("./")
-        logger.info(f"Built charm {built_charm_path}")
-
         image_path = METADATA["resources"]["oci-image"]["upstream-source"]
         resources = {"oci-image": image_path}
+        # Keep the option to run the integration tests locally
+        # by building the charm and then deploying
+        entity_url = (
+            await ops_test.build_charm("./")
+            if not (entity_url := request.config.getoption("--charm-path"))
+            else entity_url
+        )
 
         await ops_test.model.deploy(
-            entity_url=built_charm_path,
+            entity_url=entity_url,
             application_name=APP_NAME,
             resources=resources,
             trust=True,