feat: convert bundle tests to pytest-operator (#85)

This adds a pytest-operator driven test suite for the bundle.  Previously, the bundle was deployed "manually" in integrate.yaml.  This refactors the test suite to deploy using pytest-operator and by substituting locally built charms into a template bundle.

This commit includes some transferable helpers that should in future be migrated to a shared repo (either in pytest-operator or somewhere else like charmed-kubeflow-chisme)

Also included: 
* Adds notes to disabled test_pipelines.py

.github/workflows/integrate.yaml

@@ -103,65 +103,50 @@ jobs:
           app: ${{ matrix.charm }}
           model: testing
 
-## TODO: Temporarily disabled
-#  test-bundle:
-#    name: Test the bundle
-#    runs-on: ubuntu-latest
-#
-#    steps:
-#    - uses: actions/checkout@v2
-#    - uses: balchua/microk8s-actions@v0.2.2
-#      with:
-#        addons: '["dns", "storage", "rbac"]'
-#        channel: 1.21/stable
-#
-#    - name: Install dependencies
-#      run: |
-#        set -eux
-#        sudo pip3 install charmcraft==1.0.0 pytest kfp
-#        sudo snap install juju --classic
-#        sudo snap install juju-helpers --classic
-#        sudo snap install juju-wait --classic
-#
-#    # Avoid race condition with storage taking a long time to initialize
-#    - name: Wait for storage
-#      run: |
-#        sg microk8s -c 'microk8s kubectl rollout status deployment/hostpath-provisioner -n kube-system'
-#
-#    - name: Bootstrap Juju
-#      run: |
-#        set -eux
-#        sg microk8s -c 'juju bootstrap microk8s uk8s'
-#        juju add-model kubeflow
-#
-#    - name: Deploy istio-pilot
-#      run: |
-#        set -eux
-#        juju deploy cs:istio-pilot
-#        juju wait -wvt 300
-#
-#    - name: Deploy bundle
-#      run: |
-#        set -eux
-#        juju bundle deploy --build
-#        juju relate istio-pilot:ingress kfp-ui:ingress
-#        juju wait -wvt 600
-#
-#    - name: Test bundle
-#      run: sg microk8s -c 'pytest -svv --ignore charms/kfp-profile-controller'
-#
-#    - name: Get all
-#      run: kubectl get all -A
-#      if: failure()
-#
-#    - name: Get juju status
-#      run: juju status
-#      if: failure()
-#
-#    - name: Get workload logs
-#      run: kubectl logs --tail 100 -nkubeflow -ljuju-app
-#      if: failure()
-#
-#    - name: Get operator logs
-#      run: kubectl logs --tail 100 -nkubeflow -ljuju-operator
-#      if: failure()
+  test-bundle:
+    name: Test the bundle
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v2
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: microk8s
+          channel: 1.21/stable
+          charmcraft-channel: latest/candidate
+
+      # Required until https://github.com/charmed-kubernetes/actions-operator/pull/33 is merged
+      - run: sg microk8s -c "microk8s enable metallb:'10.64.140.43-10.64.140.49,192.168.0.105-192.168.0.111'"
+
+
+      # TODO: Remove once the actions-operator does this automatically
+      - name: Configure kubectl
+        run: |
+          sg microk8s -c "microk8s config > ~/.kube/config"
+
+      - name: Run test
+        run: |
+          # Requires the model to be called kubeflow due to kfp-viewer
+          juju add-model kubeflow
+          # Remove destructive-mode once these bugs are fixed:
+          # https://github.com/canonical/charmcraft/issues/554
+          # https://github.com/canonical/craft-providers/issues/96
+          tox -e bundle-integration -- --model kubeflow --destructive-mode
+
+      - name: Get all
+        run: kubectl get all -A
+        if: failure()
+
+      - name: Get juju status
+        run: juju status
+        if: failure()
+
+      - name: Get workload logs
+        run: kubectl logs --tail 100 -nkubeflow -ljuju-app
+        if: failure()
+
+      - name: Get operator logs
+        run: kubectl logs --tail 100 -nkubeflow -ljuju-operator
+        if: failure()

tests/conftest.py

@@ -0,0 +1,12 @@
+from _pytest.config.argparsing import Parser
+
+
+def pytest_addoption(parser: Parser):
+    parser.addoption(
+        "--bundle",
+        default="./tests/integration/data/kfp_against_latest_edge.yaml",
+        help="Path to bundle file to use as the template for tests.  This must include all charms"
+             "built by this bundle, where the locally built charms will replace those specified. "
+             "This is useful for testing this bundle against different external dependencies. "
+             "An example file is in ./tests/integration/data/kfp_against_latest_edge.yaml",
+    )

tests/integration/data/kfp_against_latest_edge.yaml

@@ -0,0 +1,100 @@
+applications:
+  argo-controller:
+    channel: latest/edge
+    charm: ch:argo-controller
+    scale: 1
+  istio-ingressgateway:
+    _github_repo_name: istio-operators
+    channel: latest/edge
+    charm: istio-gateway
+    options:
+      kind: ingress
+    scale: 1
+    trust: true
+  istio-pilot:
+    _github_repo_name: istio-operators
+    channel: latest/edge
+    charm: istio-pilot
+    options:
+      default-gateway: kubeflow-gateway
+    scale: 1
+    trust: true
+  kfp-api:
+    channel: latest/edge
+    charm: ch:kfp-api
+    scale: 1
+  kfp-db:
+    charm: cs:~charmed-osm/mariadb-k8s-35
+    options:
+      database: mlpipeline
+    scale: 1
+  kfp-persistence:
+    channel: latest/edge
+    charm: ch:kfp-persistence
+    scale: 1
+  kfp-profile-controller:
+    channel: latest/edge
+    charm: ch:kfp-profile-controller
+    scale: 1
+  kfp-schedwf:
+    channel: latest/edge
+    charm: ch:kfp-schedwf
+    scale: 1
+  kfp-ui:
+    channel: latest/edge
+    charm: ch:kfp-ui
+    scale: 1
+  kfp-viewer:
+    channel: latest/edge
+    charm: ch:kfp-viewer
+    scale: 1
+  kfp-viz:
+    channel: latest/edge
+    charm: ch:kfp-viz
+    scale: 1
+  kubeflow-dashboard:
+    _github_repo_name: kubeflow-dashboard-operator
+    channel: latest/edge
+    charm: kubeflow-dashboard
+    scale: 1
+  kubeflow-profiles:
+    _github_repo_name: kubeflow-profiles-operator
+    channel: latest/edge
+    charm: kubeflow-profiles
+    scale: 1
+  metacontroller-operator:
+    channel: latest/edge
+    charm: ch:metacontroller-operator
+    scale: 1
+    trust: true
+  minio:
+    channel: latest/edge
+    charm: ch:minio
+    scale: 1
+bundle: kubernetes
+name: kubeflow-pipelines
+relations:
+  - - kfp-api
+    - kfp-db
+  - - kfp-api:kfp-api
+    - kfp-persistence:kfp-api
+  - - kfp-api:kfp-api
+    - kfp-ui:kfp-api
+  - - kfp-api:kfp-viz
+    - kfp-viz:kfp-viz
+  - - kfp-api:object-storage
+    - minio:object-storage
+  - - kfp-profile-controller:object-storage
+    - minio:object-storage
+  - - kfp-ui:object-storage
+    - minio:object-storage
+  - - argo-controller:object-storage
+    - minio:object-storage
+  - - kubeflow-profiles
+    - kubeflow-dashboard
+  - - istio-pilot:ingress
+    - kubeflow-dashboard:ingress
+  - - istio-pilot:istio-pilot
+    - istio-ingressgateway:istio-pilot
+  - - istio-pilot:ingress
+    - kfp-ui:ingress

tests/integration/helpers/localize_bundle.py

@@ -0,0 +1,98 @@
+# Copyright 2022 Canonical Ltd.
+# See LICENSE file for licensing details.
+import copy
+from pathlib import Path
+from typing import Dict, Optional, Union
+import yaml
+from zipfile import ZipFile
+
+
+# TODO: Move this somewhere more general
+
+
+def get_charm_name(metadata_file: Union[Path, str]) -> str:
+    metadata = yaml.safe_load(Path(metadata_file).read_text())
+    return metadata["name"]
+
+
+def get_charm_file(charm_dir: Path) -> Path:
+    """Returns the path to the .charm file representing the charm in the given directory
+
+    TODO: This just assumes the suffix on the file name will be "ubuntu-20.04-amd64".
+          Fix this in future
+    """
+    charm_dir = Path(charm_dir)
+    metadata_file = charm_dir / "metadata.yaml"
+    charm_name = get_charm_name(metadata_file)
+
+    return (charm_dir / f"{charm_name}_ubuntu-20.04-amd64.charm").absolute()
+
+
+def get_resources_from_charm_dir(charm_dir: Path) -> Dict[str, str]:
+    """Returns the resources of the charm at path"""
+    metadata_file = charm_dir / "metadata.yaml"
+    metadata = yaml.safe_load(Path(metadata_file).read_text())
+    resources = metadata["resources"]
+    return {k: v["upstream-source"] for k, v in resources.items()}
+
+
+def get_resources_from_charm_file(charm_file: str) -> Dict[str, str]:
+    """Extracts the resources of a charm from a .charm (zipped) file."""
+    with ZipFile(charm_file, "r") as zip:
+        metadata_file = zip.open("metadata.yaml")
+        metadata = yaml.safe_load(metadata_file)
+        resources = metadata["resources"]
+        return {k: v["upstream-source"] for k, v in resources.items()}
+    open_charm_file = charm_file
+
+
+def localize_bundle_application(
+    bundle: dict,
+    application: str,
+    charm_dir: Optional[Path] = None,
+    charm_file: Optional[Path] = None,
+    resources: Optional[dict] = None,
+):
+    """Localize an application in a bundle, replacing its charm and resource with local files
+
+    TODO: better docstring
+    charm_file and resources can optionally be provided, otherwise they will be inferred from
+    charm_dir.  If we provide charm_file and not resources, resources will be inferred from the
+    metadata.yaml file in the charm_file.
+    """
+    bundle = copy.deepcopy(bundle)
+
+    if not (charm_file or charm_dir):
+        raise ValueError("Either charm_file or charm_dir must be provided")
+
+    if charm_file:
+        if not resources:
+            resources = get_resources_from_charm_file(charm_file)
+    else:
+        charm_file = get_charm_file(charm_dir)
+        if not resources:
+            resources = get_resources_from_charm_dir(charm_dir)
+
+    bundle["applications"][application]["charm"] = str(charm_file)
+    bundle["applications"][application]["resources"] = resources
+    bundle["applications"][application]["_channel"] = bundle["applications"][application][
+        "channel"
+    ]
+    del bundle["applications"][application]["channel"]
+
+    return bundle
+
+
+def main(bundle_file: str, application: str, charm_dir: str, output_file: str):
+    bundle = yaml.safe_load(Path(bundle_file).read_text())
+    charm_dir = Path(charm_dir)
+    output_bundle = localize_bundle_application(bundle, application, charm_dir)
+
+    with open(output_file, "w") as fout:
+        yaml.dump(output_bundle, fout)
+
+
+if __name__ == "__main__":
+    import typer
+
+    typer.run(main)