Improve the test reliability by modularization (#104)

A variety of measures to improve the reliability of integration tests include:

- Increasing the margin of safety when waiting
- Providing a more accurate timeout deadline in the charm code
- Reducing the amount of unnecessary and insignificant integration tests
- Modularizing integration tests

.github/workflows/integration_test.yaml

@@ -1,69 +1,64 @@
-name: Integration tests
+name: Integration Test
 
 on:
   pull_request:
   workflow_call:
 
 jobs:
-  get-runner-image:
-    name: Get runner image
-    uses: canonical/operator-workflows/.github/workflows/get_runner_image.yaml@main
-  build-images:
-    name: Build images
-    uses: canonical/operator-workflows/.github/workflows/build_images.yaml@main
-    needs: get-runner-image
-    with:
-      owner: ${{ github.repository_owner }}
-      registry: ghcr.io
-      runs-on: ${{ needs.get-runner-image.outputs.runs-on }}
-      trivy-image-config: trivy.yaml
-  integration-tests-with-secrets:
-    uses: ./.github/workflows/integration_test_secrets.yaml
-    needs: [get-runner-image, build-images]
+  integration-test:
+    name: Integration Test
+    uses: canonical/operator-workflows/.github/workflows/integration_test.yaml@main
     secrets: inherit
     with:
-      images: ${{ needs.build-images.outputs.images }}
-      runs-on: ${{ needs.get-runner-image.outputs.runs-on }}
-  integration-tests:
-    strategy:
-      fail-fast: false
-      matrix:
-        args:
-          - name: num_units=3, db_from_config
-            value: "--test-db-from-config --num-units=3"
-          - name: num_units=1, db_from_config
-            value: "--test-db-from-config --num-units=1"
-          - name: num_units=3, db_from_relation
-            value: "--num-units=3"
-          - name: num_units=1, db_from_relation
-            value: "--num-units=1"
-    uses: canonical/operator-workflows/.github/workflows/integration_test_run.yaml@main
-    needs: [get-runner-image, build-images]
-    with:
-      images: ${{ needs.build-images.outputs.images }}
-      extra-arguments: |
-        -m "not (requires_secret)" \
-        --openstack-rc ${GITHUB_WORKSPACE}/openrc \
-        --kube-config ${GITHUB_WORKSPACE}/kube-config \
-        --screenshot-dir /tmp \
-        ${{ matrix.args.value }}
-      modules: '["test_core", "test_error", "test_sidecar_upgrade", "test_podspec_upgrade"]'
-      owner: ${{ github.repository_owner }}
-      registry: ghcr.io
-      runs-on: ${{ needs.get-runner-image.outputs.runs-on }}
+      extra-arguments: >
+        -m "not (requires_secret)"
+        --openstack-rc ${GITHUB_WORKSPACE}/openrc
+        --kube-config ${GITHUB_WORKSPACE}/kube-config
+        --screenshot-dir /tmp
+      modules: '["test_addon", "test_core", "test_cos", "test_db_config", "test_error", "test_ingress"]'
       pre-run-script: |
         -c "sudo microk8s enable hostpath-storage
           sudo microk8s kubectl -n kube-system rollout status -w deployment/hostpath-provisioner
           sudo microk8s config > ${GITHUB_WORKSPACE}/kube-config"
       setup-devstack-swift: true
-  required_status_checks:
-    name: Required Integration Test Status Checks
+      trivy-image-config: ./trivy.yaml
+
+  integration-test-with-secrets:
     runs-on: ubuntu-latest
-    needs:
-      - integration-tests
-      - integration-tests-with-secrets
-    if: always()
+    name: Integration Test (With Secrets)
     steps:
-      - run: |
-          [ '${{ needs.integration-tests-with-secrets.result }}' = 'success' ] || (echo integration-test-with-secrets failed && false)
-          [ '${{ needs.integration-tests.result }}' = 'success' ] || (echo integration-test failed && false)
+      - uses: actions/checkout@v3
+      - name: Setup Devstack Swift
+        id: setup-devstack-swift
+        uses: canonical/setup-devstack-swift@v1
+      - name: Create OpenStack credential file
+        run: echo "${{ steps.setup-devstack-swift.outputs.credentials }}" > openrc
+      - name: Setup operator environment
+        uses: charmed-kubernetes/actions-operator@main
+        with:
+          provider: microk8s
+      - name: Enable microk8s plugins
+        run: |
+          sudo microk8s enable hostpath-storage ingress registry
+          sudo microk8s kubectl -n kube-system rollout status -w deployment/hostpath-provisioner
+          sudo microk8s kubectl -n ingress rollout status -w daemonset.apps/nginx-ingress-microk8s-controller
+          sudo microk8s kubectl -n container-registry rollout status -w deployment.apps/registry
+      - name: Dump microk8s config
+        run: sudo microk8s config > kube-config
+      - name: Install tox
+        run: python3 -m pip install tox
+      - name: Build docker image
+        run: |
+          docker build -t localhost:32000/wordpress:test -f wordpress.Dockerfile .
+          docker push localhost:32000/wordpress:test
+      - name: Run integration tests
+        run: >
+          tox -e integration --
+          --akismet-api-key ${{ secrets.TEST_AKISMET_API_KEY }}
+          --openid-username ${{ secrets.TEST_OPENID_USERNAME }}
+          --openid-password ${{ secrets.TEST_OPENID_PASSWORD }}
+          --launchpad-team ${{ secrets.TEST_LAUNCHPAD_TEAM }}
+          --openstack-rc ./openrc
+          --kube-config ${GITHUB_WORKSPACE}/kube-config
+          --wordpress-image localhost:32000/wordpress:test
+          -k test_external

pyproject.toml

@@ -3,7 +3,7 @@
 [tool.bandit]
 exclude_dirs = ["/venv/"]
 [tool.bandit.assert_used]
-skips = ["*/*test.py", "*/test_*.py"]
+skips = ["*/integration/helper.py", "*/*test*"]
 
 # Testing tools configuration
 [tool.coverage.run]
@@ -35,8 +35,7 @@ ignore = ["W503", "E501"]
 # DCO020, DCO030: Ignore missing args in docstring in mocks
 per-file-ignores = [
     "tests/unit/__init__.py:D104",
-    "tests/*:D205,D212",
-    "tests/unit/wordpress_mock.py:DCO020,DCO030"
+    "tests/**:DCO020,DCO030,DCO050,DCO060,D205,D212"
 ]
 docstring-convention = "google"
 # Check for properly formatted copyright header in each file

requirements.txt

@@ -1,4 +1,3 @@
-# Include python requirements here
+mysql-connector-python
 ops>=2.0.0
 requests
-mysql-connector-python

src/charm.py

@@ -739,7 +739,8 @@ def _start_server(self):
         logger.info("Ensure WordPress server is up")
         if self.unit.is_leader():
             msg = ""
-            for _ in range(max(1, self._DB_CHECK_TIMEOUT // self._DB_CHECK_INTERVAL)):
+            deadline = time.time() + self._DB_CHECK_TIMEOUT
+            while time.time() < deadline:
                 success, msg = self._test_database_connectivity()
                 if success:
                     break
@@ -750,11 +751,12 @@ def _start_server(self):
             if not self._wp_is_installed():
                 self._wp_install()
         else:
-            for _ in range(60):
+            deadline = time.time() + self._DB_CHECK_TIMEOUT
+            while time.time() < deadline:
                 if self._wp_is_installed():
                     break
                 self.unit.status = WaitingStatus("Waiting for leader unit to initialize database")
-                time.sleep(5)
+                time.sleep(self._DB_CHECK_INTERVAL)
             else:
                 raise exceptions.WordPressBlockedStatusException(
                     "leader unit failed to initialize WordPress database in given time."

tests/conftest.py

@@ -44,3 +44,5 @@ def pytest_addoption(parser: pytest.Parser):
     parser.addoption("--screenshot-dir", action="store", default="")
     # WordPress docker image built for the WordPress charm.
     parser.addoption("--wordpress-image", action="store")
+    # Pre-build charm file
+    parser.addoption("--charm-file", action="store")