feat: Retry assert_metrics_endpoint() by default (#124)

Part of canonical/metacontroller-operator#138

src/charmed_kubeflow_chisme/testing/README.md

@@ -49,7 +49,7 @@ async def test_alert_rules(ops_test):
 
 ## `assert_metrics_endpoint`
 
-Helper function to test metrics endpoints are defined in relation data bag and to verify that endpoint are defined in current defined targets, via Grafana agent API [1]. This function is using provides side of relation to get such data.
+Helper function to test metrics endpoints are defined in relation data bag and to verify that endpoint are defined in current defined targets, via Grafana agent API [1]. This function is using provides side of relation to get such data. Note that this function is retried 10 times by default.
 
 Example usage:
 ```python

src/charmed_kubeflow_chisme/testing/cos_integration.py

@@ -13,6 +13,7 @@
 from juju.model import Model
 from juju.relation import Relation
 from juju.unit import Unit
+from tenacity import retry, stop_after_attempt, wait_exponential
 
 log = logging.getLogger(__name__)
 
@@ -430,6 +431,11 @@ async def assert_alert_rules(app: Application, alert_rules: Set[str]) -> None:
     assert relation_alert_rules == alert_rules, f"{relation_alert_rules}\n!=\n{alert_rules}"
 
 
+@retry(
+    wait=wait_exponential(multiplier=1, min=1, max=10),
+    stop=stop_after_attempt(10),
+    reraise=True,
+)
 async def assert_metrics_endpoint(app: Application, metrics_port: int, metrics_path: str) -> None:
     """Check the endpoint in the relation data bag and verify its accessibility.
 

tests/unit/testing/test_cos.py

@@ -10,6 +10,7 @@
 from juju.model import Model
 from juju.relation import Endpoint, Relation
 from juju.unit import Unit
+from tenacity import stop_after_attempt
 
 from charmed_kubeflow_chisme.testing.cos_integration import (
     _check_url,
@@ -588,6 +589,9 @@ async def test_assert_metrics_endpoint_no_data(
     """Test assert function for metrics endpoint with empty data bag."""
     app = Mock(spec_set=Application)()
     mock_get_app_relation_data.return_value = {}
+    # Wait once instead of 10 times to speed up tests
+    # as per https://github.com/jd/tenacity/issues/106
+    assert_metrics_endpoint.retry.stop = stop_after_attempt(1)
 
     with pytest.raises(AssertionError, match="metrics-endpoint relation is missing 'scrape_jobs'"):
         await assert_metrics_endpoint(app, metrics_port=8000, metrics_path="/metrics")
@@ -625,6 +629,9 @@ async def test_assert_metrics_endpoint(
             "juju_unit": "dex-auth/0",
         },
     }
+    # Wait once instead of 10 times to speed up tests
+    # as per https://github.com/jd/tenacity/issues/106
+    assert_metrics_endpoint.retry.stop = stop_after_attempt(1)
 
     await assert_metrics_endpoint(app, metrics_port=5558, metrics_path="/metrics")