diff --git a/monitoring/api/v3/api-client/custom_metric_test.py b/monitoring/api/v3/api-client/custom_metric_test.py index 5d51c15ec8f3..1b2627a8d336 100644 --- a/monitoring/api/v3/api-client/custom_metric_test.py +++ b/monitoring/api/v3/api-client/custom_metric_test.py @@ -24,10 +24,10 @@ import random import time -from gcp_devrel.testing import eventually_consistent -from flaky import flaky +import backoff import googleapiclient.discovery import pytest +from googleapiclient.errors import HttpError from custom_metric import create_custom_metric from custom_metric import delete_metric_descriptor @@ -35,6 +35,7 @@ from custom_metric import read_timeseries from custom_metric import write_timeseries_value + PROJECT = os.environ['GCLOUD_PROJECT'] """ Custom metric domain for all custom metrics""" @@ -52,7 +53,6 @@ def client(): return googleapiclient.discovery.build('monitoring', 'v3') -@flaky def test_custom_metric(client): PROJECT_RESOURCE = "projects/{}".format(PROJECT) # Use a constant seed so psuedo random number is known ahead of time @@ -64,29 +64,42 @@ def test_custom_metric(client): INSTANCE_ID = "test_instance" METRIC_KIND = "GAUGE" - custom_metric_descriptor = create_custom_metric( - client, PROJECT_RESOURCE, METRIC_RESOURCE, METRIC_KIND) - - # wait until metric has been created, use the get call to wait until - # a response comes back with the new metric - custom_metric = None - while not custom_metric: - time.sleep(1) - custom_metric = get_custom_metric( - client, PROJECT_RESOURCE, METRIC_RESOURCE) - - write_timeseries_value(client, PROJECT_RESOURCE, - METRIC_RESOURCE, INSTANCE_ID, - METRIC_KIND) - - # Sometimes on new metric descriptors, writes have a delay in being - # read back. Use eventually_consistent to account for this. - @eventually_consistent.call - def _(): - response = read_timeseries(client, PROJECT_RESOURCE, METRIC_RESOURCE) - value = int( - response['timeSeries'][0]['points'][0]['value']['int64Value']) - # using seed of 1 will create a value of 1 - assert value == pseudo_random_value - - delete_metric_descriptor(client, custom_metric_descriptor['name']) + try: + custom_metric_descriptor = create_custom_metric( + client, PROJECT_RESOURCE, METRIC_RESOURCE, METRIC_KIND) + + # wait until metric has been created, use the get call to wait until + # a response comes back with the new metric with 10 retries. + custom_metric = None + retry_count = 0 + while not custom_metric and retry_count < 10: + time.sleep(1) + retry_count += 1 + custom_metric = get_custom_metric( + client, PROJECT_RESOURCE, METRIC_RESOURCE) + # Make sure we get the custom metric + assert custom_metric + + write_timeseries_value(client, PROJECT_RESOURCE, + METRIC_RESOURCE, INSTANCE_ID, + METRIC_KIND) + + # Sometimes on new metric descriptors, writes have a delay in being + # read back. Use eventually_consistent to account for this. + @backoff.on_exception( + backoff.expo, (AssertionError, HttpError), max_time=120) + def eventually_consistent_test(): + response = read_timeseries( + client, PROJECT_RESOURCE, METRIC_RESOURCE) + # Make sure the value is not empty. + assert 'timeSeries' in response + value = int( + response['timeSeries'][0]['points'][0]['value']['int64Value']) + # using seed of 1 will create a value of 1 + assert value == pseudo_random_value + + eventually_consistent_test() + + finally: + # cleanup + delete_metric_descriptor(client, custom_metric_descriptor['name']) diff --git a/monitoring/api/v3/api-client/requirements-test.txt b/monitoring/api/v3/api-client/requirements-test.txt index 6e69b9d07159..40857e816e6b 100644 --- a/monitoring/api/v3/api-client/requirements-test.txt +++ b/monitoring/api/v3/api-client/requirements-test.txt @@ -1,4 +1,3 @@ +backoff==1.10.0 pytest==5.3.2 -gcp-devrel-py-tools==0.0.15 -google-cloud-core==1.3.0 flaky==3.6.1 diff --git a/monitoring/api/v3/cloud-client/quickstart_test.py b/monitoring/api/v3/cloud-client/quickstart_test.py index 2befff9f07d9..045ef3c443f5 100644 --- a/monitoring/api/v3/cloud-client/quickstart_test.py +++ b/monitoring/api/v3/cloud-client/quickstart_test.py @@ -14,9 +14,9 @@ import os +import backoff import mock import pytest -from gcp_devrel.testing import eventually_consistent import quickstart @@ -37,8 +37,10 @@ def mock_project_path(): def test_quickstart(capsys, mock_project_path): - @eventually_consistent.call - def _(): + @backoff.on_exception(backoff.expo, AssertionError, max_time=60) + def eventually_consistent_test(): quickstart.run_quickstart() out, _ = capsys.readouterr() assert 'wrote' in out + + eventually_consistent_test() diff --git a/monitoring/api/v3/cloud-client/requirements-test.txt b/monitoring/api/v3/cloud-client/requirements-test.txt index 53eb762b9949..adf26b9f98bb 100644 --- a/monitoring/api/v3/cloud-client/requirements-test.txt +++ b/monitoring/api/v3/cloud-client/requirements-test.txt @@ -1,4 +1,3 @@ +backoff==1.10.0 pytest==5.3.2 mock==3.0.5 -gcp-devrel-py-tools==0.0.15 -google-cloud-core==1.3.0 diff --git a/monitoring/api/v3/cloud-client/snippets_test.py b/monitoring/api/v3/cloud-client/snippets_test.py index a6676cdf5669..117026959e3e 100644 --- a/monitoring/api/v3/cloud-client/snippets_test.py +++ b/monitoring/api/v3/cloud-client/snippets_test.py @@ -16,12 +16,12 @@ import re import pytest -from gcp_devrel.testing import eventually_consistent +import backoff from google.api_core.exceptions import NotFound - import snippets + PROJECT_ID = os.environ['GCLOUD_PROJECT'] @@ -48,12 +48,14 @@ def write_time_series(): def test_get_delete_metric_descriptor(capsys, custom_metric_descriptor): try: - @eventually_consistent.call - def __(): + @backoff.on_exception( + backoff.expo, (AssertionError, NotFound), max_time=60) + def eventually_consistent_test(): snippets.get_metric_descriptor(custom_metric_descriptor) + out, _ = capsys.readouterr() + assert 'DOUBLE' in out - out, _ = capsys.readouterr() - assert 'DOUBLE' in out + eventually_consistent_test() finally: snippets.delete_metric_descriptor(custom_metric_descriptor) out, _ = capsys.readouterr()