From 456c5a1b960557a8c7f28efd76d81cfdd22faa45 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 2 Apr 2020 16:55:48 -0700 Subject: [PATCH 01/10] added exponential backoff --- kms/api-client/snippets_test.py | 35 ++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/kms/api-client/snippets_test.py b/kms/api-client/snippets_test.py index 9da03d9c5f2b..27441f08fbdb 100644 --- a/kms/api-client/snippets_test.py +++ b/kms/api-client/snippets_test.py @@ -16,7 +16,7 @@ import time from os import environ -from google.api_core.exceptions import GoogleAPICallError +from google.api_core.exceptions import GoogleAPICallError, Aborted from google.cloud import kms_v1 from google.cloud.kms_v1 import enums from google.iam.v1.policy_pb2 import Policy @@ -68,6 +68,7 @@ class TestKMSSnippets: parent = 'projects/{}/locations/{}'.format(project_id, location) keyring_path = '{}/keyRings/{}'.format(parent, keyring_id) version = '1' + try_limit = 10 symId = 'symmetric' @@ -177,10 +178,18 @@ def test_key_policy(self): self.symId, self.member, self.role) - policy = snippets.get_crypto_key_policy(self.project_id, - self.location, - self.keyring_id, - self.symId) + try_number = 0 + policy = None + while policy is None and try_number < self.try_limit: + try: + time.sleep(2*try_number) + policy = snippets.get_crypto_key_policy(self.project_id, + self.location, + self.keyring_id, + self.symId) + except Aborted: + # aborted by backend. Try again + try_number += 1 found = False for b in list(policy.bindings): if b.role == self.role and self.member in b.members: @@ -193,10 +202,18 @@ def test_key_policy(self): self.symId, self.member, self.role) - policy = snippets.get_crypto_key_policy(self.project_id, - self.location, - self.keyring_id, - self.symId) + try_number = 0 + policy = None + while policy is None and try_number < self.try_limit: + try: + time.sleep(2*try_number) + policy = snippets.get_crypto_key_policy(self.project_id, + self.location, + self.keyring_id, + self.symId) + except Aborted: + # aborted by backend. Try again + try_number += 1 found = False for b in list(policy.bindings): if b.role == self.role and self.member in b.members: From 3a3d313bf404e87f19b65f4aad74126b6781aa27 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Thu, 2 Apr 2020 17:08:56 -0700 Subject: [PATCH 02/10] changed import order --- kms/api-client/snippets_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kms/api-client/snippets_test.py b/kms/api-client/snippets_test.py index 27441f08fbdb..790ec4aa9cf8 100644 --- a/kms/api-client/snippets_test.py +++ b/kms/api-client/snippets_test.py @@ -16,7 +16,7 @@ import time from os import environ -from google.api_core.exceptions import GoogleAPICallError, Aborted +from google.api_core.exceptions import Aborted, GoogleAPICallError from google.cloud import kms_v1 from google.cloud.kms_v1 import enums from google.iam.v1.policy_pb2 import Policy From c6fda93c77d1a57363d48360aa294785e2600c11 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 3 Apr 2020 14:50:38 -0700 Subject: [PATCH 03/10] use eventually consistent --- kms/api-client/snippets_test.py | 61 ++++++++++++++------------------- 1 file changed, 26 insertions(+), 35 deletions(-) diff --git a/kms/api-client/snippets_test.py b/kms/api-client/snippets_test.py index 790ec4aa9cf8..3bbafb14dd83 100644 --- a/kms/api-client/snippets_test.py +++ b/kms/api-client/snippets_test.py @@ -25,6 +25,8 @@ import snippets +from gcp_devrel.testing import eventually_consistent + def create_key_helper(key_id, purpose, algorithm, t): try: @@ -68,7 +70,6 @@ class TestKMSSnippets: parent = 'projects/{}/locations/{}'.format(project_id, location) keyring_path = '{}/keyRings/{}'.format(parent, keyring_id) version = '1' - try_limit = 10 symId = 'symmetric' @@ -178,23 +179,18 @@ def test_key_policy(self): self.symId, self.member, self.role) - try_number = 0 - policy = None - while policy is None and try_number < self.try_limit: - try: - time.sleep(2*try_number) - policy = snippets.get_crypto_key_policy(self.project_id, - self.location, - self.keyring_id, - self.symId) - except Aborted: - # aborted by backend. Try again - try_number += 1 - found = False - for b in list(policy.bindings): - if b.role == self.role and self.member in b.members: - found = True - assert found + + @eventually_consistent.call(exceptions=[Aborted, AssertionError]) + def _(): + policy = snippets.get_crypto_key_policy(self.project_id, + self.location, + self.keyring_id, + self.symId) + found = False + for b in list(policy.bindings): + if b.role == self.role and self.member in b.members: + found = True + assert found # remove member snippets.remove_member_from_crypto_key_policy(self.project_id, self.location, @@ -202,23 +198,18 @@ def test_key_policy(self): self.symId, self.member, self.role) - try_number = 0 - policy = None - while policy is None and try_number < self.try_limit: - try: - time.sleep(2*try_number) - policy = snippets.get_crypto_key_policy(self.project_id, - self.location, - self.keyring_id, - self.symId) - except Aborted: - # aborted by backend. Try again - try_number += 1 - found = False - for b in list(policy.bindings): - if b.role == self.role and self.member in b.members: - found = True - assert not found + + @eventually_consistent.call(exceptions=[Aborted, AssertionError]) + def _(): + policy = snippets.get_crypto_key_policy(self.project_id, + self.location, + self.keyring_id, + self.symId) + found = False + for b in list(policy.bindings): + if b.role == self.role and self.member in b.members: + found = True + assert not found def test_symmetric_encrypt_decrypt(self): cipher_bytes = snippets.encrypt_symmetric(self.project_id, From 9f1140687ebd00cc2fe0daf22e4329a0c17a430c Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 3 Apr 2020 15:52:02 -0700 Subject: [PATCH 04/10] added devreltools to requirements-test --- kms/api-client/requirements-test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/kms/api-client/requirements-test.txt b/kms/api-client/requirements-test.txt index 781d4326c947..1cfcb6852802 100644 --- a/kms/api-client/requirements-test.txt +++ b/kms/api-client/requirements-test.txt @@ -1 +1,2 @@ pytest==5.3.2 +gcp-devrel-py-tools==0.0.15 From 0b3821bb5ef1b09e1f9c1e736c3d47933419aba2 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 3 Apr 2020 17:42:41 -0700 Subject: [PATCH 05/10] added missing dependency --- kms/api-client/requirements-test.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/kms/api-client/requirements-test.txt b/kms/api-client/requirements-test.txt index 1cfcb6852802..758bc040f3a8 100644 --- a/kms/api-client/requirements-test.txt +++ b/kms/api-client/requirements-test.txt @@ -1,2 +1,3 @@ pytest==5.3.2 gcp-devrel-py-tools==0.0.15 +google-cloud-core From 48f645ec95b68236e024c3b5d13c6d6f839d7fca Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Fri, 3 Apr 2020 17:42:52 -0700 Subject: [PATCH 06/10] fixed eventually_consistent usage --- kms/api-client/snippets_test.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/kms/api-client/snippets_test.py b/kms/api-client/snippets_test.py index 3bbafb14dd83..e1961afc0752 100644 --- a/kms/api-client/snippets_test.py +++ b/kms/api-client/snippets_test.py @@ -180,8 +180,7 @@ def test_key_policy(self): self.member, self.role) - @eventually_consistent.call(exceptions=[Aborted, AssertionError]) - def _(): + def check_policy(): policy = snippets.get_crypto_key_policy(self.project_id, self.location, self.keyring_id, @@ -191,6 +190,8 @@ def _(): if b.role == self.role and self.member in b.members: found = True assert found + eventually_consistent.call(check_policy, + exceptions=[Aborted, AssertionError]) # remove member snippets.remove_member_from_crypto_key_policy(self.project_id, self.location, @@ -199,8 +200,7 @@ def _(): self.member, self.role) - @eventually_consistent.call(exceptions=[Aborted, AssertionError]) - def _(): + def check_policy(): policy = snippets.get_crypto_key_policy(self.project_id, self.location, self.keyring_id, @@ -210,6 +210,8 @@ def _(): if b.role == self.role and self.member in b.members: found = True assert not found + eventually_consistent.call(check_policy, + exceptions=[Aborted, AssertionError]) def test_symmetric_encrypt_decrypt(self): cipher_bytes = snippets.encrypt_symmetric(self.project_id, From 5145ef04508f95ee85fa32da8b7f52f6aa3b3d4c Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 7 Apr 2020 10:28:12 -0700 Subject: [PATCH 07/10] use tuple instead of list --- kms/api-client/snippets_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kms/api-client/snippets_test.py b/kms/api-client/snippets_test.py index e1961afc0752..80632c6c9942 100644 --- a/kms/api-client/snippets_test.py +++ b/kms/api-client/snippets_test.py @@ -191,7 +191,7 @@ def check_policy(): found = True assert found eventually_consistent.call(check_policy, - exceptions=[Aborted, AssertionError]) + exceptions=(Aborted, AssertionError)) # remove member snippets.remove_member_from_crypto_key_policy(self.project_id, self.location, @@ -211,7 +211,7 @@ def check_policy(): found = True assert not found eventually_consistent.call(check_policy, - exceptions=[Aborted, AssertionError]) + exceptions=(Aborted, AssertionError)) def test_symmetric_encrypt_decrypt(self): cipher_bytes = snippets.encrypt_symmetric(self.project_id, From 077e34cf79686abc5d44e23234b8c425f3a64989 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 7 Apr 2020 10:29:18 -0700 Subject: [PATCH 08/10] snake case --- kms/api-client/snippets_test.py | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/kms/api-client/snippets_test.py b/kms/api-client/snippets_test.py index 80632c6c9942..d73c1ce8ad24 100644 --- a/kms/api-client/snippets_test.py +++ b/kms/api-client/snippets_test.py @@ -53,7 +53,7 @@ def setup_module(module): except GoogleAPICallError: # keyring already exists pass - s = create_key_helper(t.symId, + s = create_key_helper(t.sym_id, enums.CryptoKey.CryptoKeyPurpose.ENCRYPT_DECRYPT, enums.CryptoKeyVersion.CryptoKeyVersionAlgorithm. GOOGLE_SYMMETRIC_ENCRYPTION, @@ -71,9 +71,9 @@ class TestKMSSnippets: keyring_path = '{}/keyRings/{}'.format(parent, keyring_id) version = '1' - symId = 'symmetric' + sym_id = 'symmetric' - sym = '{}/cryptoKeys/{}'.format(keyring_path, symId) + sym = '{}/cryptoKeys/{}'.format(keyring_path, sym_id) sym_version = '{}/cryptoKeyVersions/{}'.format(sym, version) message = 'test message 123' @@ -96,7 +96,7 @@ def test_create_key_ring(self): @pytest.mark.skip(reason="Deleting keys isn't instant, so we should avoid \ creating a large number of them in our tests") def test_create_crypto_key(self): - key_id = self.symId + '-test' + str(int(time.time())) + key_id = self.sym_id + '-test' + str(int(time.time())) snippets.create_crypto_key(self.project_id, self.location, self.keyring_id, key_id) c = kms_v1.KeyManagementServiceClient() @@ -110,30 +110,30 @@ def test_create_crypto_key(self): def test_key_change_version_state(self): client = kms_v1.KeyManagementServiceClient() name = client.crypto_key_version_path(self.project_id, self.location, - self.keyring_id, self.symId, + self.keyring_id, self.sym_id, self.version) state_enum = enums.CryptoKeyVersion.CryptoKeyVersionState # test disable snippets.disable_crypto_key_version(self.project_id, self.location, - self.keyring_id, self.symId, + self.keyring_id, self.sym_id, self.version) response = client.get_crypto_key_version(name) assert response.state == state_enum.DISABLED # test destroy snippets.destroy_crypto_key_version(self.project_id, self.location, - self.keyring_id, self.symId, + self.keyring_id, self.sym_id, self.version) response = client.get_crypto_key_version(name) assert response.state == state_enum.DESTROY_SCHEDULED # test restore snippets.restore_crypto_key_version(self.project_id, self.location, - self.keyring_id, self.symId, + self.keyring_id, self.sym_id, self.version) response = client.get_crypto_key_version(name) assert response.state == state_enum.DISABLED # test re-enable snippets.enable_crypto_key_version(self.project_id, self.location, - self.keyring_id, self.symId, + self.keyring_id, self.sym_id, self.version) response = client.get_crypto_key_version(name) assert response.state == state_enum.ENABLED @@ -176,7 +176,7 @@ def test_key_policy(self): snippets.add_member_to_crypto_key_policy(self.project_id, self.location, self.keyring_id, - self.symId, + self.sym_id, self.member, self.role) @@ -184,7 +184,7 @@ def check_policy(): policy = snippets.get_crypto_key_policy(self.project_id, self.location, self.keyring_id, - self.symId) + self.sym_id) found = False for b in list(policy.bindings): if b.role == self.role and self.member in b.members: @@ -196,7 +196,7 @@ def check_policy(): snippets.remove_member_from_crypto_key_policy(self.project_id, self.location, self.keyring_id, - self.symId, + self.sym_id, self.member, self.role) @@ -204,7 +204,7 @@ def check_policy(): policy = snippets.get_crypto_key_policy(self.project_id, self.location, self.keyring_id, - self.symId) + self.sym_id) found = False for b in list(policy.bindings): if b.role == self.role and self.member in b.members: @@ -217,12 +217,12 @@ def test_symmetric_encrypt_decrypt(self): cipher_bytes = snippets.encrypt_symmetric(self.project_id, self.location, self.keyring_id, - self.symId, + self.sym_id, self.message_bytes) plain_bytes = snippets.decrypt_symmetric(self.project_id, self.location, self.keyring_id, - self.symId, + self.sym_id, cipher_bytes) assert plain_bytes == self.message_bytes assert cipher_bytes != self.message_bytes From 7ab519775f47931e9179093cb01b3a394301e3bf Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 7 Apr 2020 10:38:52 -0700 Subject: [PATCH 09/10] removed hanging indents --- kms/api-client/snippets_test.py | 43 ++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/kms/api-client/snippets_test.py b/kms/api-client/snippets_test.py index d73c1ce8ad24..239a48783e67 100644 --- a/kms/api-client/snippets_test.py +++ b/kms/api-client/snippets_test.py @@ -173,18 +173,20 @@ def test_ring_policy(self): # tests get/add/remove policy members def test_key_policy(self): # add member - snippets.add_member_to_crypto_key_policy(self.project_id, - self.location, - self.keyring_id, - self.sym_id, - self.member, - self.role) + snippets.add_member_to_crypto_key_policy( + self.project_id, + self.location, + self.keyring_id, + self.sym_id, + self.member, + self.role) def check_policy(): - policy = snippets.get_crypto_key_policy(self.project_id, - self.location, - self.keyring_id, - self.sym_id) + policy = snippets.get_crypto_key_policy( + self.project_id, + self.location, + self.keyring_id, + self.sym_id) found = False for b in list(policy.bindings): if b.role == self.role and self.member in b.members: @@ -194,24 +196,25 @@ def check_policy(): exceptions=(Aborted, AssertionError)) # remove member snippets.remove_member_from_crypto_key_policy(self.project_id, - self.location, - self.keyring_id, - self.sym_id, - self.member, - self.role) + self.location, + self.keyring_id, + self.sym_id, + self.member, + self.role) def check_policy(): policy = snippets.get_crypto_key_policy(self.project_id, - self.location, - self.keyring_id, - self.sym_id) + self.location, + self.keyring_id, + self.sym_id) found = False for b in list(policy.bindings): if b.role == self.role and self.member in b.members: found = True assert not found - eventually_consistent.call(check_policy, - exceptions=(Aborted, AssertionError)) + eventually_consistent.call( + check_policy, + exceptions=(Aborted, AssertionError)) def test_symmetric_encrypt_decrypt(self): cipher_bytes = snippets.encrypt_symmetric(self.project_id, From 34a6261b81614ee2becfaf5ae519403731454568 Mon Sep 17 00:00:00 2001 From: Daniel Sanche Date: Tue, 7 Apr 2020 10:53:34 -0700 Subject: [PATCH 10/10] fixed indents --- kms/api-client/snippets_test.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kms/api-client/snippets_test.py b/kms/api-client/snippets_test.py index 239a48783e67..9b2c7ad462a8 100644 --- a/kms/api-client/snippets_test.py +++ b/kms/api-client/snippets_test.py @@ -195,7 +195,8 @@ def check_policy(): eventually_consistent.call(check_policy, exceptions=(Aborted, AssertionError)) # remove member - snippets.remove_member_from_crypto_key_policy(self.project_id, + snippets.remove_member_from_crypto_key_policy( + self.project_id, self.location, self.keyring_id, self.sym_id, @@ -203,7 +204,8 @@ def check_policy(): self.role) def check_policy(): - policy = snippets.get_crypto_key_policy(self.project_id, + policy = snippets.get_crypto_key_policy( + self.project_id, self.location, self.keyring_id, self.sym_id)