From 20e5bdca511fed4358a8d33a0beeb48cb6c3d8a1 Mon Sep 17 00:00:00 2001 From: Sean Kane <68240067+seankane-msft@users.noreply.github.com> Date: Tue, 29 Jun 2021 15:31:10 -0400 Subject: [PATCH] [Test] Updates to devtools utils for storage/tables (#19040) --- .../tests/_shared/testcase.py | 27 -- .../tests/_shared/testcase.py | 267 +----------------- .../tests/test_append_blob.py | 4 +- .../tests/test_append_blob_async.py | 2 +- .../tests/test_blob_access_conditions.py | 3 +- .../test_blob_access_conditions_async.py | 2 +- .../tests/test_blob_api_version.py | 4 +- .../tests/test_blob_api_version_async.py | 2 +- .../tests/test_blob_client.py | 6 +- .../tests/test_blob_client_async.py | 4 +- .../tests/test_blob_encryption.py | 3 +- .../tests/test_blob_encryption_async.py | 2 +- .../tests/test_blob_retry.py | 9 +- .../tests/test_blob_retry_async.py | 6 +- .../tests/test_blob_service_properties.py | 4 +- .../test_blob_service_properties_async.py | 2 +- .../tests/test_blob_service_stats.py | 3 +- .../tests/test_blob_service_stats_async.py | 2 +- .../tests/test_blob_storage_account.py | 3 +- .../tests/test_blob_storage_account_async.py | 2 +- .../tests/test_blob_tags.py | 4 +- .../tests/test_blob_tags_async.py | 2 +- .../tests/test_block_blob.py | 3 +- .../tests/test_block_blob_async.py | 2 +- .../tests/test_block_blob_sync_copy.py | 3 +- .../tests/test_block_blob_sync_copy_async.py | 2 +- .../tests/test_common_blob.py | 3 +- .../tests/test_common_blob_async.py | 6 +- .../tests/test_container.py | 1 + .../tests/test_container_async.py | 2 +- .../azure-storage-blob/tests/test_cpk.py | 3 +- .../tests/test_cpk_async.py | 2 +- .../azure-storage-blob/tests/test_cpk_n.py | 3 +- .../tests/test_cpk_n_async.py | 2 +- .../azure-storage-blob/tests/test_get_blob.py | 3 +- .../tests/test_get_blob_async.py | 2 +- .../tests/test_large_block_blob.py | 3 +- .../tests/test_large_block_blob_async.py | 2 +- .../tests/test_largest_block_blob.py | 3 +- .../tests/test_largest_block_blob_async.py | 2 +- .../azure-storage-blob/tests/test_logging.py | 7 +- .../tests/test_logging_async.py | 2 +- .../azure-storage-blob/tests/test_ors.py | 3 +- .../tests/test_ors_async.py | 6 +- .../tests/test_page_blob.py | 3 +- .../tests/test_page_blob_async.py | 2 +- .../tests/test_quick_query.py | 5 +- .../azure-storage-blob/tests/test_retry.py | 12 +- .../tests/test_retry_async.py | 12 +- .../tests/test_upload_chunking.py | 4 +- .../tests/_shared/testcase.py | 2 +- .../azure-data-tables/tests/test_retry.py | 7 +- .../tests/test_retry_async.py | 6 +- .../devtools_testutils/README.md | 40 +++ .../devtools_testutils/__init__.py | 5 + .../devtools_testutils/fake_credential.py | 15 + .../devtools_testutils/helpers.py | 30 ++ .../devtools_testutils/powershell_preparer.py | 3 +- .../devtools_testutils/storage/__init__.py | 5 + .../storage/aio/__init__.py | 3 + .../storage/aio}/asynctestcase.py | 29 +- .../storage/api_version_policy.py | 13 + .../devtools_testutils/storage/processors.py | 21 ++ .../storage}/service_versions.py | 4 +- .../devtools_testutils/storage/testcase.py | 205 ++++++++++++++ .../devtools_testutils/storage_testcase.py | 18 +- 66 files changed, 448 insertions(+), 424 deletions(-) create mode 100644 tools/azure-sdk-tools/devtools_testutils/README.md create mode 100644 tools/azure-sdk-tools/devtools_testutils/fake_credential.py create mode 100644 tools/azure-sdk-tools/devtools_testutils/helpers.py create mode 100644 tools/azure-sdk-tools/devtools_testutils/storage/__init__.py create mode 100644 tools/azure-sdk-tools/devtools_testutils/storage/aio/__init__.py rename {sdk/storage/azure-storage-blob/tests/_shared => tools/azure-sdk-tools/devtools_testutils/storage/aio}/asynctestcase.py (75%) create mode 100644 tools/azure-sdk-tools/devtools_testutils/storage/api_version_policy.py create mode 100644 tools/azure-sdk-tools/devtools_testutils/storage/processors.py rename {sdk/storage/azure-storage-blob/tests/_shared => tools/azure-sdk-tools/devtools_testutils/storage}/service_versions.py (90%) create mode 100644 tools/azure-sdk-tools/devtools_testutils/storage/testcase.py diff --git a/sdk/storage/azure-storage-blob-changefeed/tests/_shared/testcase.py b/sdk/storage/azure-storage-blob-changefeed/tests/_shared/testcase.py index 972c9a8b2da1..b427b770b64d 100644 --- a/sdk/storage/azure-storage-blob-changefeed/tests/_shared/testcase.py +++ b/sdk/storage/azure-storage-blob-changefeed/tests/_shared/testcase.py @@ -303,33 +303,6 @@ def skip_test_if_targeting_emulator(self): return skip_test_if_targeting_emulator -class RetryCounter(object): - def __init__(self): - self.count = 0 - - def simple_count(self, retry_context): - self.count += 1 - - -class ResponseCallback(object): - def __init__(self, status=None, new_status=None): - self.status = status - self.new_status = new_status - self.first = True - self.count = 0 - - def override_first_status(self, response): - if self.first and response.http_response.status_code == self.status: - response.http_response.status_code = self.new_status - self.first = False - self.count += 1 - - def override_status(self, response): - if response.http_response.status_code == self.status: - response.http_response.status_code = self.new_status - self.count += 1 - - class LogCaptured(object): def __init__(self, test_case=None): # accept the test case so that we may reset logging after capturing logs diff --git a/sdk/storage/azure-storage-blob/tests/_shared/testcase.py b/sdk/storage/azure-storage-blob/tests/_shared/testcase.py index 34b7e2159610..8c5db62d4efa 100644 --- a/sdk/storage/azure-storage-blob/tests/_shared/testcase.py +++ b/sdk/storage/azure-storage-blob/tests/_shared/testcase.py @@ -7,19 +7,7 @@ from __future__ import division import os.path import time -from datetime import datetime, timedelta - -try: - import unittest.mock as mock -except ImportError: - import mock - -import zlib -import math -import sys import os -import random -import re import logging from devtools_testutils import ( AzureMgmtTestCase, @@ -28,16 +16,12 @@ StorageAccountPreparer, FakeResource, ) -from azure_devtools.scenario_tests import RecordingProcessor, AzureTestError, create_random_name try: from cStringIO import StringIO # Python 2 except ImportError: from io import StringIO -from azure.core.pipeline.policies import SansIOHTTPPolicy from azure.core.exceptions import ResourceNotFoundError, HttpResponseError -from azure.core.credentials import AccessToken -from azure.storage.blob import generate_account_sas, AccountSasPermissions, ResourceTypes from azure.mgmt.storage.models import StorageAccount, Endpoints try: # Running locally - use configuration in settings_real.py @@ -46,15 +30,9 @@ # Running on the pipeline - use fake values in order to create rg, etc. from .settings_fake import * -try: - from devtools_testutils import mgmt_settings_real as settings -except ImportError: - from devtools_testutils import mgmt_settings_fake as settings - -from .service_versions import service_version_map - import pytest +from devtools_testutils.storage import StorageTestCase LOGGING_FORMAT = '%(asctime)s %(name)-20s %(levelname)-5s %(message)s' os.environ['AZURE_STORAGE_ACCOUNT_NAME'] = STORAGE_ACCOUNT_NAME @@ -63,35 +41,6 @@ os.environ['AZURE_SKIP_LIVE_RECORDING'] = os.environ.get('AZURE_SKIP_LIVE_RECORDING', None) or SKIP_LIVE_RECORDING -class FakeTokenCredential(object): - """Protocol for classes able to provide OAuth tokens. - :param str scopes: Lets you specify the type of access needed. - """ - def __init__(self): - self.token = AccessToken("YOU SHALL NOT PASS", 0) - self.get_token_count = 0 - - def get_token(self, *args): - self.get_token_count += 1 - return self.token - - -class XMSRequestIDBody(RecordingProcessor): - """This process is used for Storage batch call only, to avoid the echo policy. - """ - def process_response(self, response): - content_type = None - for key, value in response.get('headers', {}).items(): - if key.lower() == 'content-type': - content_type = (value[0] if isinstance(value, list) else value).lower() - break - - if content_type and 'multipart/mixed' in content_type: - response['body']['string'] = re.sub(b"x-ms-client-request-id: [a-f0-9-]+\r\n", b"", response['body']['string']) - - return response - - class GlobalStorageAccountPreparer(AzureMgmtPreparer): def __init__(self): super(GlobalStorageAccountPreparer, self).__init__( @@ -153,226 +102,12 @@ def create_resource(self, name, **kwargs): } -class StorageTestCase(AzureMgmtTestCase): - - def __init__(self, *args, **kwargs): - super(StorageTestCase, self).__init__(*args, **kwargs) - self.replay_processors.append(XMSRequestIDBody()) - self.logger = logging.getLogger('azure.storage') - self.configure_logging() - - def connection_string(self, account, key): - return "DefaultEndpointsProtocol=https;AcCounTName=" + account.name + ";AccOuntKey=" + str(key) + ";EndpoIntSuffix=core.windows.net" - - def account_url(self, storage_account, storage_type): - """Return an url of storage account. - - :param str storage_account: Storage account name - :param str storage_type: The Storage type part of the URL. Should be "blob", or "queue", etc. - """ - try: - if storage_type == "blob": - return storage_account.primary_endpoints.blob.rstrip("/") - if storage_type == "queue": - return storage_account.primary_endpoints.queue.rstrip("/") - if storage_type == "file": - return storage_account.primary_endpoints.file.rstrip("/") - else: - raise ValueError("Unknown storage type {}".format(storage_type)) - except AttributeError: # Didn't find "primary_endpoints" - return 'https://{}.{}.core.windows.net'.format(storage_account, storage_type) - - def configure_logging(self): - enable_logging = ENABLE_LOGGING - - self.enable_logging() if enable_logging else self.disable_logging() - - def enable_logging(self): - handler = logging.StreamHandler() - handler.setFormatter(logging.Formatter(LOGGING_FORMAT)) - self.logger.handlers = [handler] - self.logger.setLevel(logging.DEBUG) - self.logger.propagate = True - self.logger.disabled = False - - def disable_logging(self): - self.logger.propagate = False - self.logger.disabled = True - self.logger.handlers = [] - - def sleep(self, seconds): - if self.is_live: - time.sleep(seconds) - - def get_random_bytes(self, size): - # recordings don't like random stuff. making this more - # deterministic. - return b'a'*size - - def get_random_text_data(self, size): - '''Returns random unicode text data exceeding the size threshold for - chunking blob upload.''' - checksum = zlib.adler32(self.qualified_test_name.encode()) & 0xffffffff - rand = random.Random(checksum) - text = u'' - words = [u'hello', u'world', u'python', u'啊齄丂狛狜'] - while (len(text) < size): - index = int(rand.random()*(len(words) - 1)) - text = text + u' ' + words[index] - - return text - - @staticmethod - def _set_test_proxy(service, settings): - if settings.USE_PROXY: - service.set_proxy( - settings.PROXY_HOST, - settings.PROXY_PORT, - settings.PROXY_USER, - settings.PROXY_PASSWORD, - ) - - def assertNamedItemInContainer(self, container, item_name, msg=None): - def _is_string(obj): - if sys.version_info >= (3,): - return isinstance(obj, str) - else: - return isinstance(obj, basestring) - for item in container: - if _is_string(item): - if item == item_name: - return - elif isinstance(item, dict): - if item_name == item['name']: - return - elif item.name == item_name: - return - elif hasattr(item, 'snapshot') and item.snapshot == item_name: - return - - - standardMsg = '{0} not found in {1}'.format( - repr(item_name), [str(c) for c in container]) - self.fail(self._formatMessage(msg, standardMsg)) - - def assertNamedItemNotInContainer(self, container, item_name, msg=None): - for item in container: - if item.name == item_name: - standardMsg = '{0} unexpectedly found in {1}'.format( - repr(item_name), repr(container)) - self.fail(self._formatMessage(msg, standardMsg)) - - def assert_upload_progress(self, size, max_chunk_size, progress, unknown_size=False): - '''Validates that the progress chunks align with our chunking procedure.''' - index = 0 - total = None if unknown_size else size - small_chunk_size = size % max_chunk_size - self.assertEqual(len(progress), math.ceil(size / max_chunk_size)) - for i in progress: - self.assertTrue(i[0] % max_chunk_size == 0 or i[0] % max_chunk_size == small_chunk_size) - self.assertEqual(i[1], total) - - def assert_download_progress(self, size, max_chunk_size, max_get_size, progress): - '''Validates that the progress chunks align with our chunking procedure.''' - if size <= max_get_size: - self.assertEqual(len(progress), 1) - self.assertTrue(progress[0][0], size) - self.assertTrue(progress[0][1], size) - else: - small_chunk_size = (size - max_get_size) % max_chunk_size - self.assertEqual(len(progress), 1 + math.ceil((size - max_get_size) / max_chunk_size)) - - self.assertTrue(progress[0][0], max_get_size) - self.assertTrue(progress[0][1], size) - for i in progress[1:]: - self.assertTrue(i[0] % max_chunk_size == 0 or i[0] % max_chunk_size == small_chunk_size) - self.assertEqual(i[1], size) - - def generate_oauth_token(self): - if self.is_live: - from azure.identity import ClientSecretCredential - return ClientSecretCredential( - self.get_settings_value("TENANT_ID"), - self.get_settings_value("CLIENT_ID"), - self.get_settings_value("CLIENT_SECRET"), - ) - return self.generate_fake_token() - - def generate_sas_token(self): - fake_key = 'a'*30 + 'b'*30 - - return '?' + generate_account_sas( - account_name = 'test', # name of the storage account - account_key = fake_key, # key for the storage account - resource_types = ResourceTypes(object=True), - permission = AccountSasPermissions(read=True,list=True), - start = datetime.now() - timedelta(hours = 24), - expiry = datetime.now() + timedelta(days = 8) - ) - - def generate_fake_token(self): - return FakeTokenCredential() - - def _get_service_version(self, **kwargs): - env_version = service_version_map.get(os.environ.get("AZURE_LIVE_TEST_SERVICE_VERSION","LATEST")) - return kwargs.pop("service_version", env_version) - - def create_storage_client(self, client, *args, **kwargs): - kwargs["api_version"] = self._get_service_version(**kwargs) - kwargs["_additional_pipeline_policies"] = [ApiVersionAssertPolicy(kwargs["api_version"])] - return client(*args, **kwargs) - - def create_storage_client_from_conn_str(self, client, *args, **kwargs): - kwargs["api_version"] = self._get_service_version(**kwargs) - kwargs["_additional_pipeline_policies"] = [ApiVersionAssertPolicy(kwargs["api_version"])] - return client.from_connection_string(*args, **kwargs) - - -class ApiVersionAssertPolicy(SansIOHTTPPolicy): - """ - Assert the ApiVersion is set properly on the response - """ - - def __init__(self, api_version): - self.api_version = api_version - - def on_request(self, request): - assert request.http_request.headers['x-ms-version'] == self.api_version - - def not_for_emulator(test): def skip_test_if_targeting_emulator(self): test(self) return skip_test_if_targeting_emulator -class RetryCounter(object): - def __init__(self): - self.count = 0 - - def simple_count(self, retry_context): - self.count += 1 - - -class ResponseCallback(object): - def __init__(self, status=None, new_status=None): - self.status = status - self.new_status = new_status - self.first = True - self.count = 0 - - def override_first_status(self, response): - if self.first and response.http_response.status_code == self.status: - response.http_response.status_code = self.new_status - self.first = False - self.count += 1 - - def override_status(self, response): - if response.http_response.status_code == self.status: - response.http_response.status_code = self.new_status - self.count += 1 - - class LogCaptured(object): def __init__(self, test_case=None): # accept the test case so that we may reset logging after capturing logs diff --git a/sdk/storage/azure-storage-blob/tests/test_append_blob.py b/sdk/storage/azure-storage-blob/tests/test_append_blob.py index 334498438c44..4f044ffbd81b 100644 --- a/sdk/storage/azure-storage-blob/tests/test_append_blob.py +++ b/sdk/storage/azure-storage-blob/tests/test_append_blob.py @@ -26,8 +26,8 @@ BlobSasPermissions) from azure.storage.blob._shared.policies import StorageContentValidation -from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer, StorageAccountPreparer, \ - GlobalResourceGroupPreparer +from _shared.testcase import GlobalStorageAccountPreparer, StorageAccountPreparer, GlobalResourceGroupPreparer +from devtools_testutils.storage import StorageTestCase # ------------------------------------------------------------------------------ TEST_BLOB_PREFIX = 'blob' diff --git a/sdk/storage/azure-storage-blob/tests/test_append_blob_async.py b/sdk/storage/azure-storage-blob/tests/test_append_blob_async.py index 72a405bd0a36..153c0caa08f5 100644 --- a/sdk/storage/azure-storage-blob/tests/test_append_blob_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_append_blob_async.py @@ -31,7 +31,7 @@ BlobClient, ) from _shared.testcase import GlobalStorageAccountPreparer, GlobalResourceGroupPreparer, StorageAccountPreparer -from _shared.asynctestcase import AsyncStorageTestCase +from devtools_testutils.storage.aio import AsyncStorageTestCase # ------------------------------------------------------------------------------ TEST_BLOB_PREFIX = 'blob' diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions.py b/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions.py index 378b293f0996..b9d12dbefdc8 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions.py @@ -29,7 +29,8 @@ ResourceTypes, AccountSasPermissions, generate_container_sas, ContainerClient, CustomerProvidedEncryptionKey, ) -from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer +from _shared.testcase import GlobalStorageAccountPreparer +from devtools_testutils.storage import StorageTestCase # ------------------------------------------------------------------------------ LARGE_APPEND_BLOB_SIZE = 64 * 1024 diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions_async.py b/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions_async.py index da93231c4747..33c548b25fdb 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_access_conditions_async.py @@ -29,7 +29,7 @@ generate_container_sas, CustomerProvidedEncryptionKey, ) from _shared.testcase import GlobalStorageAccountPreparer -from _shared.asynctestcase import AsyncStorageTestCase +from devtools_testutils.storage.aio import AsyncStorageTestCase from azure.storage.blob.aio import ( BlobServiceClient, diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_api_version.py b/sdk/storage/azure-storage-blob/tests/test_blob_api_version.py index 759885f9d7d1..b3f7498386e9 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_api_version.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_api_version.py @@ -19,8 +19,8 @@ ) from azure.storage.blob._shared.constants import X_MS_VERSION from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer -from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer - +from _shared.testcase import GlobalStorageAccountPreparer +from devtools_testutils.storage import StorageTestCase # ------------------------------------------------------------------------------ TEST_BLOB_PREFIX = 'blob' diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_api_version_async.py b/sdk/storage/azure-storage-blob/tests/test_blob_api_version_async.py index 3a1757c6af54..d6e73cd79334 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_api_version_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_api_version_async.py @@ -17,7 +17,7 @@ BlobClient, ) from _shared.testcase import GlobalStorageAccountPreparer -from _shared.asynctestcase import AsyncStorageTestCase +from devtools_testutils.storage.aio import AsyncStorageTestCase # ------------------------------------------------------------------------------ TEST_BLOB_PREFIX = 'blob' diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_client.py b/sdk/storage/azure-storage-blob/tests/test_blob_client.py index 08e15585366c..5956f7a92e85 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_client.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_client.py @@ -16,8 +16,8 @@ BlobClient, ) from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer -from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer -#from azure.storage.common import TokenCredential +from _shared.testcase import GlobalStorageAccountPreparer +from devtools_testutils.storage import StorageTestCase # ------------------------------------------------------------------------------ SERVICES = { @@ -610,7 +610,7 @@ def test_error_with_malformed_conn_str(self): # Act with self.assertRaises(ValueError) as e: service = service_type[0].from_connection_string(conn_str, blob_name="test", container_name="foo/bar") - + if conn_str in("", "foobar", "foo;bar;baz", ";"): self.assertEqual( str(e.exception), "Connection string is either blank or malformed.") diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_client_async.py b/sdk/storage/azure-storage-blob/tests/test_blob_client_async.py index 5e9e9ed2aa8c..ac3f89d70f1b 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_client_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_client_async.py @@ -19,7 +19,7 @@ from azure.core.pipeline.transport import AioHttpTransport from multidict import CIMultiDict, CIMultiDictProxy from _shared.testcase import GlobalStorageAccountPreparer -from _shared.asynctestcase import AsyncStorageTestCase +from devtools_testutils.storage.aio import AsyncStorageTestCase # ------------------------------------------------------------------------------ SERVICES = { @@ -602,5 +602,5 @@ async def test_closing_pipeline_client_simple(self, resource_group, location, st service = client( self.account_url(storage_account, "blob"), credential=storage_account_key, container_name='foo', blob_name='bar') await service.close() - + # ------------------------------------------------------------------------------ diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption.py index dce2316c20c0..fe2b38b32b5b 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption.py @@ -41,7 +41,8 @@ KeyResolver, RSAKeyWrapper, ) -from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer +from _shared.testcase import GlobalStorageAccountPreparer +from devtools_testutils.storage import StorageTestCase # ------------------------------------------------------------------------------ TEST_CONTAINER_PREFIX = 'encryption_container' diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_async.py b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_async.py index 4fdcb3f8a418..e67220966d54 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_encryption_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_encryption_async.py @@ -44,7 +44,7 @@ RSAKeyWrapper, ) from _shared.testcase import GlobalStorageAccountPreparer -from _shared.asynctestcase import AsyncStorageTestCase +from devtools_testutils.storage.aio import AsyncStorageTestCase # ------------------------------------------------------------------------------ TEST_CONTAINER_PREFIX = 'encryption_container' diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_retry.py b/sdk/storage/azure-storage-blob/tests/test_blob_retry.py index cec89834215d..d88be2c8cece 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_retry.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_retry.py @@ -13,13 +13,10 @@ BlobClient, ExponentialRetry ) -from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer +from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer, ResponseCallback from azure.core.exceptions import ResourceExistsError, HttpResponseError -from _shared.testcase import ( - StorageTestCase, - ResponseCallback, - GlobalStorageAccountPreparer -) +from _shared.testcase import GlobalStorageAccountPreparer +from devtools_testutils.storage import StorageTestCase # test constants PUT_BLOCK_SIZE = 4 * 1024 diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_retry_async.py b/sdk/storage/azure-storage-blob/tests/test_blob_retry_async.py index 9197c46a8d2d..3e66c7003211 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_retry_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_retry_async.py @@ -14,12 +14,12 @@ ContainerClient, BlobClient, ) -from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer +from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer, ResponseCallback from azure.core.exceptions import ResourceExistsError, HttpResponseError from azure.core.pipeline.transport import AioHttpTransport from multidict import CIMultiDict, CIMultiDictProxy -from _shared.testcase import ResponseCallback, GlobalStorageAccountPreparer -from _shared.asynctestcase import AsyncStorageTestCase +from _shared.testcase import GlobalStorageAccountPreparer +from devtools_testutils.storage.aio import AsyncStorageTestCase # test constants PUT_BLOCK_SIZE = 4 * 1024 diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_service_properties.py b/sdk/storage/azure-storage-blob/tests/test_blob_service_properties.py index 9db90529b86f..a4bd7697c1ef 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_service_properties.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_service_properties.py @@ -21,8 +21,8 @@ StaticWebsite, ) -from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer - +from _shared.testcase import GlobalStorageAccountPreparer +from devtools_testutils.storage import StorageTestCase # ------------------------------------------------------------------------------ diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_service_properties_async.py b/sdk/storage/azure-storage-blob/tests/test_blob_service_properties_async.py index bd800bb4b659..342f74f25dea 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_service_properties_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_service_properties_async.py @@ -29,7 +29,7 @@ StaticWebsite, ) from _shared.testcase import GlobalStorageAccountPreparer -from _shared.asynctestcase import AsyncStorageTestCase +from devtools_testutils.storage.aio import AsyncStorageTestCase # ------------------------------------------------------------------------------ diff --git a/sdk/storage/azure-storage-blob/tests/test_blob_service_stats.py b/sdk/storage/azure-storage-blob/tests/test_blob_service_stats.py index 5a51e10461d6..1de16c8a6538 100644 --- a/sdk/storage/azure-storage-blob/tests/test_blob_service_stats.py +++ b/sdk/storage/azure-storage-blob/tests/test_blob_service_stats.py @@ -8,8 +8,9 @@ from azure.storage.blob import BlobServiceClient from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer +from devtools_testutils.storage import StorageTestCase -from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer, GlobalResourceGroupPreparer +from _shared.testcase import GlobalStorageAccountPreparer, GlobalResourceGroupPreparer SERVICE_UNAVAILABLE_RESP_BODY = 'unavailableunavailable= (3,): from urllib.parse import parse_qs, quote, urlparse diff --git a/sdk/storage/azure-storage-blob/tests/test_logging_async.py b/sdk/storage/azure-storage-blob/tests/test_logging_async.py index 20ea907b809b..d0ed5ae7e0a0 100644 --- a/sdk/storage/azure-storage-blob/tests/test_logging_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_logging_async.py @@ -32,7 +32,7 @@ from _shared.testcase import ( LogCaptured, GlobalStorageAccountPreparer ) -from _shared.asynctestcase import AsyncStorageTestCase +from devtools_testutils.storage.aio import AsyncStorageTestCase if sys.version_info >= (3,): diff --git a/sdk/storage/azure-storage-blob/tests/test_ors.py b/sdk/storage/azure-storage-blob/tests/test_ors.py index 0c5f80c3e173..7791b28a85a4 100644 --- a/sdk/storage/azure-storage-blob/tests/test_ors.py +++ b/sdk/storage/azure-storage-blob/tests/test_ors.py @@ -6,7 +6,8 @@ # license information. # -------------------------------------------------------------------------- import pytest -from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer +from _shared.testcase import GlobalStorageAccountPreparer +from devtools_testutils.storage import StorageTestCase from azure.storage.blob import ( BlobServiceClient, diff --git a/sdk/storage/azure-storage-blob/tests/test_ors_async.py b/sdk/storage/azure-storage-blob/tests/test_ors_async.py index 9ecab6354a79..f60a21370404 100644 --- a/sdk/storage/azure-storage-blob/tests/test_ors_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_ors_async.py @@ -9,8 +9,8 @@ from azure.core.pipeline.transport import AioHttpTransport from multidict import CIMultiDict, CIMultiDictProxy -from _shared.asynctestcase import AsyncStorageTestCase -from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer +from devtools_testutils.storage.aio import AsyncStorageTestCase +from _shared.testcase import GlobalStorageAccountPreparer from azure.storage.blob import BlobProperties from azure.storage.blob.aio import BlobServiceClient @@ -28,7 +28,7 @@ async def send(self, request, **config): return response -class StorageObjectReplicationTest(StorageTestCase): +class StorageObjectReplicationTest(AsyncStorageTestCase): SRC_CONTAINER = "test1" DST_CONTAINER = "test2" BLOB_NAME = "bla.txt" diff --git a/sdk/storage/azure-storage-blob/tests/test_page_blob.py b/sdk/storage/azure-storage-blob/tests/test_page_blob.py index ec8c669868f3..13c804cba882 100644 --- a/sdk/storage/azure-storage-blob/tests/test_page_blob.py +++ b/sdk/storage/azure-storage-blob/tests/test_page_blob.py @@ -28,7 +28,8 @@ generate_blob_sas) from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer from azure.storage.blob._shared.policies import StorageContentValidation -from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer, GlobalResourceGroupPreparer +from _shared.testcase import GlobalStorageAccountPreparer, GlobalResourceGroupPreparer +from devtools_testutils.storage import StorageTestCase #------------------------------------------------------------------------------ TEST_BLOB_PREFIX = 'blob' diff --git a/sdk/storage/azure-storage-blob/tests/test_page_blob_async.py b/sdk/storage/azure-storage-blob/tests/test_page_blob_async.py index 35e5890ef8ff..affdb1556bb8 100644 --- a/sdk/storage/azure-storage-blob/tests/test_page_blob_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_page_blob_async.py @@ -37,7 +37,7 @@ ) from _shared.testcase import GlobalStorageAccountPreparer, GlobalResourceGroupPreparer -from _shared.asynctestcase import AsyncStorageTestCase +from devtools_testutils.storage.aio import AsyncStorageTestCase #------------------------------------------------------------------------------ TEST_BLOB_PREFIX = 'blob' diff --git a/sdk/storage/azure-storage-blob/tests/test_quick_query.py b/sdk/storage/azure-storage-blob/tests/test_quick_query.py index cf2e1d62ed98..2df72d50240a 100644 --- a/sdk/storage/azure-storage-blob/tests/test_quick_query.py +++ b/sdk/storage/azure-storage-blob/tests/test_quick_query.py @@ -9,7 +9,8 @@ import pytest -from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer +from _shared.testcase import GlobalStorageAccountPreparer +from devtools_testutils.storage import StorageTestCase from azure.storage.blob import ( BlobServiceClient, DelimitedTextDialect, @@ -441,7 +442,7 @@ def on_error(error): data = [] for record in resp.records(): data.append(record) - + self.assertEqual(len(errors), 1) self.assertEqual(resp._size, 43) self.assertEqual(data, [b'']) diff --git a/sdk/storage/azure-storage-blob/tests/test_retry.py b/sdk/storage/azure-storage-blob/tests/test_retry.py index bcdd75691f78..7226ee45c00e 100644 --- a/sdk/storage/azure-storage-blob/tests/test_retry.py +++ b/sdk/storage/azure-storage-blob/tests/test_retry.py @@ -15,7 +15,7 @@ from azure.core.pipeline.transport import( RequestsTransport ) -from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer +from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer, RetryCounter, ResponseCallback from azure.storage.blob import ( BlobServiceClient, ContainerClient, @@ -25,19 +25,15 @@ ExponentialRetry, ) -from _shared.testcase import ( - StorageTestCase, - ResponseCallback, - RetryCounter, - GlobalStorageAccountPreparer -) +from _shared.testcase import GlobalStorageAccountPreparer +from devtools_testutils.storage import StorageTestCase class RetryRequestTransport(RequestsTransport): """Transport to test retry""" def __init__(self, *args, **kwargs): super(RetryRequestTransport, self).__init__(*args, **kwargs) self.count = 0 - + def send(self, request, **kwargs): self.count += 1 response = super(RetryRequestTransport, self).send(request, **kwargs) diff --git a/sdk/storage/azure-storage-blob/tests/test_retry_async.py b/sdk/storage/azure-storage-blob/tests/test_retry_async.py index 7c8d46d5da26..3fb0668507bf 100644 --- a/sdk/storage/azure-storage-blob/tests/test_retry_async.py +++ b/sdk/storage/azure-storage-blob/tests/test_retry_async.py @@ -28,13 +28,9 @@ BlobClient, ) -from _shared.testcase import ( - ResponseCallback, - RetryCounter, - GlobalStorageAccountPreparer -) -from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer -from _shared.asynctestcase import AsyncStorageTestCase +from _shared.testcase import GlobalStorageAccountPreparer +from devtools_testutils import ResourceGroupPreparer, StorageAccountPreparer, RetryCounter, ResponseCallback +from devtools_testutils.storage.aio import AsyncStorageTestCase class AiohttpTestTransport(AioHttpTransport): @@ -161,7 +157,7 @@ async def test_retry_on_socket_timeout_async(self, resource_group, location, sto try: with self.assertRaises(AzureError) as error: await service.create_container(container_name) - + # Assert # 3 retries + 1 original == 4 diff --git a/sdk/storage/azure-storage-blob/tests/test_upload_chunking.py b/sdk/storage/azure-storage-blob/tests/test_upload_chunking.py index 16a50b604c7d..f179cce27e76 100644 --- a/sdk/storage/azure-storage-blob/tests/test_upload_chunking.py +++ b/sdk/storage/azure-storage-blob/tests/test_upload_chunking.py @@ -14,8 +14,8 @@ from threading import Lock from io import (BytesIO, SEEK_SET) -from _shared.testcase import StorageTestCase, GlobalStorageAccountPreparer - +from _shared.testcase import GlobalStorageAccountPreparer +from devtools_testutils.storage import StorageTestCase # ------------------------------------------------------------------------------ diff --git a/sdk/tables/azure-data-tables/tests/_shared/testcase.py b/sdk/tables/azure-data-tables/tests/_shared/testcase.py index 12123d4268b1..5afa95b348ac 100644 --- a/sdk/tables/azure-data-tables/tests/_shared/testcase.py +++ b/sdk/tables/azure-data-tables/tests/_shared/testcase.py @@ -495,4 +495,4 @@ def __init__(self): self.count = 0 def simple_count(self, retry_context): - self.count += 1 + self.count += 1 \ No newline at end of file diff --git a/sdk/tables/azure-data-tables/tests/test_retry.py b/sdk/tables/azure-data-tables/tests/test_retry.py index 2e651efef0c1..bc7faf72d056 100644 --- a/sdk/tables/azure-data-tables/tests/test_retry.py +++ b/sdk/tables/azure-data-tables/tests/test_retry.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureTestCase, ResponseCallback from azure.core.exceptions import ( HttpResponseError, @@ -16,10 +16,7 @@ from azure.core.pipeline.transport import RequestsTransport from azure.data.tables import TableServiceClient -from _shared.testcase import ( - TableTestCase, - ResponseCallback, -) +from _shared.testcase import TableTestCase from preparers import tables_decorator diff --git a/sdk/tables/azure-data-tables/tests/test_retry_async.py b/sdk/tables/azure-data-tables/tests/test_retry_async.py index b1f597c4553c..a389686e2617 100644 --- a/sdk/tables/azure-data-tables/tests/test_retry_async.py +++ b/sdk/tables/azure-data-tables/tests/test_retry_async.py @@ -3,10 +3,9 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -import unittest import pytest -from devtools_testutils import AzureTestCase +from devtools_testutils import AzureTestCase, ResponseCallback from azure.core.exceptions import ( @@ -22,9 +21,6 @@ from azure.data.tables.aio import TableServiceClient from _shared.asynctestcase import AsyncTableTestCase -from _shared.testcase import ( - ResponseCallback, -) from async_preparers import tables_decorator_async diff --git a/tools/azure-sdk-tools/devtools_testutils/README.md b/tools/azure-sdk-tools/devtools_testutils/README.md new file mode 100644 index 000000000000..648c275c6f12 --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/README.md @@ -0,0 +1,40 @@ +# Devtools Testutils + +## Objects in this package for use with Azure Testing +* [`AzureMgmtTestCase`][azure_mgmt_testcase]: Base class for Management plane test classes +* [`AzureMgmtPreparer`][azure_mgmt_preparer]: Base class for Management-plane resource preparers +* [`AzureTestCase`][azure_testcase]: Base class for data plane test classes +* [`is_live`][is_live]: Helper method for determining whether a test run is in live or playback mode +* [`get_region_override`][get_region_override]: Helper method for determining resource region +* [`FakeResource`][fake_resource]: +* [`ResourceGroupPreparer`][rg_preparer]: +* [`RandomNameResourceGroupPreparer`][random_name_rg_preparer]: +* [`CachedResourceGroupPreparer`][cached_rg_preparer]: +* [`FakeStorageAccount`][fake_storage_account]: +* [`StorageAccountPreparer`][storage_account_preparer]: +* [`CachedStorageAccountPreparer`][cached_storage_account_preparer]: +* [`KeyVaultPreparer`][kv_preparer]: +* [`PowerShellPreparer`][powershell_preparer]: Abstract preparer for delivering secrets from environment variables to individual tests +* [`RetryCounter`][retry_counter]: Object for counting retries on a request. +* [`ResponseCallback`][response_callback]: Object for mocking response callbacks. +* [`FakeCredential`][fake_credential]: Fake credential used for authenticating in playback mode. + + + +[azure_mgmt_testcase]: https://github.com/Azure/azure-sdk-for-python/blob/520ea7175e10a971eae9d3e6cd0735efd80447b1/tools/azure-sdk-tools/devtools_testutils/mgmt_testcase.py#L57 +[azure_mgmt_preparer]: https://github.com/Azure/azure-sdk-for-python/blob/520ea7175e10a971eae9d3e6cd0735efd80447b1/tools/azure-sdk-tools/devtools_testutils/mgmt_testcase.py#L128 +[azure_testcase]: https://github.com/Azure/azure-sdk-for-python/blob/520ea7175e10a971eae9d3e6cd0735efd80447b1/tools/azure-sdk-tools/devtools_testutils/azure_testcase.py#L104 +[is_live]: https://github.com/Azure/azure-sdk-for-python/blob/520ea7175e10a971eae9d3e6cd0735efd80447b1/tools/azure-sdk-tools/devtools_testutils/azure_testcase.py#L77 +[get_region_override]: https://github.com/Azure/azure-sdk-for-python/blob/520ea7175e10a971eae9d3e6cd0735efd80447b1/tools/azure-sdk-tools/devtools_testutils/azure_testcase.py#L87 +[fake_resource]: https://github.com/Azure/azure-sdk-for-python/blob/master/tools/azure-sdk-tools/devtools_testutils/resource_testcase.py#L27 +[rg_preparer]: https://github.com/Azure/azure-sdk-for-python/blob/520ea7175e10a971eae9d3e6cd0735efd80447b1/tools/azure-sdk-tools/devtools_testutils/resource_testcase.py#L30 +[random_name_rg_preparer]: https://github.com/Azure/azure-sdk-for-python/blob/master/tools/azure-sdk-tools/devtools_testutils/resource_testcase.py#L119 +[cached_rg_preparer]: https://github.com/Azure/azure-sdk-for-python/blob/master/tools/azure-sdk-tools/devtools_testutils/resource_testcase.py#L120 +[fake_storage_account]: https://github.com/Azure/azure-sdk-for-python/blob/master/tools/azure-sdk-tools/devtools_testutils/storage_testcase.py#L25 +[storage_account_preparer]: https://github.com/Azure/azure-sdk-for-python/blob/520ea7175e10a971eae9d3e6cd0735efd80447b1/tools/azure-sdk-tools/devtools_testutils/storage_testcase.py#L29 +[cached_storage_account_preparer]: https://github.com/Azure/azure-sdk-for-python/blob/master/tools/azure-sdk-tools/devtools_testutils/storage_testcase.py#L140 +[kv_preparer]: https://github.com/Azure/azure-sdk-for-python/blob/520ea7175e10a971eae9d3e6cd0735efd80447b1/tools/azure-sdk-tools/devtools_testutils/keyvault_preparer.py#L49 +[powershell_preparer]: https://github.com/Azure/azure-sdk-for-python/blob/520ea7175e10a971eae9d3e6cd0735efd80447b1/tools/azure-sdk-tools/devtools_testutils/powershell_preparer.py#L14 +[retry_counter]: https://github.com/Azure/azure-sdk-for-python/blob/ab7e7f1a7b2a6d7255abdc77a40e2d6a86c9de0a/tools/azure-sdk-tools/devtools_testutils/helpers.py#L6 +[response_callback]: https://github.com/Azure/azure-sdk-for-python/blob/ab7e7f1a7b2a6d7255abdc77a40e2d6a86c9de0a/tools/azure-sdk-tools/devtools_testutils/helpers.py#L14 +[fake_credential]: https://github.com/Azure/azure-sdk-for-python/blob/65ffc49fbdd0f4f83e68eb5c8e0c6d293f0569cd/tools/azure-sdk-tools/devtools_testutils/fake_credential.py \ No newline at end of file diff --git a/tools/azure-sdk-tools/devtools_testutils/__init__.py b/tools/azure-sdk-tools/devtools_testutils/__init__.py index 1a4bcf0ed1e0..640da9b62531 100644 --- a/tools/azure-sdk-tools/devtools_testutils/__init__.py +++ b/tools/azure-sdk-tools/devtools_testutils/__init__.py @@ -14,6 +14,8 @@ ) from .keyvault_preparer import KeyVaultPreparer from .powershell_preparer import PowerShellPreparer +from .helpers import ResponseCallback, RetryCounter +from .fake_credential import FakeTokenCredential __all__ = [ "AzureMgmtTestCase", @@ -31,4 +33,7 @@ "RandomNameResourceGroupPreparer", "CachedResourceGroupPreparer", "PowerShellPreparer", + "ResponseCallback", + "RetryCounter", + "FakeTokenCredential", ] diff --git a/tools/azure-sdk-tools/devtools_testutils/fake_credential.py b/tools/azure-sdk-tools/devtools_testutils/fake_credential.py new file mode 100644 index 000000000000..67dca16a2435 --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/fake_credential.py @@ -0,0 +1,15 @@ +from azure.core.credentials import AccessToken + + +class FakeTokenCredential(object): + """Protocol for classes able to provide OAuth tokens. + :param str scopes: Lets you specify the type of access needed. + """ + + def __init__(self): + self.token = AccessToken("YOU SHALL NOT PASS", 0) + self.get_token_count = 0 + + def get_token(self, *args): + self.get_token_count += 1 + return self.token diff --git a/tools/azure-sdk-tools/devtools_testutils/helpers.py b/tools/azure-sdk-tools/devtools_testutils/helpers.py new file mode 100644 index 000000000000..e751e09cbd55 --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/helpers.py @@ -0,0 +1,30 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +class RetryCounter(object): + def __init__(self): + self.count = 0 + + def simple_count(self, retry_context): + self.count += 1 + + +class ResponseCallback(object): + def __init__(self, status=None, new_status=None): + self.status = status + self.new_status = new_status + self.first = True + self.count = 0 + + def override_first_status(self, response): + if self.first and response.http_response.status_code == self.status: + response.http_response.status_code = self.new_status + self.first = False + self.count += 1 + + def override_status(self, response): + if response.http_response.status_code == self.status: + response.http_response.status_code = self.new_status + self.count += 1 diff --git a/tools/azure-sdk-tools/devtools_testutils/powershell_preparer.py b/tools/azure-sdk-tools/devtools_testutils/powershell_preparer.py index ae19710d048d..562bb825bec6 100644 --- a/tools/azure-sdk-tools/devtools_testutils/powershell_preparer.py +++ b/tools/azure-sdk-tools/devtools_testutils/powershell_preparer.py @@ -3,14 +3,13 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -import functools import os from . import AzureMgmtPreparer -from .resource_testcase import RESOURCE_GROUP_PARAM from azure_devtools.scenario_tests.exceptions import AzureTestError from dotenv import load_dotenv, find_dotenv + class PowerShellPreparer(AzureMgmtPreparer): def __init__( self, diff --git a/tools/azure-sdk-tools/devtools_testutils/storage/__init__.py b/tools/azure-sdk-tools/devtools_testutils/storage/__init__.py new file mode 100644 index 000000000000..d5de005a22db --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/storage/__init__.py @@ -0,0 +1,5 @@ +from .api_version_policy import ApiVersionAssertPolicy +from .service_versions import service_version_map, ServiceVersion, is_version_before +from .testcase import StorageTestCase + +__all__ = ["ApiVersionAssertPolicy", "service_version_map", "StorageTestCase", "ServiceVersion", "is_version_before"] diff --git a/tools/azure-sdk-tools/devtools_testutils/storage/aio/__init__.py b/tools/azure-sdk-tools/devtools_testutils/storage/aio/__init__.py new file mode 100644 index 000000000000..ee8d633673cf --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/storage/aio/__init__.py @@ -0,0 +1,3 @@ +from .asynctestcase import AsyncStorageTestCase + +__all__ = ["AsyncStorageTestCase"] diff --git a/sdk/storage/azure-storage-blob/tests/_shared/asynctestcase.py b/tools/azure-sdk-tools/devtools_testutils/storage/aio/asynctestcase.py similarity index 75% rename from sdk/storage/azure-storage-blob/tests/_shared/asynctestcase.py rename to tools/azure-sdk-tools/devtools_testutils/storage/aio/asynctestcase.py index 0aea8459a332..51f540fdb58a 100644 --- a/sdk/storage/azure-storage-blob/tests/_shared/asynctestcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/storage/aio/asynctestcase.py @@ -1,32 +1,12 @@ - -# coding: utf-8 -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- import asyncio import functools +from .. import StorageTestCase +from ...fake_async_credential import AsyncFakeCredential + from azure_devtools.scenario_tests.patches import mock_in_unit_test from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function -from azure.core.credentials import AccessToken - -from .testcase import StorageTestCase - -LOGGING_FORMAT = '%(asctime)s %(name)-20s %(levelname)-5s %(message)s' - -class AsyncFakeTokenCredential(object): - """Protocol for classes able to provide OAuth tokens. - :param str scopes: Lets you specify the type of access needed. - """ - def __init__(self): - self.token = AccessToken("YOU SHALL NOT PASS", 0) - - async def get_token(self, *args): - return self.token - def patch_play_responses(unit_test): """Fixes a bug affecting blob tests by applying https://github.com/kevin1024/vcrpy/pull/511 to vcrpy 3.0.0""" @@ -77,6 +57,7 @@ def run(test_class_instance, *args, **kwargs): def generate_oauth_token(self): if self.is_live: from azure.identity.aio import ClientSecretCredential + return ClientSecretCredential( self.get_settings_value("TENANT_ID"), self.get_settings_value("CLIENT_ID"), @@ -85,4 +66,4 @@ def generate_oauth_token(self): return self.generate_fake_token() def generate_fake_token(self): - return AsyncFakeTokenCredential() + return AsyncFakeCredential() diff --git a/tools/azure-sdk-tools/devtools_testutils/storage/api_version_policy.py b/tools/azure-sdk-tools/devtools_testutils/storage/api_version_policy.py new file mode 100644 index 000000000000..1d19f48d7c2f --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/storage/api_version_policy.py @@ -0,0 +1,13 @@ +from azure.core.pipeline.policies import SansIOHTTPPolicy + + +class ApiVersionAssertPolicy(SansIOHTTPPolicy): + """ + Assert the ApiVersion is set properly on the response + """ + + def __init__(self, api_version): + self.api_version = api_version + + def on_request(self, request): + assert request.http_request.headers["x-ms-version"] == self.api_version diff --git a/tools/azure-sdk-tools/devtools_testutils/storage/processors.py b/tools/azure-sdk-tools/devtools_testutils/storage/processors.py new file mode 100644 index 000000000000..68f2cd0480f6 --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/storage/processors.py @@ -0,0 +1,21 @@ +import re + +from azure_devtools.scenario_tests import RecordingProcessor + + +class XMSRequestIDBody(RecordingProcessor): + """This process is used for Storage batch call only, to avoid the echo policy.""" + + def process_response(self, response): + content_type = None + for key, value in response.get("headers", {}).items(): + if key.lower() == "content-type": + content_type = (value[0] if isinstance(value, list) else value).lower() + break + + if content_type and "multipart/mixed" in content_type: + response["body"]["string"] = re.sub( + b"x-ms-client-request-id: [a-f0-9-]+\r\n", b"", response["body"]["string"] + ) + + return response diff --git a/sdk/storage/azure-storage-blob/tests/_shared/service_versions.py b/tools/azure-sdk-tools/devtools_testutils/storage/service_versions.py similarity index 90% rename from sdk/storage/azure-storage-blob/tests/_shared/service_versions.py rename to tools/azure-sdk-tools/devtools_testutils/storage/service_versions.py index e6d422da22e9..3e58f71ba540 100644 --- a/sdk/storage/azure-storage-blob/tests/_shared/service_versions.py +++ b/tools/azure-sdk-tools/devtools_testutils/storage/service_versions.py @@ -25,12 +25,12 @@ class ServiceVersion(str, Enum): "V2020_06_12": ServiceVersion.V2020_06_12, "V2020_08_04": ServiceVersion.V2020_08_04, "LATEST": ServiceVersion.V2020_08_04, - "LATEST_PLUS_1": ServiceVersion.V2020_06_12 + "LATEST_PLUS_1": ServiceVersion.V2020_06_12, } def is_version_before(test_version): - """ Return True if the current version is after a given one or if the + """Return True if the current version is after a given one or if the service version is not set. """ current_version = service_version_map.get(os.environ.get("AZURE_LIVE_TEST_SERVICE_VERSION")) diff --git a/tools/azure-sdk-tools/devtools_testutils/storage/testcase.py b/tools/azure-sdk-tools/devtools_testutils/storage/testcase.py new file mode 100644 index 000000000000..93e7a77ddc95 --- /dev/null +++ b/tools/azure-sdk-tools/devtools_testutils/storage/testcase.py @@ -0,0 +1,205 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from __future__ import division + +from datetime import datetime, timedelta +import logging +import math +import os +import random +import sys +import time +import zlib + +from .processors import XMSRequestIDBody +from . import ApiVersionAssertPolicy, service_version_map +from .. import AzureMgmtTestCase, FakeTokenCredential + +from azure.storage.blob import generate_account_sas, AccountSasPermissions, ResourceTypes + +LOGGING_FORMAT = "%(asctime)s %(name)-20s %(levelname)-5s %(message)s" + +ENABLE_LOGGING = True + + +class StorageTestCase(AzureMgmtTestCase): + def __init__(self, *args, **kwargs): + super(StorageTestCase, self).__init__(*args, **kwargs) + self.replay_processors.append(XMSRequestIDBody()) + self.logger = logging.getLogger("azure.storage") + self.configure_logging() + + def connection_string(self, account, key): + return ( + "DefaultEndpointsProtocol=https;AcCounTName=" + + account.name + + ";AccOuntKey=" + + str(key) + + ";EndpoIntSuffix=core.windows.net" + ) + + def account_url(self, storage_account, storage_type): + """Return an url of storage account. + + :param str storage_account: Storage account name + :param str storage_type: The Storage type part of the URL. Should be "blob", or "queue", etc. + """ + try: + if storage_type == "blob": + return storage_account.primary_endpoints.blob.rstrip("/") + if storage_type == "queue": + return storage_account.primary_endpoints.queue.rstrip("/") + if storage_type == "file": + return storage_account.primary_endpoints.file.rstrip("/") + else: + raise ValueError("Unknown storage type {}".format(storage_type)) + except AttributeError: # Didn't find "primary_endpoints" + return "https://{}.{}.core.windows.net".format(storage_account, storage_type) + + def configure_logging(self): + enable_logging = ENABLE_LOGGING + + self.enable_logging() if enable_logging else self.disable_logging() + + def enable_logging(self): + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter(LOGGING_FORMAT)) + self.logger.handlers = [handler] + self.logger.setLevel(logging.DEBUG) + self.logger.propagate = True + self.logger.disabled = False + + def disable_logging(self): + self.logger.propagate = False + self.logger.disabled = True + self.logger.handlers = [] + + def sleep(self, seconds): + if self.is_live: + time.sleep(seconds) + + def get_random_bytes(self, size): + # recordings don't like random stuff. making this more + # deterministic. + return b"a" * size + + def get_random_text_data(self, size): + """Returns random unicode text data exceeding the size threshold for + chunking blob upload.""" + checksum = zlib.adler32(self.qualified_test_name.encode()) & 0xFFFFFFFF + rand = random.Random(checksum) + text = u"" + words = [u"hello", u"world", u"python", u"啊齄丂狛狜"] + while len(text) < size: + index = int(rand.random() * (len(words) - 1)) + text = text + u" " + words[index] + + return text + + @staticmethod + def _set_test_proxy(service, settings): + if settings.USE_PROXY: + service.set_proxy( + settings.PROXY_HOST, + settings.PROXY_PORT, + settings.PROXY_USER, + settings.PROXY_PASSWORD, + ) + + def assertNamedItemInContainer(self, container, item_name, msg=None): + def _is_string(obj): + if sys.version_info >= (3,): + return isinstance(obj, str) + else: + return isinstance(obj, basestring) + + for item in container: + if _is_string(item): + if item == item_name: + return + elif isinstance(item, dict): + if item_name == item["name"]: + return + elif item.name == item_name: + return + elif hasattr(item, "snapshot") and item.snapshot == item_name: + return + + standardMsg = "{0} not found in {1}".format(repr(item_name), [str(c) for c in container]) + self.fail(self._formatMessage(msg, standardMsg)) + + def assertNamedItemNotInContainer(self, container, item_name, msg=None): + for item in container: + if item.name == item_name: + standardMsg = "{0} unexpectedly found in {1}".format(repr(item_name), repr(container)) + self.fail(self._formatMessage(msg, standardMsg)) + + def assert_upload_progress(self, size, max_chunk_size, progress, unknown_size=False): + """Validates that the progress chunks align with our chunking procedure.""" + index = 0 + total = None if unknown_size else size + small_chunk_size = size % max_chunk_size + self.assertEqual(len(progress), math.ceil(size / max_chunk_size)) + for i in progress: + self.assertTrue(i[0] % max_chunk_size == 0 or i[0] % max_chunk_size == small_chunk_size) + self.assertEqual(i[1], total) + + def assert_download_progress(self, size, max_chunk_size, max_get_size, progress): + """Validates that the progress chunks align with our chunking procedure.""" + if size <= max_get_size: + self.assertEqual(len(progress), 1) + self.assertTrue(progress[0][0], size) + self.assertTrue(progress[0][1], size) + else: + small_chunk_size = (size - max_get_size) % max_chunk_size + self.assertEqual(len(progress), 1 + math.ceil((size - max_get_size) / max_chunk_size)) + + self.assertTrue(progress[0][0], max_get_size) + self.assertTrue(progress[0][1], size) + for i in progress[1:]: + self.assertTrue(i[0] % max_chunk_size == 0 or i[0] % max_chunk_size == small_chunk_size) + self.assertEqual(i[1], size) + + def generate_oauth_token(self): + if self.is_live: + from azure.identity import ClientSecretCredential + + return ClientSecretCredential( + self.get_settings_value("TENANT_ID"), + self.get_settings_value("CLIENT_ID"), + self.get_settings_value("CLIENT_SECRET"), + ) + return self.generate_fake_token() + + def generate_sas_token(self): + fake_key = "a" * 30 + "b" * 30 + + return "?" + generate_account_sas( + account_name="test", # name of the storage account + account_key=fake_key, # key for the storage account + resource_types=ResourceTypes(object=True), + permission=AccountSasPermissions(read=True, list=True), + start=datetime.now() - timedelta(hours=24), + expiry=datetime.now() + timedelta(days=8), + ) + + def generate_fake_token(self): + return FakeTokenCredential() + + def _get_service_version(self, **kwargs): + env_version = service_version_map.get(os.environ.get("AZURE_LIVE_TEST_SERVICE_VERSION", "LATEST")) + return kwargs.pop("service_version", env_version) + + def create_storage_client(self, client, *args, **kwargs): + kwargs["api_version"] = self._get_service_version(**kwargs) + kwargs["_additional_pipeline_policies"] = [ApiVersionAssertPolicy(kwargs["api_version"])] + return client(*args, **kwargs) + + def create_storage_client_from_conn_str(self, client, *args, **kwargs): + kwargs["api_version"] = self._get_service_version(**kwargs) + kwargs["_additional_pipeline_policies"] = [ApiVersionAssertPolicy(kwargs["api_version"])] + return client.from_connection_string(*args, **kwargs) diff --git a/tools/azure-sdk-tools/devtools_testutils/storage_testcase.py b/tools/azure-sdk-tools/devtools_testutils/storage_testcase.py index c3a9ebee87c3..f015cc48c88f 100644 --- a/tools/azure-sdk-tools/devtools_testutils/storage_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/storage_testcase.py @@ -16,7 +16,13 @@ try: # Note: these models are only available from v17.0.0 and higher, if you need them you'll also need azure-core 1.4.0 and higher from azure.mgmt.storage import StorageManagementClient - from azure.mgmt.storage.models import StorageAccount, Endpoints, LastAccessTimeTrackingPolicy, BlobServiceProperties, DeleteRetentionPolicy + from azure.mgmt.storage.models import ( + StorageAccount, + Endpoints, + LastAccessTimeTrackingPolicy, + BlobServiceProperties, + DeleteRetentionPolicy, + ) except ImportError: pass @@ -146,10 +152,7 @@ def _get_resource_group(self, **kwargs): class BlobAccountPreparer(StorageAccountPreparer): - def __init__( - self, - **kwargs - ): + def __init__(self, **kwargs): self.is_versioning_enabled = kwargs.pop("is_versioning_enabled", None) self.is_last_access_time_enabled = kwargs.pop("is_last_access_time_enabled", None) self.container_retention_days = kwargs.pop("container_retention_days", None) @@ -172,8 +175,9 @@ def _create_account(self, resource_group_name, account_name): if self.is_versioning_enabled is True: props.is_versioning_enabled = True if self.container_retention_days: - props.container_delete_retention_policy = DeleteRetentionPolicy(enabled=True, - days=self.container_retention_days) + props.container_delete_retention_policy = DeleteRetentionPolicy( + enabled=True, days=self.container_retention_days + ) if self.is_last_access_time_enabled: props.last_access_time_tracking_policy = LastAccessTimeTrackingPolicy(enable=True)