From 90b4322316b673794c775fb4dcdfddf57aa77b52 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Fri, 8 Oct 2021 16:14:15 -0700 Subject: [PATCH 1/3] Update docs --- doc/dev/perfstress_tests.md | 2 +- sdk/storage/azure-storage-blob/tests/perfstress_tests/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/dev/perfstress_tests.md b/doc/dev/perfstress_tests.md index 7759e45175aa..adb986e9320d 100644 --- a/doc/dev/perfstress_tests.md +++ b/doc/dev/perfstress_tests.md @@ -82,7 +82,7 @@ The framework has a series of common command line options built in: - `-w --warm-up=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5. - `--sync` Whether to run the tests in sync or async. Default is False (async). - `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted). -- `-x --test-proxy` Whether to run the tests against the test proxy server. Specfiy the URL for the proxy endpoint (e.g. "https://localhost:5001"). +- `-x --test-proxies` Whether to run the tests against the test proxy server. Specify the URL(s) for the proxy endpoint(s) (e.g. "https://localhost:5001"). - `--profile` Whether to run the perftest with cProfile. If enabled (default is False), the output file of the **last completed single iteration** will be written to the current working directory in the format `"cProfile---.pstats"`. diff --git a/sdk/storage/azure-storage-blob/tests/perfstress_tests/README.md b/sdk/storage/azure-storage-blob/tests/perfstress_tests/README.md index fbe153c8f370..9519a5564cdc 100644 --- a/sdk/storage/azure-storage-blob/tests/perfstress_tests/README.md +++ b/sdk/storage/azure-storage-blob/tests/perfstress_tests/README.md @@ -44,7 +44,7 @@ These options are available for all perf tests: - `-w --warm-up=5` Number of seconds to spend warming up the connection before measuring begins. Default is 5. - `--sync` Whether to run the tests in sync or async. Default is False (async). This flag must be used for Storage legacy tests, which do not support async. - `--no-cleanup` Whether to keep newly created resources after test run. Default is False (resources will be deleted). -- `-x --test-proxy` Whether to run the tests against the test proxy server. Specfiy the URL for the proxy endpoint (e.g. "https://localhost:5001"). WARNING: When using with Legacy tests - only HTTPS is supported. +- `-x --test-proxies` Whether to run the tests against the test proxy server. Specfiy the URL(s) for the proxy endpoint(s) (e.g. "https://localhost:5001"). WARNING: When using with Legacy tests - only HTTPS is supported. - `--profile` Whether to run the perftest with cProfile. If enabled (default is False), the output file of the **last completed single iteration** will be written to the current working directory in the format `"cProfile---.pstats"`. ### Common Blob command line options From f106de24457ad8cae0124d716885a8c3558308f3 Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Fri, 8 Oct 2021 16:35:15 -0700 Subject: [PATCH 2/3] Change option from --test-proxy to --test-proxies --- .../T1_legacy_tests/_test_base_legacy.py | 4 ++-- .../tests/perfstress_tests/_test_base.py | 2 +- .../perfstress_tests/perf_stress_runner.py | 7 ++++--- .../perfstress_tests/perf_stress_test.py | 18 ++++++++++++------ 4 files changed, 19 insertions(+), 12 deletions(-) diff --git a/sdk/storage/azure-storage-blob/tests/perfstress_tests/T1_legacy_tests/_test_base_legacy.py b/sdk/storage/azure-storage-blob/tests/perfstress_tests/T1_legacy_tests/_test_base_legacy.py index a5a3d607f972..2cc007240ba9 100644 --- a/sdk/storage/azure-storage-blob/tests/perfstress_tests/T1_legacy_tests/_test_base_legacy.py +++ b/sdk/storage/azure-storage-blob/tests/perfstress_tests/T1_legacy_tests/_test_base_legacy.py @@ -37,7 +37,7 @@ def __init__(self, arguments): super().__init__(arguments) connection_string = self.get_from_env("AZURE_STORAGE_CONNECTION_STRING") session = None - if self.args.test_proxy: + if self.args.test_proxies: session = requests.Session() session.verify = False if not _LegacyServiceTest.service_client or self.args.no_client_share: @@ -50,7 +50,7 @@ def __init__(self, arguments): self.async_service_client = None self.service_client = _LegacyServiceTest.service_client - if self.args.test_proxy: + if self.args.test_proxies: self.service_client.request_callback = functools.partial( test_proxy_callback, self._test_proxy_policy diff --git a/sdk/storage/azure-storage-blob/tests/perfstress_tests/_test_base.py b/sdk/storage/azure-storage-blob/tests/perfstress_tests/_test_base.py index ca46e67ffccb..3405d9fadbed 100644 --- a/sdk/storage/azure-storage-blob/tests/perfstress_tests/_test_base.py +++ b/sdk/storage/azure-storage-blob/tests/perfstress_tests/_test_base.py @@ -20,7 +20,7 @@ class _ServiceTest(PerfStressTest): def __init__(self, arguments): super().__init__(arguments) connection_string = self.get_from_env("AZURE_STORAGE_CONNECTION_STRING") - if self.args.test_proxy: + if self.args.test_proxies: self._client_kwargs['_additional_pipeline_policies'] = self._client_kwargs['per_retry_policies'] self._client_kwargs['max_single_put_size'] = self.args.max_put_size self._client_kwargs['max_block_size'] = self.args.max_block_size diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/perf_stress_runner.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/perf_stress_runner.py index a48ded31e10c..3d6de2dc5e68 100644 --- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/perf_stress_runner.py +++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/perf_stress_runner.py @@ -97,7 +97,8 @@ def _parse_args(self): "--profile", action="store_true", help="Run tests with profiler. Default is False.", default=False ) per_test_arg_parser.add_argument( - "-x", "--test-proxy", help="URI of TestProxy Server" + "-x", "--test-proxies", help="URIs of TestProxy Servers (separated by ';')", + type=lambda s: s.split(';') ) # Per-test args @@ -142,7 +143,7 @@ async def start(self): await asyncio.gather(*[test.setup() for test in tests]) self.logger.info("") - if self.per_test_args.test_proxy: + if self.per_test_args.test_proxies: self.logger.info("=== Record and Start Playback ===") await asyncio.gather(*[test.record_and_start_playback() for test in tests]) self.logger.info("") @@ -162,7 +163,7 @@ async def start(self): except Exception as e: print("Exception: " + str(e)) finally: - if self.per_test_args.test_proxy: + if self.per_test_args.test_proxies: self.logger.info("=== Stop Playback ===") await asyncio.gather(*[test.stop_playback() for test in tests]) self.logger.info("") diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/perf_stress_test.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/perf_stress_test.py index 582251125fdb..bc26ebc12e15 100644 --- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/perf_stress_test.py +++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/perf_stress_test.py @@ -21,15 +21,20 @@ class PerfStressTest: """ args = {} + _global_parallel_index = 0 def __init__(self, arguments): self.args = arguments self._session = None + self._test_proxy = None self._test_proxy_policy = None self._client_kwargs = {} self._recording_id = None - if self.args.test_proxy: + self._parallel_index = PerfStressTest._global_parallel_index + PerfStressTest._global_parallel_index += 1 + + if self.args.test_proxies: self._session = aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False)) # SSL will be disabled for the test proxy requests, so suppress warnings @@ -38,7 +43,8 @@ def __init__(self, arguments): warnings.simplefilter('ignore', InsecureRequestWarning) # Add policy to redirect requests to the test proxy - self._test_proxy_policy = PerfTestProxyPolicy(self.args.test_proxy) + self._test_proxy = self.args.test_proxies[self._parallel_index % len(self.args.test_proxies)] + self._test_proxy_policy = PerfTestProxyPolicy(self._test_proxy) self._client_kwargs['per_retry_policies'] = [self._test_proxy_policy] async def global_setup(self): @@ -74,7 +80,7 @@ async def stop_playback(self): "x-recording-id": self._recording_id, "x-purge-inmemory-recording": "true" } - url = urljoin(self.args.test_proxy, "/playback/stop") + url = urljoin(self._test_proxy, "/playback/stop") async with self._session.post(url, headers=headers) as resp: assert resp.status == 200 @@ -98,20 +104,20 @@ async def run_async(self): raise Exception("run_async must be implemented for {}".format(self.__class__.__name__)) async def _start_recording(self): - url = urljoin(self.args.test_proxy, "/record/start") + url = urljoin(self._test_proxy, "/record/start") async with self._session.post(url) as resp: assert resp.status == 200 self._recording_id = resp.headers["x-recording-id"] async def _stop_recording(self): headers = {"x-recording-id": self._recording_id} - url = urljoin(self.args.test_proxy, "/record/stop") + url = urljoin(self._test_proxy, "/record/stop") async with self._session.post(url, headers=headers) as resp: assert resp.status == 200 async def _start_playback(self): headers = {"x-recording-id": self._recording_id} - url = urljoin(self.args.test_proxy, "/playback/start") + url = urljoin(self._test_proxy, "/playback/start") async with self._session.post(url, headers=headers) as resp: assert resp.status == 200 self._recording_id = resp.headers["x-recording-id"] From 0c24d693267daf6adfc8ad5c561488c343a6b4df Mon Sep 17 00:00:00 2001 From: Mike Harder Date: Tue, 19 Oct 2021 12:50:28 -0700 Subject: [PATCH 3/3] Add locking around _global_parallel_index --- .../azure_devtools/perfstress_tests/perf_stress_test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tools/azure-devtools/src/azure_devtools/perfstress_tests/perf_stress_test.py b/tools/azure-devtools/src/azure_devtools/perfstress_tests/perf_stress_test.py index bc26ebc12e15..a31da53ffa11 100644 --- a/tools/azure-devtools/src/azure_devtools/perfstress_tests/perf_stress_test.py +++ b/tools/azure-devtools/src/azure_devtools/perfstress_tests/perf_stress_test.py @@ -4,6 +4,7 @@ # -------------------------------------------------------------------------------------------- import os +import threading import aiohttp from urllib.parse import urljoin @@ -21,6 +22,7 @@ class PerfStressTest: """ args = {} + _global_parallel_index_lock = threading.Lock() _global_parallel_index = 0 def __init__(self, arguments): @@ -31,8 +33,9 @@ def __init__(self, arguments): self._client_kwargs = {} self._recording_id = None - self._parallel_index = PerfStressTest._global_parallel_index - PerfStressTest._global_parallel_index += 1 + with PerfStressTest._global_parallel_index_lock: + self._parallel_index = PerfStressTest._global_parallel_index + PerfStressTest._global_parallel_index += 1 if self.args.test_proxies: self._session = aiohttp.ClientSession(connector=aiohttp.TCPConnector(verify_ssl=False))