diff --git a/.gitignore b/.gitignore index 64fd5d89aa41..0d79f0f45aec 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ build/ # Test results TestResults/ ENV_DIR/ +.assets/ # Perf test profiling cProfile-*.pstats diff --git a/scripts/devops_tasks/tox_harness.py b/scripts/devops_tasks/tox_harness.py index de37e6a19b9c..e495e4157699 100644 --- a/scripts/devops_tasks/tox_harness.py +++ b/scripts/devops_tasks/tox_harness.py @@ -165,7 +165,6 @@ def individual_workload(tox_command_tuple, workload_results): if in_ci(): shutil.rmtree(tox_dir) - def execute_tox_parallel(tox_command_tuples): pool = ThreadPool(pool_size) workload_results = {} @@ -256,6 +255,7 @@ def build_whl_for_req(req, package_path): else: return req + def replace_dev_reqs(file, pkg_root): adjusted_req_lines = [] @@ -355,12 +355,12 @@ def collect_log_files(working_dir): for f in glob.glob(os.path.join(root_dir, "_tox_logs", "*")): logging.info("Log file: {}".format(f)) - def execute_tox_serial(tox_command_tuples): return_code = 0 for index, cmd_tuple in enumerate(tox_command_tuples): tox_dir = os.path.abspath(os.path.join(cmd_tuple[1], "./.tox/")) + clone_dir = os.path.abspath(os.path.join(cmd_tuple[1], "..", "..", "..", "l")) logging.info("tox_dir: {}".format(tox_dir)) logging.info( @@ -378,6 +378,16 @@ def execute_tox_serial(tox_command_tuples): collect_log_files(cmd_tuple[1]) shutil.rmtree(tox_dir) + if os.path.exists(clone_dir): + try: + shutil.rmtree(clone_dir) + except Exception as e: + # git has a permissions problem. one of the files it drops + # cannot be removed as no one has the permission to do so. + # lets log just in case, but this should really only affect windows machines. + logging.info(e) + pass + return return_code diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_startup.py b/tools/azure-sdk-tools/devtools_testutils/proxy_startup.py index 4bda8fe0ac25..d12448ef674f 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_startup.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_startup.py @@ -31,6 +31,7 @@ PROXY_CHECK_URL = PROXY_URL.rstrip("/") + "/Info/Available" TOOL_ENV_VAR = "PROXY_PID" +discovered_roots = [] def get_image_tag(repo_root: str) -> str: """Gets the test proxy Docker image tag from the target_version.txt file in /eng/common/testproxy""" @@ -59,6 +60,8 @@ def ascend_to_root(start_dir_or_file: str) -> str: # we need the git check to prevent ascending out of the repo if os.path.exists(possible_root): + if current_dir not in discovered_roots: + discovered_roots.append(current_dir) return current_dir else: current_dir = os.path.dirname(current_dir) @@ -140,10 +143,15 @@ def start_test_proxy(request) -> None: log = open(os.path.join(root, "_proxy_log_{}.log".format(envname)), "a") _LOGGER.info("{} is calculated repo root".format(root)) + + os.environ["PROXY_ASSETS_FOLDER"] = os.path.join(root, "l", envname) + if not os.path.exists(os.environ["PROXY_ASSETS_FOLDER"]): + os.makedirs(os.environ["PROXY_ASSETS_FOLDER"]) + proc = subprocess.Popen( shlex.split('test-proxy start --storage-location="{}" -- --urls "{}"'.format(root, PROXY_URL)), stdout=log, - stderr=log, + stderr=log ) os.environ[TOOL_ENV_VAR] = str(proc.pid) else: diff --git a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py index 1d7954fae911..0f68130a5455 100644 --- a/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py +++ b/tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py @@ -6,6 +6,7 @@ import logging import requests import six +import os from typing import TYPE_CHECKING import urllib.parse as url_parse @@ -19,6 +20,7 @@ from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function from .config import PROXY_URL from .helpers import get_test_id, is_live, is_live_and_not_recording, set_recording_id +from .proxy_startup import discovered_roots if TYPE_CHECKING: from typing import Callable, Dict, Tuple @@ -34,6 +36,27 @@ PLAYBACK_START_URL = "{}/playback/start".format(PROXY_URL) PLAYBACK_STOP_URL = "{}/playback/stop".format(PROXY_URL) +def get_recording_assets(test_id: str) -> str: + """ + Used to retrieve the assets.json given a PYTEST_CURRENT_TEST test id. + """ + for root in discovered_roots: + current_dir = os.path.dirname(test_id) + while current_dir is not None and not (os.path.dirname(current_dir) == current_dir): + possible_assets = os.path.join(current_dir, "assets.json") + possible_root = os.path.join(current_dir, ".git") + + # we need to check for assets.json first! + if os.path.exists(os.path.join(root, possible_assets)): + complete_path = os.path.abspath(os.path.join(root, possible_assets)) + return os.path.relpath(complete_path, root).replace("\\", "/") + # we need the git check to prevent ascending out of the repo + elif os.path.exists(os.path.join(root, possible_root)): + return None + else: + current_dir = os.path.dirname(current_dir) + + return None def start_record_or_playback(test_id: str) -> "Tuple[str, Dict[str, str]]": """Sends a request to begin recording or playing back the provided test. @@ -42,11 +65,16 @@ def start_record_or_playback(test_id: str) -> "Tuple[str, Dict[str, str]]": test variables to values. If no variable dictionary was stored when the test was recorded, b is an empty dictionary. """ variables = {} # this stores a dictionary of test variable values that could have been stored with a recording + + json_payload = {"x-recording-file": test_id} + assets_json = get_recording_assets(test_id) + if assets_json: + json_payload["x-recording-assets-file"] = assets_json if is_live(): result = requests.post( RECORDING_START_URL, - json={"x-recording-file": test_id}, + json=json_payload, ) if result.status_code != 200: message = six.ensure_str(result._content) @@ -56,7 +84,7 @@ def start_record_or_playback(test_id: str) -> "Tuple[str, Dict[str, str]]": else: result = requests.post( PLAYBACK_START_URL, - json={"x-recording-file": test_id}, + json=json_payload, ) if result.status_code != 200: message = six.ensure_str(result._content)