[Asset-Sync] extensions to the test framework to allow external assets

.gitignore

@@ -37,6 +37,7 @@ build/
 # Test results
 TestResults/
 ENV_DIR/
+.assets/
 
 # Perf test profiling
 cProfile-*.pstats

scripts/devops_tasks/tox_harness.py

@@ -165,7 +165,6 @@ def individual_workload(tox_command_tuple, workload_results):
         if in_ci():
             shutil.rmtree(tox_dir)
 
-
 def execute_tox_parallel(tox_command_tuples):
     pool = ThreadPool(pool_size)
     workload_results = {}
@@ -256,6 +255,7 @@ def build_whl_for_req(req, package_path):
     else:
         return req
 
+
 def replace_dev_reqs(file, pkg_root):
     adjusted_req_lines = []
 
@@ -355,12 +355,12 @@ def collect_log_files(working_dir):
     for f in glob.glob(os.path.join(root_dir, "_tox_logs", "*")):
         logging.info("Log file: {}".format(f))
 
-
 def execute_tox_serial(tox_command_tuples):
     return_code = 0
 
     for index, cmd_tuple in enumerate(tox_command_tuples):
         tox_dir = os.path.abspath(os.path.join(cmd_tuple[1], "./.tox/"))
+        clone_dir = os.path.abspath(os.path.join(cmd_tuple[1], "..", "..", "..", "l"))
         logging.info("tox_dir: {}".format(tox_dir))
 
         logging.info(
@@ -378,6 +378,16 @@ def execute_tox_serial(tox_command_tuples):
             collect_log_files(cmd_tuple[1])
             shutil.rmtree(tox_dir)
 
+            if os.path.exists(clone_dir):
+                try:
+                    shutil.rmtree(clone_dir)
+                except Exception as e:
+                    # git has a permissions problem. one of the files it drops
+                    # cannot be removed as no one has the permission to do so.
+                    # lets log just in case, but this should really only affect windows machines.
+                    logging.info(e)
+                    pass
+
     return return_code
 
 

tools/azure-sdk-tools/devtools_testutils/proxy_startup.py

@@ -31,6 +31,7 @@
 PROXY_CHECK_URL = PROXY_URL.rstrip("/") + "/Info/Available"
 TOOL_ENV_VAR = "PROXY_PID"
 
+discovered_roots = []
 
 def get_image_tag(repo_root: str) -> str:
     """Gets the test proxy Docker image tag from the target_version.txt file in /eng/common/testproxy"""
@@ -59,6 +60,8 @@ def ascend_to_root(start_dir_or_file: str) -> str:
 
         # we need the git check to prevent ascending out of the repo
         if os.path.exists(possible_root):
+            if current_dir not in discovered_roots:
+                discovered_roots.append(current_dir)
             return current_dir
         else:
             current_dir = os.path.dirname(current_dir)
@@ -140,10 +143,15 @@ def start_test_proxy(request) -> None:
                 log = open(os.path.join(root, "_proxy_log_{}.log".format(envname)), "a")
 
                 _LOGGER.info("{} is calculated repo root".format(root))
+
+                os.environ["PROXY_ASSETS_FOLDER"] = os.path.join(root, "l", envname)
+                if not os.path.exists(os.environ["PROXY_ASSETS_FOLDER"]):
+                    os.makedirs(os.environ["PROXY_ASSETS_FOLDER"])
+
                 proc = subprocess.Popen(
                     shlex.split('test-proxy start --storage-location="{}" -- --urls "{}"'.format(root, PROXY_URL)),
                     stdout=log,
-                    stderr=log,
+                    stderr=log
                 )
                 os.environ[TOOL_ENV_VAR] = str(proc.pid)
         else:

tools/azure-sdk-tools/devtools_testutils/proxy_testcase.py

@@ -6,6 +6,7 @@
 import logging
 import requests
 import six
+import os
 from typing import TYPE_CHECKING
 import urllib.parse as url_parse
 
@@ -19,6 +20,7 @@
 from azure_devtools.scenario_tests.utilities import trim_kwargs_from_test_function
 from .config import PROXY_URL
 from .helpers import get_test_id, is_live, is_live_and_not_recording, set_recording_id
+from .proxy_startup import discovered_roots
 
 if TYPE_CHECKING:
     from typing import Callable, Dict, Tuple
@@ -34,6 +36,27 @@
 PLAYBACK_START_URL = "{}/playback/start".format(PROXY_URL)
 PLAYBACK_STOP_URL = "{}/playback/stop".format(PROXY_URL)
 
+def get_recording_assets(test_id: str) -> str:
+    """
+    Used to retrieve the assets.json given a PYTEST_CURRENT_TEST test id.
+    """
+    for root in discovered_roots:
+        current_dir = os.path.dirname(test_id)
+        while current_dir is not None and not (os.path.dirname(current_dir) == current_dir):
+            possible_assets = os.path.join(current_dir, "assets.json")
+            possible_root = os.path.join(current_dir, ".git")
+
+            # we need to check for assets.json first!
+            if os.path.exists(os.path.join(root, possible_assets)):
+                complete_path = os.path.abspath(os.path.join(root, possible_assets))
+                return os.path.relpath(complete_path, root).replace("\\", "/")
+            # we need the git check to prevent ascending out of the repo
+            elif os.path.exists(os.path.join(root, possible_root)):
+                return None
+            else:
+                current_dir = os.path.dirname(current_dir)
+
+    return None
 
 def start_record_or_playback(test_id: str) -> "Tuple[str, Dict[str, str]]":
     """Sends a request to begin recording or playing back the provided test.
@@ -42,11 +65,16 @@ def start_record_or_playback(test_id: str) -> "Tuple[str, Dict[str, str]]":
     test variables to values. If no variable dictionary was stored when the test was recorded, b is an empty dictionary.
     """
     variables = {}  # this stores a dictionary of test variable values that could have been stored with a recording
+
+    json_payload = {"x-recording-file": test_id}
+    assets_json = get_recording_assets(test_id)
+    if assets_json:
+        json_payload["x-recording-assets-file"] = assets_json    
 
     if is_live():
         result = requests.post(
             RECORDING_START_URL,
-            json={"x-recording-file": test_id},
+            json=json_payload,
         )
         if result.status_code != 200:
             message = six.ensure_str(result._content)
@@ -56,7 +84,7 @@ def start_record_or_playback(test_id: str) -> "Tuple[str, Dict[str, str]]":
     else:
         result = requests.post(
             PLAYBACK_START_URL,
-            json={"x-recording-file": test_id},
+            json=json_payload,
         )
         if result.status_code != 200:
             message = six.ensure_str(result._content)