From a2a83ba292077ca0c404f790e0b957f7de3820c8 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 25 Jul 2023 12:29:43 -0700 Subject: [PATCH 01/19] Switch pwsh to powershell. --- scripts/benchmarks_local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 8b5cf8d8660..5930b6f50c8 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -96,7 +96,7 @@ def build_runtime_dependency(parsed_args: Namespace, repo_path: str, subset: str # Run the command if is_windows(parsed_args): build_libs_and_corerun_command = [ - "pwsh", + "powershell", "-File", "build.ps1" ] From 31df6a5799f6ffda57bb9045090831636dfbf844 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 31 Jul 2023 14:25:27 -0700 Subject: [PATCH 02/19] Added build args and some benchmark run args for the added WasmWasm RunType. --- scripts/benchmarks_local.py | 55 +++++++++++++++++++++++++++++++++---- 1 file changed, 49 insertions(+), 6 deletions(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 5930b6f50c8..fec1822a66b 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -49,6 +49,7 @@ class RunType(Enum): MonoAOTLLVM = 2 MonoInterpreter = 3 MonoJIT = 4 + WasmWasm = 5 def is_windows(parsed_args: Namespace): return parsed_args.os == "windows" @@ -92,7 +93,7 @@ def copy_directory_contents(src_dir: str, dest_dir: str): shutil.copy2(os.path.join(src_dirpath, src_filename), dest_dirpath) # Builds libs and corerun by default -def build_runtime_dependency(parsed_args: Namespace, repo_path: str, subset: str = "clr+libs", configuration: str = "Release", additional_args: list = []): +def build_runtime_dependency(parsed_args: Namespace, repo_path: str, subset: str = "clr+libs", configuration: str = "Release", os_override = "", arch_override = "", additional_args: list = []): # Run the command if is_windows(parsed_args): build_libs_and_corerun_command = [ @@ -108,8 +109,8 @@ def build_runtime_dependency(parsed_args: Namespace, repo_path: str, subset: str build_libs_and_corerun_command += [ "-subset", subset, "-configuration", configuration, - "-os", parsed_args.os, - "-arch", parsed_args.architecture, + "-os", os_override if os_override else parsed_args.os, + "-arch", arch_override if arch_override else parsed_args.architecture, "-framework", parsed_args.framework, "-bl" ] + additional_args @@ -194,12 +195,35 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co getLogger().info(f"dotnet_mono already exists in {dest_dir_mono_interpreter} and {dest_dir_mono_jit}. Skipping generation.") if check_for_runtype_specified(parsed_args, [RunType.MonoAOTLLVM]): - raise NotImplementedError("MonoAOTLLVM is not yet implemented.") + raise NotImplementedError("MonoAOTLLVM is not yet implemented.") # TODO: Finish MonoAOTLLVM Build stuff build_runtime_dependency(parsed_args, repo_path, "mono+libs+host+packs", additional_args=['/p:CrossBuild=false' '/p:MonoLLVMUseCxx11Abi=false']) # TODO: Finish MonoAOTLLVM Build stuff # Clean up the build results shutil.rmtree(os.path.join(repo_path, "artifacts"), ignore_errors=True) # TODO: Can we trust the build system to update these when necessary or do we need to clean them up ourselves? - + + if check_for_runtype_specified(parsed_args, [RunType.WasmWasm]): + dest_dir_wasm = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, commit), "wasm_bundle") + if force_regenerate or not os.path.exists(dest_dir_wasm): + build_runtime_dependency(parsed_args, repo_path, "mono+libs+host+packs", os_override="browser", arch_override="wasm", additional_args=[f'/p:AotHostArchitecture={parsed_args.architecture}', f'/p:AotHostOS={parsed_args.os}']) + + src_dir = os.path.join(repo_path, "artifacts", "BrowserWasm", "staging", "dotnet-latest") + dest_dir = os.path.join(repo_path, "artifacts", "bin", "wasm", "dotnet") + copy_directory_contents(src_dir, dest_dir) + src_dir = os.path.join(repo_path, "artifacts", "BrowserWasm", "staging", "build-nugets") + dest_dir = os.path.join(repo_path, "artifacts", "bin", "wasm") + copy_directory_contents(src_dir, dest_dir) + src_file = os.path.join(repo_path, "src", "mono", "wasm", "test-main.js") + dest_file = os.path.join(repo_path, "artifacts", "bin", "wasm", "wasm-data", "test-main.js") + shutil.copy2(src_file, dest_file) + + # Store the dotnet_mono in the artifact storage path + dotnet_wasm_path = os.path.join(repo_path, "artifacts", "bin", "wasm") + shutil.rmtree(dest_dir_wasm, ignore_errors=True) + copy_directory_contents(dotnet_wasm_path, dest_dir_wasm) + else: + getLogger().info(f"wasm_bundle already exists in {dest_dir_wasm}. Skipping generation.") + + getLogger().info(f"Finished generating dependencies for {' '.join(map(str, parsed_args.run_type_names))} run types in {repo_path} and stored in {parsed_args.artifact_storage_path}.") def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunType, all_commits: list) -> list: @@ -267,6 +291,22 @@ def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunTyp corerun_path = corerun_capture[0] bdn_args_unescaped += [ corerun_path ] + elif specific_run_type == RunType.WasmWasm: + benchmark_ci_args += [ '--wasm' ] + bdn_args_unescaped += [ + '--anyCategories', 'Libraries', 'Runtime', + '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono', + '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, commit), "wasm_bundle", "wasm-data"), + '--wasmEngine', parsed_args.wasm_engine_path, + '--wasmArgs', '--experimental-wasm-eh --expose_wasm --module', + # '--cli', '', + '--logBuildOutput', + '--generateBinLog' + ] + + # for commit in all_commits: # TODO see if there is a way to run multiple Wasm's at once. + # bdn_args_unescaped += [ os.path.join(get_run_artifact_path(parsed_args, RunType.CoreRun, commit), "Core_Root", f'corerun{".exe" if is_windows(parsed_args) else ""}') ] + if parsed_args.bdn_arguments: bdn_args_unescaped += [parsed_args.bdn_arguments] benchmark_ci_args += [f'--bdn-arguments={" ".join(bdn_args_unescaped)}'] @@ -371,7 +411,8 @@ def __is_valid_run_type(value): parser.add_argument('--os', choices=['windows', 'linux', 'osx'], default=get_default_os(), help='Specifies the operating system of the system') parser.add_argument('--filter', type=str, default='*', help='Specifies the benchmark filter to pass to BenchmarkDotNet') parser.add_argument('-f', '--framework', choices=ChannelMap.get_supported_frameworks(), default='net8.0', help='The target framework to run the benchmarks against.') # Can and should this accept multiple frameworks? - parser.add_argument('--csproj', type=str, default=os.path.join("..", "src", "benchmarks", "micro", "MicroBenchmarks.csproj"), help='The path to the csproj file to run benchmarks against.') + parser.add_argument('--csproj', type=str, default=os.path.join("..", "src", "benchmarks", "micro", "MicroBenchmarks.csproj"), help='The path to the csproj file to run benchmarks against.') + parser.add_argument('--wasm-engine-path', type=str, help='The full path to the wasm engine to use for the benchmarks. e.g. /usr/local/bin/v8') def get_default_os(): system = platform.system().lower() @@ -404,6 +445,8 @@ def __main(args: list): getLogger().info(folder) return + # TODO: Add check to make sure there is only one commit specified if running for wasm + # Check to make sure we have something specified to test if parsed_args.commits or parsed_args.local_test_repo: if parsed_args.commits: From 35b1d70603a85926f248c6c39cabed039e30e0f0 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 7 Aug 2023 12:50:53 -0700 Subject: [PATCH 03/19] Add additional setup steps for the runtimes for each run. --- scripts/benchmarks_local.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index fec1822a66b..d87afca07a8 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -204,7 +204,17 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co if check_for_runtype_specified(parsed_args, [RunType.WasmWasm]): dest_dir_wasm = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, commit), "wasm_bundle") if force_regenerate or not os.path.exists(dest_dir_wasm): - build_runtime_dependency(parsed_args, repo_path, "mono+libs+host+packs", os_override="browser", arch_override="wasm", additional_args=[f'/p:AotHostArchitecture={parsed_args.architecture}', f'/p:AotHostOS={parsed_args.os}']) + provision_emsdk = [ + "make", + "-C", + os.path.join("src", "mono", "wasm"), + "provision-emsdk" + ] + RunCommand(provision_emsdk, verbose=True).run(os.path.join(repo_path)) + if not is_windows(parsed_args): + RunCommand(["export", f"EMSDK_PATH={os.path.join(repo_path, 'src', 'mono', 'wasm', 'emsdk')}"], verbose=True).run(os.path.join(repo_path)) + + build_runtime_dependency(parsed_args, repo_path, "mono+libs", os_override="browser", arch_override="wasm", additional_args=[f'/p:AotHostArchitecture={parsed_args.architecture}', f'/p:AotHostOS={parsed_args.os}']) src_dir = os.path.join(repo_path, "artifacts", "BrowserWasm", "staging", "dotnet-latest") dest_dir = os.path.join(repo_path, "artifacts", "bin", "wasm", "dotnet") From 8ddd09c69616a8eb0673e1029f5e3e93d5328851 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 7 Aug 2023 13:23:01 -0700 Subject: [PATCH 04/19] Add prereqs section, fix EMSDK_PATH setting, fix provision name. --- scripts/benchmarks_local.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index d87afca07a8..8701b6d89f2 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -14,6 +14,11 @@ # * Add the run type to the RunType enum # * Add the build instructions to the generate_all_runtime_artifacts function # * Add the BDN run arguments to the generate_benchmark_ci_args function +# +# Prereqs: +# Normal prereqs for building the target runtime +# Python 3 +# gitpython (pip install gitpython) import ctypes @@ -39,7 +44,7 @@ # Assumptions: We are only testing this Performance repo, should allow single run or multiple runs # For dotnet_version based runs, use the benchmarks_monthly .py script instead # Verify the input commands -# What are supported default cases: MonoJIT, MonoAOTLLVM, MonoInterpreter, Corerun, etc. (WASM) +# What are supported default cases: MonoJIT, MonoInterpreter, Corerun, WasmWasm etc. (WIP: MONOAOTLLVM, WASMAOT, WASMINTERPRETER) start_time = datetime.now() local_shared_string = "local" @@ -202,18 +207,19 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co shutil.rmtree(os.path.join(repo_path, "artifacts"), ignore_errors=True) # TODO: Can we trust the build system to update these when necessary or do we need to clean them up ourselves? if check_for_runtype_specified(parsed_args, [RunType.WasmWasm]): + ## TODO: Figure out prereq check flow + # Must have jsvu installed dest_dir_wasm = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, commit), "wasm_bundle") if force_regenerate or not os.path.exists(dest_dir_wasm): - provision_emsdk = [ + provision_wasm = [ "make", "-C", os.path.join("src", "mono", "wasm"), - "provision-emsdk" + "provision-wasm" ] - RunCommand(provision_emsdk, verbose=True).run(os.path.join(repo_path)) - if not is_windows(parsed_args): - RunCommand(["export", f"EMSDK_PATH={os.path.join(repo_path, 'src', 'mono', 'wasm', 'emsdk')}"], verbose=True).run(os.path.join(repo_path)) - + RunCommand(provision_wasm, verbose=True).run(os.path.join(repo_path)) + os.environ["EMSDK_PATH"] =os.path.join(repo_path, 'src', 'mono', 'wasm', 'emsdk') + build_runtime_dependency(parsed_args, repo_path, "mono+libs", os_override="browser", arch_override="wasm", additional_args=[f'/p:AotHostArchitecture={parsed_args.architecture}', f'/p:AotHostOS={parsed_args.os}']) src_dir = os.path.join(repo_path, "artifacts", "BrowserWasm", "staging", "dotnet-latest") From a4a46b2483848171f4797557fa9c0efb1a09f486 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 7 Aug 2023 14:29:52 -0700 Subject: [PATCH 05/19] Make sure directories are made for file copy. --- scripts/benchmarks_local.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 8701b6d89f2..6159bfeaea1 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -221,7 +221,6 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co os.environ["EMSDK_PATH"] =os.path.join(repo_path, 'src', 'mono', 'wasm', 'emsdk') build_runtime_dependency(parsed_args, repo_path, "mono+libs", os_override="browser", arch_override="wasm", additional_args=[f'/p:AotHostArchitecture={parsed_args.architecture}', f'/p:AotHostOS={parsed_args.os}']) - src_dir = os.path.join(repo_path, "artifacts", "BrowserWasm", "staging", "dotnet-latest") dest_dir = os.path.join(repo_path, "artifacts", "bin", "wasm", "dotnet") copy_directory_contents(src_dir, dest_dir) @@ -229,7 +228,10 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co dest_dir = os.path.join(repo_path, "artifacts", "bin", "wasm") copy_directory_contents(src_dir, dest_dir) src_file = os.path.join(repo_path, "src", "mono", "wasm", "test-main.js") - dest_file = os.path.join(repo_path, "artifacts", "bin", "wasm", "wasm-data", "test-main.js") + dest_dir = os.path.join(repo_path, "artifacts", "bin", "wasm", "wasm-data") + dest_file = os.path.join(dest_dir, "test-main.js") + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) shutil.copy2(src_file, dest_file) # Store the dotnet_mono in the artifact storage path From 7949d24f7282a9daf11faaa7cb1cf29c90e8d816 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 7 Aug 2023 14:39:28 -0700 Subject: [PATCH 06/19] Mitigate WasmWasm commit issue due to now have a way to run multiple wasm benchmarks at once. --- scripts/benchmarks_local.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 6159bfeaea1..14d4d6e712e 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -310,11 +310,12 @@ def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunTyp bdn_args_unescaped += [ corerun_path ] elif specific_run_type == RunType.WasmWasm: + # TODO: Fail or print somewhere that we only support one commit for WasmWasm. benchmark_ci_args += [ '--wasm' ] bdn_args_unescaped += [ '--anyCategories', 'Libraries', 'Runtime', '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono', - '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, commit), "wasm_bundle", "wasm-data"), + '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, all_commits[0]), "wasm_bundle", "wasm-data"), '--wasmEngine', parsed_args.wasm_engine_path, '--wasmArgs', '--experimental-wasm-eh --expose_wasm --module', # '--cli', '', From f0f8b26c3cce8527c3745baf24b4d9ff15f0c72c Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 7 Aug 2023 16:31:13 -0700 Subject: [PATCH 07/19] Add consistent dotnet instance so that tools (wasm-tools, etc.) can be installed and then passed into BDN. --- scripts/benchmarks_local.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 14d4d6e712e..9a64d82cc97 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -35,6 +35,7 @@ from subprocess import CalledProcessError import benchmarks_ci +import dotnet from channel_map import ChannelMap from git import GitCommandError from git.repo import Repo @@ -251,6 +252,7 @@ def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunTyp benchmark_ci_args += ['--architecture', parsed_args.architecture] benchmark_ci_args += ['--frameworks', parsed_args.framework] benchmark_ci_args += ['--filter', parsed_args.filter] + benchmark_ci_args += ['--dotnet-path', parsed_args.dotnet_path] benchmark_ci_args += ['--csproj', parsed_args.csproj] benchmark_ci_args += ['--incremental', "no"] benchmark_ci_args += ['--bdn-artifacts', os.path.join(parsed_args.artifact_storage_path, f"BenchmarkDotNet.Artifacts.{specific_run_type.name}.{start_time.strftime('%y%m%d_%H%M%S')}")] @@ -317,7 +319,7 @@ def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunTyp '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono', '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, all_commits[0]), "wasm_bundle", "wasm-data"), '--wasmEngine', parsed_args.wasm_engine_path, - '--wasmArgs', '--experimental-wasm-eh --expose_wasm --module', + '--wasmArgs', '\"--experimental-wasm-eh --expose_wasm --module\"', # '--cli', '', '--logBuildOutput', '--generateBinLog' @@ -373,6 +375,11 @@ def run_benchmarks(parsed_args: Namespace, commits: list) -> None: getLogger().info(f"Finished running benchmark for {run_type} at {commits}.") +def install_dotnet(parsed_args: Namespace) -> None: + if not os.path.exists(parsed_args.dotnet_path): #TODO Do we want to just always install dotnet? + dotnet.install(parsed_args.architecture, ["main"], parsed_args.dotnet_versions, parsed_args.verbose, parsed_args.dotnet_path) + dotnet.setup_dotnet(parsed_args.dotnet_path) + # Check if the specified references exist in the given repository URL. # If a reference does not exist, raise an exception. # @@ -403,6 +410,8 @@ def check_references_exist_and_add_branch_commits(repo_url: str, references: lis raise Exception(f"Reference {reference} does not exist in {repo_url}.") def add_arguments(parser): + dotnet.add_arguments(parser) + # Arguments for the local runner script parser.add_argument('--list-cached-builds', action='store_true', help='Lists the cached builds located in the artifact-storage-path.') parser.add_argument('--commits', nargs='+', type=str, help='The commits to test.') @@ -410,7 +419,7 @@ def add_arguments(parser): parser.add_argument('--local-test-repo', type=str, help='Path to a local repo with the runtime source code to test from.') parser.add_argument('--separate-repos', action='store_true', help='Whether to test each runtime version from their own separate repo directory.') # TODO: Do we want to have this as an actual option? It made sense before a shared build cache was added parser.add_argument('--repo-storage-path', type=str, default='.', help='The path to store the cloned repositories in.') - parser.add_argument('--artifact-storage-path', type=str, default=f'{os.getcwd()}{os.path.sep}runtime-testing-artifacts', help=f'The path to store the artifacts in (builds, results, etc). Default is {os.getcwd()}{os.path.sep}runtime-testing-artifacts') + parser.add_argument('--artifact-storage-path', type=str, default=os.path.join(os.getcwd(), "runtime-testing-artifacts"), help=f'The path to store the artifacts in (builds, results, etc). Default is {os.path.join(os.getcwd(), "runtime-testing-artifacts")}') parser.add_argument('--rebuild-artifacts', action='store_true', help='Whether to rebuild the artifacts for the specified commits before benchmarking.') parser.add_argument('--build-only', action='store_true', help='Whether to only build the artifacts for the specified commits and not run the benchmarks.') parser.add_argument('--skip-local-rebuild', action='store_true', help='Whether to skip rebuilding the local repo and use the already built version (if already built). Useful if you need to run against local changes again.') @@ -423,7 +432,7 @@ def __is_valid_run_type(value): return value parser.add_argument('--run-types', dest='run_type_names', nargs='+', type=__is_valid_run_type, choices=[run_type.name for run_type in RunType], help='The types of runs to perform.') parser.add_argument('--quiet', dest='verbose', action='store_false', help='Whether to not print verbose output.') - + # Arguments specifically for dependency generation and BDN parser.add_argument('--bdn-arguments', type=str, default="", help='Command line arguments to be passed to BenchmarkDotNet, wrapped in quotes') parser.add_argument('--architecture', choices=['x64', 'x86', 'arm64', 'arm'], default=get_machine_architecture(), help='Specifies the SDK processor architecture') @@ -444,9 +453,10 @@ def get_default_os(): def __main(args: list): # Define the ArgumentParser - parser = ArgumentParser(description='Run local benchmarks for the Performance repo.') + parser = ArgumentParser(description='Run local benchmarks for the Performance repo.', conflict_handler='resolve') add_arguments(parser) parsed_args = parser.parse_args(args) + parsed_args.dotnet_path = os.path.join(parsed_args.artifact_storage_path, "dotnet") setup_loggers(verbose=parsed_args.verbose) @@ -489,6 +499,9 @@ def __main(args: list): getLogger().info("Killing any running dotnet, vstest, or msbuild processes... (ignore system cannot find path specified)") kill_dotnet_processes(parsed_args) + # Install Dotnet so we can add tools + install_dotnet(parsed_args) + # Generate the artifacts for each of the remote versions if parsed_args.commits: getLogger().info(f"References {parsed_args.commits} exist in {repo_url}.") From 027c3fd84f960d23e307cac3fa0dc400a885e2ca Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 8 Aug 2023 12:18:02 -0700 Subject: [PATCH 08/19] Add wasmsdk tool install for WasmWasm runs. --- scripts/benchmarks_local.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 9a64d82cc97..13503e97395 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -239,6 +239,9 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co dotnet_wasm_path = os.path.join(repo_path, "artifacts", "bin", "wasm") shutil.rmtree(dest_dir_wasm, ignore_errors=True) copy_directory_contents(dotnet_wasm_path, dest_dir_wasm) + + # Add wasm-tools to dotnet instance: + RunCommand([parsed_args.dotnet_dir_path, "workload", "install", "wasmsdk"], verbose=True).run() else: getLogger().info(f"wasm_bundle already exists in {dest_dir_wasm}. Skipping generation.") @@ -252,7 +255,7 @@ def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunTyp benchmark_ci_args += ['--architecture', parsed_args.architecture] benchmark_ci_args += ['--frameworks', parsed_args.framework] benchmark_ci_args += ['--filter', parsed_args.filter] - benchmark_ci_args += ['--dotnet-path', parsed_args.dotnet_path] + benchmark_ci_args += ['--dotnet-path', parsed_args.dotnet_dir_path] benchmark_ci_args += ['--csproj', parsed_args.csproj] benchmark_ci_args += ['--incremental', "no"] benchmark_ci_args += ['--bdn-artifacts', os.path.join(parsed_args.artifact_storage_path, f"BenchmarkDotNet.Artifacts.{specific_run_type.name}.{start_time.strftime('%y%m%d_%H%M%S')}")] @@ -376,9 +379,9 @@ def run_benchmarks(parsed_args: Namespace, commits: list) -> None: getLogger().info(f"Finished running benchmark for {run_type} at {commits}.") def install_dotnet(parsed_args: Namespace) -> None: - if not os.path.exists(parsed_args.dotnet_path): #TODO Do we want to just always install dotnet? - dotnet.install(parsed_args.architecture, ["main"], parsed_args.dotnet_versions, parsed_args.verbose, parsed_args.dotnet_path) - dotnet.setup_dotnet(parsed_args.dotnet_path) + if not os.path.exists(parsed_args.dotnet_dir_path): #TODO Do we want to just always install dotnet? + dotnet.install(parsed_args.architecture, ["main"], parsed_args.dotnet_versions, parsed_args.verbose, parsed_args.dotnet_dir_path) + dotnet.setup_dotnet(parsed_args.dotnet_dir_path) # Check if the specified references exist in the given repository URL. # If a reference does not exist, raise an exception. @@ -456,7 +459,7 @@ def __main(args: list): parser = ArgumentParser(description='Run local benchmarks for the Performance repo.', conflict_handler='resolve') add_arguments(parser) parsed_args = parser.parse_args(args) - parsed_args.dotnet_path = os.path.join(parsed_args.artifact_storage_path, "dotnet") + parsed_args.dotnet_dir_path = os.path.join(parsed_args.artifact_storage_path, "dotnet") setup_loggers(verbose=parsed_args.verbose) From 74123480e1092f6705e433697a464de0f360d309 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 8 Aug 2023 15:00:13 -0700 Subject: [PATCH 09/19] Cleanup some comments. --- scripts/benchmarks_local.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 13503e97395..7698c3eb583 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -100,7 +100,6 @@ def copy_directory_contents(src_dir: str, dest_dir: str): # Builds libs and corerun by default def build_runtime_dependency(parsed_args: Namespace, repo_path: str, subset: str = "clr+libs", configuration: str = "Release", os_override = "", arch_override = "", additional_args: list = []): - # Run the command if is_windows(parsed_args): build_libs_and_corerun_command = [ "powershell", @@ -154,7 +153,6 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co core_root_path = os.path.join(repo_path, "artifacts", "tests", "coreclr", f"{parsed_args.os}.{parsed_args.architecture}.Release", "Tests", "Core_Root") shutil.rmtree(dest_dir, ignore_errors=True) copy_directory_contents(core_root_path, dest_dir) - # shutil.rmtree(os.path.join(repo_path, "artifacts"), ignore_errors=True) else: getLogger().info(f"CoreRun already exists in {dest_dir}. Skipping generation.") @@ -196,20 +194,18 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co copy_directory_contents(dotnet_mono_path, dest_dir_mono_interpreter) shutil.rmtree(dest_dir_mono_jit, ignore_errors=True) copy_directory_contents(dotnet_mono_path, dest_dir_mono_jit) - # shutil.rmtree(os.path.join(repo_path, "artifacts"), ignore_errors=True) else: getLogger().info(f"dotnet_mono already exists in {dest_dir_mono_interpreter} and {dest_dir_mono_jit}. Skipping generation.") if check_for_runtype_specified(parsed_args, [RunType.MonoAOTLLVM]): raise NotImplementedError("MonoAOTLLVM is not yet implemented.") # TODO: Finish MonoAOTLLVM Build stuff build_runtime_dependency(parsed_args, repo_path, "mono+libs+host+packs", additional_args=['/p:CrossBuild=false' '/p:MonoLLVMUseCxx11Abi=false']) - # TODO: Finish MonoAOTLLVM Build stuff # Clean up the build results shutil.rmtree(os.path.join(repo_path, "artifacts"), ignore_errors=True) # TODO: Can we trust the build system to update these when necessary or do we need to clean them up ourselves? if check_for_runtype_specified(parsed_args, [RunType.WasmWasm]): - ## TODO: Figure out prereq check flow - # Must have jsvu installed + # TODO: Figure out prereq check flow + # Must have jsvu installed also dest_dir_wasm = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, commit), "wasm_bundle") if force_regenerate or not os.path.exists(dest_dir_wasm): provision_wasm = [ @@ -315,7 +311,6 @@ def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunTyp bdn_args_unescaped += [ corerun_path ] elif specific_run_type == RunType.WasmWasm: - # TODO: Fail or print somewhere that we only support one commit for WasmWasm. benchmark_ci_args += [ '--wasm' ] bdn_args_unescaped += [ '--anyCategories', 'Libraries', 'Runtime', @@ -323,7 +318,6 @@ def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunTyp '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, all_commits[0]), "wasm_bundle", "wasm-data"), '--wasmEngine', parsed_args.wasm_engine_path, '--wasmArgs', '\"--experimental-wasm-eh --expose_wasm --module\"', - # '--cli', '', '--logBuildOutput', '--generateBinLog' ] @@ -531,7 +525,7 @@ def __main(args: list): finally: kill_dotnet_processes(parsed_args) - # TODO: Compare the results of the benchmarks || This is doable with just BDN as a start for now + # TODO: Compare the results of the benchmarks with results comparer (Currently will need to be done manually) if __name__ == "__main__": __main(sys.argv[1:]) \ No newline at end of file From 54645d0fe313bb34f20b6448331453dfbe26ebfa Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 8 Aug 2023 16:09:46 -0700 Subject: [PATCH 10/19] Fix workload being installed. --- scripts/benchmarks_local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 7698c3eb583..63c80b3c5fd 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -237,7 +237,7 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co copy_directory_contents(dotnet_wasm_path, dest_dir_wasm) # Add wasm-tools to dotnet instance: - RunCommand([parsed_args.dotnet_dir_path, "workload", "install", "wasmsdk"], verbose=True).run() + RunCommand([parsed_args.dotnet_dir_path, "workload", "install", "wasm-tools"], verbose=True).run() else: getLogger().info(f"wasm_bundle already exists in {dest_dir_wasm}. Skipping generation.") From 4a8f0fdbd7113273632416edad43f67d3756d089 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 8 Aug 2023 16:31:57 -0700 Subject: [PATCH 11/19] Add WasmAOT for testing and fixed build-nugets -> built-nugets typo. --- scripts/benchmarks_local.py | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 63c80b3c5fd..79385b24279 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -45,7 +45,7 @@ # Assumptions: We are only testing this Performance repo, should allow single run or multiple runs # For dotnet_version based runs, use the benchmarks_monthly .py script instead # Verify the input commands -# What are supported default cases: MonoJIT, MonoInterpreter, Corerun, WasmWasm etc. (WIP: MONOAOTLLVM, WASMAOT, WASMINTERPRETER) +# What are supported default cases: MonoJIT, MonoInterpreter, Corerun, WasmWasm etc. (WIP: MONOAOTLLVM) start_time = datetime.now() local_shared_string = "local" @@ -56,6 +56,7 @@ class RunType(Enum): MonoInterpreter = 3 MonoJIT = 4 WasmWasm = 5 + WasmAOT = 6 def is_windows(parsed_args: Namespace): return parsed_args.os == "windows" @@ -203,11 +204,12 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co # Clean up the build results shutil.rmtree(os.path.join(repo_path, "artifacts"), ignore_errors=True) # TODO: Can we trust the build system to update these when necessary or do we need to clean them up ourselves? - if check_for_runtype_specified(parsed_args, [RunType.WasmWasm]): + if check_for_runtype_specified(parsed_args, [RunType.WasmWasm, RunType.WasmAOT]): # TODO: Figure out prereq check flow # Must have jsvu installed also - dest_dir_wasm = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, commit), "wasm_bundle") - if force_regenerate or not os.path.exists(dest_dir_wasm): + dest_dir_wasm_wasm = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, commit), "wasm_bundle") + dest_dir_wasm_aot = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle") + if force_regenerate or not os.path.exists(dest_dir_wasm_wasm) or not os.path.exists(dest_dir_wasm_aot): provision_wasm = [ "make", "-C", @@ -221,7 +223,7 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co src_dir = os.path.join(repo_path, "artifacts", "BrowserWasm", "staging", "dotnet-latest") dest_dir = os.path.join(repo_path, "artifacts", "bin", "wasm", "dotnet") copy_directory_contents(src_dir, dest_dir) - src_dir = os.path.join(repo_path, "artifacts", "BrowserWasm", "staging", "build-nugets") + src_dir = os.path.join(repo_path, "artifacts", "BrowserWasm", "staging", "built-nugets") dest_dir = os.path.join(repo_path, "artifacts", "bin", "wasm") copy_directory_contents(src_dir, dest_dir) src_file = os.path.join(repo_path, "src", "mono", "wasm", "test-main.js") @@ -233,13 +235,15 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co # Store the dotnet_mono in the artifact storage path dotnet_wasm_path = os.path.join(repo_path, "artifacts", "bin", "wasm") - shutil.rmtree(dest_dir_wasm, ignore_errors=True) - copy_directory_contents(dotnet_wasm_path, dest_dir_wasm) + shutil.rmtree(dest_dir_wasm_wasm, ignore_errors=True) + copy_directory_contents(dotnet_wasm_path, dest_dir_wasm_wasm) + shutil.rmtree(dest_dir_wasm_aot, ignore_errors=True) + copy_directory_contents(dotnet_wasm_path, dest_dir_wasm_aot) # Add wasm-tools to dotnet instance: RunCommand([parsed_args.dotnet_dir_path, "workload", "install", "wasm-tools"], verbose=True).run() else: - getLogger().info(f"wasm_bundle already exists in {dest_dir_wasm}. Skipping generation.") + getLogger().info(f"wasm_bundle already exists in {dest_dir_wasm_wasm} and {dest_dir_wasm_aot}. Skipping generation.") getLogger().info(f"Finished generating dependencies for {' '.join(map(str, parsed_args.run_type_names))} run types in {repo_path} and stored in {parsed_args.artifact_storage_path}.") @@ -325,6 +329,22 @@ def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunTyp # for commit in all_commits: # TODO see if there is a way to run multiple Wasm's at once. # bdn_args_unescaped += [ os.path.join(get_run_artifact_path(parsed_args, RunType.CoreRun, commit), "Core_Root", f'corerun{".exe" if is_windows(parsed_args) else ""}') ] + elif specific_run_type == RunType.WasmAOT: + benchmark_ci_args += [ '--wasm' ] + bdn_args_unescaped += [ + '--anyCategories', 'Libraries', 'Runtime', + '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono', + '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, all_commits[0]), "wasm_bundle", "wasm-data"), + '--wasmEngine', parsed_args.wasm_engine_path, + '--wasmArgs', '\"--experimental-wasm-eh --expose_wasm --module\"', + '--aotcompilermode', 'wasm' + '--logBuildOutput', + '--generateBinLog' + ] + + # for commit in all_commits: # TODO see if there is a way to run multiple Wasm's at once. + # bdn_args_unescaped += [ os.path.join(get_run_artifact_path(parsed_args, RunType.CoreRun, commit), "Core_Root", f'corerun{".exe" if is_windows(parsed_args) else ""}') ] + if parsed_args.bdn_arguments: bdn_args_unescaped += [parsed_args.bdn_arguments] benchmark_ci_args += [f'--bdn-arguments={" ".join(bdn_args_unescaped)}'] From 9e07d231236c67350a1421caa4121be6dad6415b Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 9 Aug 2023 10:14:41 -0700 Subject: [PATCH 12/19] Fix WasmAOT BDN arg spacing typo. --- scripts/benchmarks_local.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 79385b24279..9d143fa993a 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -240,8 +240,8 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co shutil.rmtree(dest_dir_wasm_aot, ignore_errors=True) copy_directory_contents(dotnet_wasm_path, dest_dir_wasm_aot) - # Add wasm-tools to dotnet instance: - RunCommand([parsed_args.dotnet_dir_path, "workload", "install", "wasm-tools"], verbose=True).run() + # Add wasm-tools to dotnet instance: # TODO: Check if dotnet.exe is windows + RunCommand([os.path.join(parsed_args.dotnet_dir_path, f'dotnet{".exe" if is_windows(parsed_args) else ""}'), "workload", "install", "wasm-tools"], verbose=True).run() else: getLogger().info(f"wasm_bundle already exists in {dest_dir_wasm_wasm} and {dest_dir_wasm_aot}. Skipping generation.") @@ -337,7 +337,7 @@ def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunTyp '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, all_commits[0]), "wasm_bundle", "wasm-data"), '--wasmEngine', parsed_args.wasm_engine_path, '--wasmArgs', '\"--experimental-wasm-eh --expose_wasm --module\"', - '--aotcompilermode', 'wasm' + '--aotcompilermode', 'wasm', '--logBuildOutput', '--generateBinLog' ] From 40574338917d59a331ab32cb411402f57e5eda36 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 14 Aug 2023 13:06:43 -0700 Subject: [PATCH 13/19] Cleanup some comments, add reinstall-dotnet for benchmark building dotnet version, and prepare for moving to two paths for running benchmarks. --- scripts/benchmarks_local.py | 21 ++++++--------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 9d143fa993a..6603eb41f31 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -16,9 +16,10 @@ # * Add the BDN run arguments to the generate_benchmark_ci_args function # # Prereqs: -# Normal prereqs for building the target runtime +# Normal prereqs for building the target runtime: https://github.com/dotnet/runtime/blob/main/docs/workflow/requirements/linux-requirements.md # Python 3 # gitpython (pip install gitpython) +# Wasm need jsvu and https://github.com/dotnet/runtime/blob/main/docs/workflow/building/libraries/webassembly-instructions.md import ctypes @@ -201,11 +202,8 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co if check_for_runtype_specified(parsed_args, [RunType.MonoAOTLLVM]): raise NotImplementedError("MonoAOTLLVM is not yet implemented.") # TODO: Finish MonoAOTLLVM Build stuff build_runtime_dependency(parsed_args, repo_path, "mono+libs+host+packs", additional_args=['/p:CrossBuild=false' '/p:MonoLLVMUseCxx11Abi=false']) - # Clean up the build results - shutil.rmtree(os.path.join(repo_path, "artifacts"), ignore_errors=True) # TODO: Can we trust the build system to update these when necessary or do we need to clean them up ourselves? if check_for_runtype_specified(parsed_args, [RunType.WasmWasm, RunType.WasmAOT]): - # TODO: Figure out prereq check flow # Must have jsvu installed also dest_dir_wasm_wasm = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, commit), "wasm_bundle") dest_dir_wasm_aot = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle") @@ -240,12 +238,11 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co shutil.rmtree(dest_dir_wasm_aot, ignore_errors=True) copy_directory_contents(dotnet_wasm_path, dest_dir_wasm_aot) - # Add wasm-tools to dotnet instance: # TODO: Check if dotnet.exe is windows + # Add wasm-tools to dotnet instance RunCommand([os.path.join(parsed_args.dotnet_dir_path, f'dotnet{".exe" if is_windows(parsed_args) else ""}'), "workload", "install", "wasm-tools"], verbose=True).run() else: getLogger().info(f"wasm_bundle already exists in {dest_dir_wasm_wasm} and {dest_dir_wasm_aot}. Skipping generation.") - getLogger().info(f"Finished generating dependencies for {' '.join(map(str, parsed_args.run_type_names))} run types in {repo_path} and stored in {parsed_args.artifact_storage_path}.") def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunType, all_commits: list) -> list: @@ -314,6 +311,7 @@ def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunTyp corerun_path = corerun_capture[0] bdn_args_unescaped += [ corerun_path ] + # for commit in all_commits: There is not a way to run multiple Wasm's at once via CI, instead will split single run vs multi-run scenarios elif specific_run_type == RunType.WasmWasm: benchmark_ci_args += [ '--wasm' ] bdn_args_unescaped += [ @@ -325,9 +323,6 @@ def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunTyp '--logBuildOutput', '--generateBinLog' ] - - # for commit in all_commits: # TODO see if there is a way to run multiple Wasm's at once. - # bdn_args_unescaped += [ os.path.join(get_run_artifact_path(parsed_args, RunType.CoreRun, commit), "Core_Root", f'corerun{".exe" if is_windows(parsed_args) else ""}') ] elif specific_run_type == RunType.WasmAOT: benchmark_ci_args += [ '--wasm' ] @@ -341,9 +336,6 @@ def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunTyp '--logBuildOutput', '--generateBinLog' ] - - # for commit in all_commits: # TODO see if there is a way to run multiple Wasm's at once. - # bdn_args_unescaped += [ os.path.join(get_run_artifact_path(parsed_args, RunType.CoreRun, commit), "Core_Root", f'corerun{".exe" if is_windows(parsed_args) else ""}') ] if parsed_args.bdn_arguments: bdn_args_unescaped += [parsed_args.bdn_arguments] @@ -393,7 +385,7 @@ def run_benchmarks(parsed_args: Namespace, commits: list) -> None: getLogger().info(f"Finished running benchmark for {run_type} at {commits}.") def install_dotnet(parsed_args: Namespace) -> None: - if not os.path.exists(parsed_args.dotnet_dir_path): #TODO Do we want to just always install dotnet? + if not os.path.exists(parsed_args.dotnet_dir_path) or parsed_args.reinstall_dotnet: dotnet.install(parsed_args.architecture, ["main"], parsed_args.dotnet_versions, parsed_args.verbose, parsed_args.dotnet_dir_path) dotnet.setup_dotnet(parsed_args.dotnet_dir_path) @@ -438,6 +430,7 @@ def add_arguments(parser): parser.add_argument('--repo-storage-path', type=str, default='.', help='The path to store the cloned repositories in.') parser.add_argument('--artifact-storage-path', type=str, default=os.path.join(os.getcwd(), "runtime-testing-artifacts"), help=f'The path to store the artifacts in (builds, results, etc). Default is {os.path.join(os.getcwd(), "runtime-testing-artifacts")}') parser.add_argument('--rebuild-artifacts', action='store_true', help='Whether to rebuild the artifacts for the specified commits before benchmarking.') + parser.add_argument('--reinstall-dotnet', action='store_true', help='Whether to reinstall dotnet for use in building the benchmarks before running the benchmarks.') parser.add_argument('--build-only', action='store_true', help='Whether to only build the artifacts for the specified commits and not run the benchmarks.') parser.add_argument('--skip-local-rebuild', action='store_true', help='Whether to skip rebuilding the local repo and use the already built version (if already built). Useful if you need to run against local changes again.') parser.add_argument('--allow-non-admin-execution', action='store_true', help='Whether to allow non-admin execution of the script. Admin execution is highly recommended as it minimizes the chance of encountering errors, but may not be possible in all cases.') @@ -491,8 +484,6 @@ def __main(args: list): getLogger().info(folder) return - # TODO: Add check to make sure there is only one commit specified if running for wasm - # Check to make sure we have something specified to test if parsed_args.commits or parsed_args.local_test_repo: if parsed_args.commits: From 0b97124ea713ad22bee1358a2e8cdd01c602684d Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 14 Aug 2023 13:21:14 -0700 Subject: [PATCH 14/19] Split benchmark running and arg generation into multi-commit and single commit runs as Wasm scenarios only support one test set at a time via command line. --- scripts/benchmarks_local.py | 100 ++++++++++++++++++++++++++++++++---- 1 file changed, 91 insertions(+), 9 deletions(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 6603eb41f31..28d0b80b19f 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -13,7 +13,7 @@ # * Adding a new run type: # * Add the run type to the RunType enum # * Add the build instructions to the generate_all_runtime_artifacts function -# * Add the BDN run arguments to the generate_benchmark_ci_args function +# * Add the BDN run arguments to the generate_combined_benchmark_ci_args function # # Prereqs: # Normal prereqs for building the target runtime: https://github.com/dotnet/runtime/blob/main/docs/workflow/requirements/linux-requirements.md @@ -245,7 +245,7 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co getLogger().info(f"Finished generating dependencies for {' '.join(map(str, parsed_args.run_type_names))} run types in {repo_path} and stored in {parsed_args.artifact_storage_path}.") -def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunType, all_commits: list) -> list: +def generate_combined_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunType, all_commits: list[str]) -> list: getLogger().info(f"Generating benchmark_ci.py arguments for {specific_run_type.name} run type using artifacts in {parsed_args.artifact_storage_path}.") benchmark_ci_args = [] bdn_args_unescaped = [] @@ -311,13 +311,86 @@ def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunTyp corerun_path = corerun_capture[0] bdn_args_unescaped += [ corerun_path ] + # for commit in all_commits: There is not a way to run multiple Wasm's at once via CI, instead will split single run vs multi-run scenarios + elif specific_run_type == RunType.WasmWasm: + raise TypeError("WasmWasm does not support combined benchmark ci arg generation, use single benchmark generation and loop the benchmark_ci.py calls.") + + elif specific_run_type == RunType.WasmAOT: + raise TypeError("WasmWasm does not support combined benchmark ci arg generation, use single benchmark generation and loop the benchmark_ci.py calls.") + + if parsed_args.bdn_arguments: + bdn_args_unescaped += [parsed_args.bdn_arguments] + benchmark_ci_args += [f'--bdn-arguments={" ".join(bdn_args_unescaped)}'] + getLogger().info(f"Finished generating benchmark_ci.py arguments for {specific_run_type.name} run type using artifacts in {parsed_args.artifact_storage_path}.") + return benchmark_ci_args + +def generate_single_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunType, commit: str) -> list: + getLogger().info(f"Generating benchmark_ci.py arguments for {specific_run_type.name} run type using artifacts in {parsed_args.artifact_storage_path}.") + benchmark_ci_args = [] + bdn_args_unescaped = [] + benchmark_ci_args += ['--architecture', parsed_args.architecture] + benchmark_ci_args += ['--frameworks', parsed_args.framework] + benchmark_ci_args += ['--filter', parsed_args.filter] + benchmark_ci_args += ['--dotnet-path', parsed_args.dotnet_dir_path] + benchmark_ci_args += ['--csproj', parsed_args.csproj] + benchmark_ci_args += ['--incremental', "no"] + benchmark_ci_args += ['--bdn-artifacts', os.path.join(parsed_args.artifact_storage_path, f"BenchmarkDotNet.Artifacts.{specific_run_type.name}.{start_time.strftime('%y%m%d_%H%M%S')}")] + + if specific_run_type == RunType.CoreRun: + bdn_args_unescaped += [ + '--anyCategories', 'Libraries', 'Runtime', + '--logBuildOutput', + '--generateBinLog' + ] + bdn_args_unescaped += [ '--corerun', os.path.join(get_run_artifact_path(parsed_args, RunType.CoreRun, commit), "Core_Root", f'corerun{".exe" if is_windows(parsed_args) else ""}') ] + + elif specific_run_type == RunType.MonoAOTLLVM: + raise NotImplementedError("MonoAOTLLVM is not yet implemented.") + + elif specific_run_type == RunType.MonoInterpreter: + bdn_args_unescaped += [ + '--anyCategories', 'Libraries', 'Runtime', + '--category-exclusion-filter', 'NoInterpreter', 'NoMono', + '--logBuildOutput', + '--generateBinLog' + ] + + # We can force only one capture because the artifact_paths include the commit hash which is what we get the corerun from. + corerun_capture = glob.glob(os.path.join(get_run_artifact_path(parsed_args, RunType.MonoInterpreter, commit), "dotnet_mono", "shared", "Microsoft.NETCore.App", "*", f'corerun{".exe" if is_windows(parsed_args) else ""}')) + if len(corerun_capture) == 0: + raise Exception(f"Could not find corerun in {get_run_artifact_path(parsed_args, RunType.MonoInterpreter, commit)}") + elif len(corerun_capture) > 1: + raise Exception(f"Found multiple corerun in {get_run_artifact_path(parsed_args, RunType.MonoInterpreter, commit)}") + else: + corerun_path = corerun_capture[0] + bdn_args_unescaped += [ '--corerun', corerun_path ] + bdn_args_unescaped += ['--envVars', 'MONO_ENV_OPTIONS:--interpreter'] + + elif specific_run_type == RunType.MonoJIT: + bdn_args_unescaped += [ + '--anyCategories', 'Libraries', 'Runtime', + '--category-exclusion-filter', 'NoInterpreter', 'NoMono', + '--logBuildOutput', + '--generateBinLog' + ] + + # We can force only one capture because the artifact_paths include the commit hash which is what we get the corerun from. + corerun_capture = glob.glob(os.path.join(get_run_artifact_path(parsed_args, RunType.MonoJIT, commit), "dotnet_mono", "shared", "Microsoft.NETCore.App", "*", f'corerun{".exe" if is_windows(parsed_args) else ""}')) + if len(corerun_capture) == 0: + raise Exception(f"Could not find corerun in {get_run_artifact_path(parsed_args, RunType.MonoJIT, commit)}") + elif len(corerun_capture) > 1: + raise Exception(f"Found multiple corerun in {get_run_artifact_path(parsed_args, RunType.MonoJIT, commit)}") + else: + corerun_path = corerun_capture[0] + bdn_args_unescaped += [ '--corerun', corerun_path ] + # for commit in all_commits: There is not a way to run multiple Wasm's at once via CI, instead will split single run vs multi-run scenarios elif specific_run_type == RunType.WasmWasm: benchmark_ci_args += [ '--wasm' ] bdn_args_unescaped += [ '--anyCategories', 'Libraries', 'Runtime', '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono', - '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, all_commits[0]), "wasm_bundle", "wasm-data"), + '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, commit), "wasm_bundle", "wasm-data"), '--wasmEngine', parsed_args.wasm_engine_path, '--wasmArgs', '\"--experimental-wasm-eh --expose_wasm --module\"', '--logBuildOutput', @@ -329,7 +402,7 @@ def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunTyp bdn_args_unescaped += [ '--anyCategories', 'Libraries', 'Runtime', '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono', - '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, all_commits[0]), "wasm_bundle", "wasm-data"), + '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "wasm-data"), '--wasmEngine', parsed_args.wasm_engine_path, '--wasmArgs', '\"--experimental-wasm-eh --expose_wasm --module\"', '--aotcompilermode', 'wasm', @@ -340,7 +413,7 @@ def generate_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunTyp if parsed_args.bdn_arguments: bdn_args_unescaped += [parsed_args.bdn_arguments] benchmark_ci_args += [f'--bdn-arguments={" ".join(bdn_args_unescaped)}'] - getLogger().info(f"Finished generating benchmark_ci.py arguments for {specific_run_type.name} run type using artifacts in {parsed_args.artifact_storage_path}.") + getLogger().info(f"Finished generating benchmark_ci.py arguments for {specific_run_type.name} run type commit {commit} using artifacts in {parsed_args.artifact_storage_path}.") return benchmark_ci_args def generate_artifacts_for_commit(parsed_args: Namespace, repo_url: str, repo_dir: str, commit: str, is_local: bool = False) -> None: @@ -374,10 +447,19 @@ def run_benchmarks(parsed_args: Namespace, commits: list) -> None: for run_type in enum_name_list_to_enum_list(RunType, parsed_args.run_type_names): # Run the benchmarks_ci.py test and save results try: - benchmark_ci_args = generate_benchmark_ci_args(parsed_args, run_type, commits) - getLogger().info(f"Running benchmarks_ci.py for {run_type} at {commits} with arguments \"{' '.join(benchmark_ci_args)}\".") - kill_dotnet_processes(parsed_args) - benchmarks_ci.__main(benchmark_ci_args) # Build the runtime includes a download of dotnet at this location + if run_type in [RunType.CoreRun, RunType.MonoInterpreter, RunType.MonoJIT]: + benchmark_ci_args = generate_combined_benchmark_ci_args(parsed_args, run_type, commits) + getLogger().info(f"Running benchmarks_ci.py for {run_type} at {commits} with arguments \"{' '.join(benchmark_ci_args)}\".") + kill_dotnet_processes(parsed_args) + benchmarks_ci.__main(benchmark_ci_args) # Build the runtime includes a download of dotnet at this location + elif run_type in [RunType.WasmWasm, RunType.WasmAOT]: + for commit in commits: + benchmark_ci_args = generate_single_benchmark_ci_args(parsed_args, run_type, commit) + getLogger().info(f"Running single benchmarks_ci.py for {run_type} at {commit} with arguments \"{' '.join(benchmark_ci_args)}\".") + kill_dotnet_processes(parsed_args) + benchmarks_ci.__main(benchmark_ci_args) + else: + raise TypeError(f"Run type {run_type} is not supported. Please check the run type and try again.") except CalledProcessError: getLogger().error('benchmarks_ci exited with non zero exit code, please check the log and report benchmark failure') raise From 8a051fb44ba228c4c53ef19b311285cce59a2090 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Wed, 16 Aug 2023 15:13:55 -0700 Subject: [PATCH 15/19] Update prereqs area for correctness, and added commit to single BDN run artifacts. --- scripts/benchmarks_local.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 28d0b80b19f..8d2fae7d977 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -16,10 +16,10 @@ # * Add the BDN run arguments to the generate_combined_benchmark_ci_args function # # Prereqs: -# Normal prereqs for building the target runtime: https://github.com/dotnet/runtime/blob/main/docs/workflow/requirements/linux-requirements.md +# Normal prereqs for building the target runtime: https://github.com/dotnet/runtime/blob/main/docs/workflow/README.md#Build_Requirements # Python 3 # gitpython (pip install gitpython) -# Wasm need jsvu and https://github.com/dotnet/runtime/blob/main/docs/workflow/building/libraries/webassembly-instructions.md +# Wasm need jsvu installed and setup (No need to setup EMSDK, the tool does that automatically when building) import ctypes @@ -255,7 +255,7 @@ def generate_combined_benchmark_ci_args(parsed_args: Namespace, specific_run_typ benchmark_ci_args += ['--dotnet-path', parsed_args.dotnet_dir_path] benchmark_ci_args += ['--csproj', parsed_args.csproj] benchmark_ci_args += ['--incremental', "no"] - benchmark_ci_args += ['--bdn-artifacts', os.path.join(parsed_args.artifact_storage_path, f"BenchmarkDotNet.Artifacts.{specific_run_type.name}.{start_time.strftime('%y%m%d_%H%M%S')}")] + benchmark_ci_args += ['--bdn-artifacts', os.path.join(parsed_args.artifact_storage_path, f"BenchmarkDotNet.Artifacts.{specific_run_type.name}.{start_time.strftime('%y%m%d_%H%M%S')}")] # We don't include the commit hash in the artifact path because we are combining multiple runs into one if specific_run_type == RunType.CoreRun: bdn_args_unescaped += [ @@ -334,7 +334,7 @@ def generate_single_benchmark_ci_args(parsed_args: Namespace, specific_run_type: benchmark_ci_args += ['--dotnet-path', parsed_args.dotnet_dir_path] benchmark_ci_args += ['--csproj', parsed_args.csproj] benchmark_ci_args += ['--incremental', "no"] - benchmark_ci_args += ['--bdn-artifacts', os.path.join(parsed_args.artifact_storage_path, f"BenchmarkDotNet.Artifacts.{specific_run_type.name}.{start_time.strftime('%y%m%d_%H%M%S')}")] + benchmark_ci_args += ['--bdn-artifacts', os.path.join(parsed_args.artifact_storage_path, f"BenchmarkDotNet.Artifacts.{specific_run_type.name}.{commit}.{start_time.strftime('%y%m%d_%H%M%S')}")] # We add the commit hash to the artifact path because we are only running one commit at a time and they would clobber if running more than one commit perf type if specific_run_type == RunType.CoreRun: bdn_args_unescaped += [ From 931cfa21c274344e3ec1680d500bd277675992ec Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Thu, 17 Aug 2023 10:33:17 -0700 Subject: [PATCH 16/19] Rename WasmWasm to WasmInterpreter. --- scripts/benchmarks_local.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index 8d2fae7d977..ac636ca3343 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -46,7 +46,7 @@ # Assumptions: We are only testing this Performance repo, should allow single run or multiple runs # For dotnet_version based runs, use the benchmarks_monthly .py script instead # Verify the input commands -# What are supported default cases: MonoJIT, MonoInterpreter, Corerun, WasmWasm etc. (WIP: MONOAOTLLVM) +# What are supported default cases: MonoJIT, MonoInterpreter, Corerun, WasmInterpreter etc. (WIP: MONOAOTLLVM) start_time = datetime.now() local_shared_string = "local" @@ -56,7 +56,7 @@ class RunType(Enum): MonoAOTLLVM = 2 MonoInterpreter = 3 MonoJIT = 4 - WasmWasm = 5 + WasmInterpreter = 5 WasmAOT = 6 def is_windows(parsed_args: Namespace): @@ -203,9 +203,9 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co raise NotImplementedError("MonoAOTLLVM is not yet implemented.") # TODO: Finish MonoAOTLLVM Build stuff build_runtime_dependency(parsed_args, repo_path, "mono+libs+host+packs", additional_args=['/p:CrossBuild=false' '/p:MonoLLVMUseCxx11Abi=false']) - if check_for_runtype_specified(parsed_args, [RunType.WasmWasm, RunType.WasmAOT]): + if check_for_runtype_specified(parsed_args, [RunType.WasmInterpreter, RunType.WasmAOT]): # Must have jsvu installed also - dest_dir_wasm_wasm = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, commit), "wasm_bundle") + dest_dir_wasm_wasm = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmInterpreter, commit), "wasm_bundle") dest_dir_wasm_aot = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle") if force_regenerate or not os.path.exists(dest_dir_wasm_wasm) or not os.path.exists(dest_dir_wasm_aot): provision_wasm = [ @@ -312,11 +312,11 @@ def generate_combined_benchmark_ci_args(parsed_args: Namespace, specific_run_typ bdn_args_unescaped += [ corerun_path ] # for commit in all_commits: There is not a way to run multiple Wasm's at once via CI, instead will split single run vs multi-run scenarios - elif specific_run_type == RunType.WasmWasm: - raise TypeError("WasmWasm does not support combined benchmark ci arg generation, use single benchmark generation and loop the benchmark_ci.py calls.") + elif specific_run_type == RunType.WasmInterpreter: + raise TypeError("WasmInterpreter does not support combined benchmark ci arg generation, use single benchmark generation and loop the benchmark_ci.py calls.") elif specific_run_type == RunType.WasmAOT: - raise TypeError("WasmWasm does not support combined benchmark ci arg generation, use single benchmark generation and loop the benchmark_ci.py calls.") + raise TypeError("WasmInterpreter does not support combined benchmark ci arg generation, use single benchmark generation and loop the benchmark_ci.py calls.") if parsed_args.bdn_arguments: bdn_args_unescaped += [parsed_args.bdn_arguments] @@ -385,12 +385,12 @@ def generate_single_benchmark_ci_args(parsed_args: Namespace, specific_run_type: bdn_args_unescaped += [ '--corerun', corerun_path ] # for commit in all_commits: There is not a way to run multiple Wasm's at once via CI, instead will split single run vs multi-run scenarios - elif specific_run_type == RunType.WasmWasm: + elif specific_run_type == RunType.WasmInterpreter: benchmark_ci_args += [ '--wasm' ] bdn_args_unescaped += [ '--anyCategories', 'Libraries', 'Runtime', '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono', - '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmWasm, commit), "wasm_bundle", "wasm-data"), + '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmInterpreter, commit), "wasm_bundle", "wasm-data"), '--wasmEngine', parsed_args.wasm_engine_path, '--wasmArgs', '\"--experimental-wasm-eh --expose_wasm --module\"', '--logBuildOutput', @@ -452,7 +452,7 @@ def run_benchmarks(parsed_args: Namespace, commits: list) -> None: getLogger().info(f"Running benchmarks_ci.py for {run_type} at {commits} with arguments \"{' '.join(benchmark_ci_args)}\".") kill_dotnet_processes(parsed_args) benchmarks_ci.__main(benchmark_ci_args) # Build the runtime includes a download of dotnet at this location - elif run_type in [RunType.WasmWasm, RunType.WasmAOT]: + elif run_type in [RunType.WasmInterpreter, RunType.WasmAOT]: for commit in commits: benchmark_ci_args = generate_single_benchmark_ci_args(parsed_args, run_type, commit) getLogger().info(f"Running single benchmarks_ci.py for {run_type} at {commit} with arguments \"{' '.join(benchmark_ci_args)}\".") From b44eea1c318c83effc8db9261321fe82b83380b5 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Mon, 21 Aug 2023 12:19:20 -0700 Subject: [PATCH 17/19] Fix spacing around =. --- scripts/benchmarks_local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index ac636ca3343..c8d8e4549b7 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -215,7 +215,7 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co "provision-wasm" ] RunCommand(provision_wasm, verbose=True).run(os.path.join(repo_path)) - os.environ["EMSDK_PATH"] =os.path.join(repo_path, 'src', 'mono', 'wasm', 'emsdk') + os.environ["EMSDK_PATH"] = os.path.join(repo_path, 'src', 'mono', 'wasm', 'emsdk') build_runtime_dependency(parsed_args, repo_path, "mono+libs", os_override="browser", arch_override="wasm", additional_args=[f'/p:AotHostArchitecture={parsed_args.architecture}', f'/p:AotHostOS={parsed_args.os}']) src_dir = os.path.join(repo_path, "artifacts", "BrowserWasm", "staging", "dotnet-latest") From 625ecd92dc8ca3d57ac295cf9ab397b0b4f3a7f0 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 22 Aug 2023 12:32:16 -0700 Subject: [PATCH 18/19] Per PR suggestions. Renamed src_dir and dest_dir instances to be individually identifiable to prevent errors in the future due to unintended consequences. Also fixed some formatting including removing spaces at the beginning and end of lists, combining chained list appends into one list append, and lining up block list appends more left. --- scripts/benchmarks_local.py | 225 ++++++++++++++++++------------------ 1 file changed, 113 insertions(+), 112 deletions(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index c8d8e4549b7..e010f81674c 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -126,11 +126,10 @@ def build_runtime_dependency(parsed_args: Namespace, repo_path: str, subset: str def generate_layout(parsed_args: Namespace, repo_path: str, additional_args: list = []): # Run the command if is_windows(parsed_args): - build_script = "build.cmd" + generate_layout_command = ["build.cmd"] else: - build_script = "./build.sh" - generate_layout_command = [ - build_script, + generate_layout_command = ["./build.sh"] + generate_layout_command += [ "release", parsed_args.architecture, "generatelayoutonly", @@ -146,23 +145,23 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co getLogger().info(f"Generating dependencies for {' '.join(map(str, parsed_args.run_type_names))} run types in {repo_path} and storing in {parsed_args.artifact_storage_path}.") if check_for_runtype_specified(parsed_args, [RunType.CoreRun]): - dest_dir = os.path.join(get_run_artifact_path(parsed_args, RunType.CoreRun, commit), "Core_Root") + artifact_core_root = os.path.join(get_run_artifact_path(parsed_args, RunType.CoreRun, commit), "Core_Root") - if force_regenerate or not os.path.exists(dest_dir): + if force_regenerate or not os.path.exists(artifact_core_root): build_runtime_dependency(parsed_args, repo_path) generate_layout(parsed_args, repo_path) # Store the corerun in the artifact storage path - core_root_path = os.path.join(repo_path, "artifacts", "tests", "coreclr", f"{parsed_args.os}.{parsed_args.architecture}.Release", "Tests", "Core_Root") - shutil.rmtree(dest_dir, ignore_errors=True) - copy_directory_contents(core_root_path, dest_dir) + generated_core_root = os.path.join(repo_path, "artifacts", "tests", "coreclr", f"{parsed_args.os}.{parsed_args.architecture}.Release", "Tests", "Core_Root") + shutil.rmtree(artifact_core_root, ignore_errors=True) + copy_directory_contents(generated_core_root, artifact_core_root) else: - getLogger().info(f"CoreRun already exists in {dest_dir}. Skipping generation.") + getLogger().info(f"CoreRun already exists in {artifact_core_root}. Skipping generation.") if check_for_runtype_specified(parsed_args, [RunType.MonoInterpreter, RunType.MonoJIT]): - dest_dir_mono_interpreter = os.path.join(get_run_artifact_path(parsed_args, RunType.MonoInterpreter, commit), "dotnet_mono") - dest_dir_mono_jit = os.path.join(get_run_artifact_path(parsed_args, RunType.MonoJIT, commit), "dotnet_mono") + artifact_mono_interpreter = os.path.join(get_run_artifact_path(parsed_args, RunType.MonoInterpreter, commit), "dotnet_mono") + artifact_mono_jit = os.path.join(get_run_artifact_path(parsed_args, RunType.MonoJIT, commit), "dotnet_mono") - if force_regenerate or not os.path.exists(dest_dir_mono_interpreter) or not os.path.exists(dest_dir_mono_jit): + if force_regenerate or not os.path.exists(artifact_mono_interpreter) or not os.path.exists(artifact_mono_jit): build_runtime_dependency(parsed_args, repo_path, "clr+mono+libs") build_runtime_dependency(parsed_args, repo_path, "libs.pretest", additional_args=['-testscope', 'innerloop', '/p:RuntimeFlavor=mono', f"/p:RuntimeArtifactsPath={os.path.join(repo_path, 'artifacts', 'bin', 'mono', f'{parsed_args.os}.{parsed_args.architecture}.Release')}"]) @@ -179,25 +178,25 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co raise RuntimeError("ProductVersion or MajorVersion element not found in Versions.props file.") # Create the mono-dotnet - src_dir = os.path.join(repo_path, "artifacts", "bin", "runtime", f"net{major_version}.0-{parsed_args.os}-Release-{parsed_args.architecture}") - dest_dir = os.path.join(repo_path, "artifacts", "bin", "testhost", f"net{major_version}.0-{parsed_args.os}-Release-{parsed_args.architecture}", "shared", "Microsoft.NETCore.App", f"{product_version}") # Wrap product_version to force string type, otherwise we get warning: Argument of type "str | Any | None" cannot be assigned to parameter "paths" of type "BytesPath" in function "join" - copy_directory_contents(src_dir, dest_dir) - src_dir = os.path.join(repo_path, "artifacts", "bin", "testhost", f"net{major_version}.0-{parsed_args.os}-Release-{parsed_args.architecture}") - dest_dir = os.path.join(repo_path, "artifacts", "dotnet_mono") - copy_directory_contents(src_dir, dest_dir) - src_file = os.path.join(repo_path, "artifacts", "bin", "coreclr", f"{parsed_args.os}.{parsed_args.architecture}.Release", f"corerun{'.exe' if is_windows(parsed_args) else ''}") - dest_dir = os.path.join(repo_path, "artifacts", "dotnet_mono", "shared", "Microsoft.NETCore.App", f"{product_version}") # Wrap product_version to force string type, otherwise we get warning: Argument of type "str | Any | None" cannot be assigned to parameter "paths" of type "BytesPath" in function "join" - dest_file = os.path.join(dest_dir, f"corerun{'.exe' if is_windows(parsed_args) else ''}") - shutil.copy2(src_file, dest_file) + src_dir_runtime = os.path.join(repo_path, "artifacts", "bin", "runtime", f"net{major_version}.0-{parsed_args.os}-Release-{parsed_args.architecture}") + dest_dir_testhost_product = os.path.join(repo_path, "artifacts", "bin", "testhost", f"net{major_version}.0-{parsed_args.os}-Release-{parsed_args.architecture}", "shared", "Microsoft.NETCore.App", f"{product_version}") # Wrap product_version to force string type, otherwise we get warning: Argument of type "str | Any | None" cannot be assigned to parameter "paths" of type "BytesPath" in function "join" + copy_directory_contents(src_dir_runtime, dest_dir_testhost_product) + src_dir_testhost = os.path.join(repo_path, "artifacts", "bin", "testhost", f"net{major_version}.0-{parsed_args.os}-Release-{parsed_args.architecture}") + dest_dir_dotnet_mono = os.path.join(repo_path, "artifacts", "dotnet_mono") + copy_directory_contents(src_dir_testhost, dest_dir_dotnet_mono) + src_file_corerun = os.path.join(repo_path, "artifacts", "bin", "coreclr", f"{parsed_args.os}.{parsed_args.architecture}.Release", f"corerun{'.exe' if is_windows(parsed_args) else ''}") + dest_dir_dotnet_mono_shared = os.path.join(repo_path, "artifacts", "dotnet_mono", "shared", "Microsoft.NETCore.App", f"{product_version}") # Wrap product_version to force string type, otherwise we get warning: Argument of type "str | Any | None" cannot be assigned to parameter "paths" of type "BytesPath" in function "join" + dest_file_corerun = os.path.join(dest_dir_dotnet_mono_shared, f"corerun{'.exe' if is_windows(parsed_args) else ''}") + shutil.copy2(src_file_corerun, dest_file_corerun) # Store the dotnet_mono in the artifact storage path - dotnet_mono_path = os.path.join(repo_path, "artifacts", "dotnet_mono") - shutil.rmtree(dest_dir_mono_interpreter, ignore_errors=True) - copy_directory_contents(dotnet_mono_path, dest_dir_mono_interpreter) - shutil.rmtree(dest_dir_mono_jit, ignore_errors=True) - copy_directory_contents(dotnet_mono_path, dest_dir_mono_jit) + src_dir_dotnet_mono = os.path.join(repo_path, "artifacts", "dotnet_mono") + shutil.rmtree(artifact_mono_interpreter, ignore_errors=True) + copy_directory_contents(src_dir_dotnet_mono, artifact_mono_interpreter) + shutil.rmtree(artifact_mono_jit, ignore_errors=True) + copy_directory_contents(src_dir_dotnet_mono, artifact_mono_jit) else: - getLogger().info(f"dotnet_mono already exists in {dest_dir_mono_interpreter} and {dest_dir_mono_jit}. Skipping generation.") + getLogger().info(f"dotnet_mono already exists in {artifact_mono_interpreter} and {artifact_mono_jit}. Skipping generation.") if check_for_runtype_specified(parsed_args, [RunType.MonoAOTLLVM]): raise NotImplementedError("MonoAOTLLVM is not yet implemented.") # TODO: Finish MonoAOTLLVM Build stuff @@ -205,80 +204,80 @@ def generate_all_runtype_dependencies(parsed_args: Namespace, repo_path: str, co if check_for_runtype_specified(parsed_args, [RunType.WasmInterpreter, RunType.WasmAOT]): # Must have jsvu installed also - dest_dir_wasm_wasm = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmInterpreter, commit), "wasm_bundle") - dest_dir_wasm_aot = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle") - if force_regenerate or not os.path.exists(dest_dir_wasm_wasm) or not os.path.exists(dest_dir_wasm_aot): - provision_wasm = [ + artifact_wasm_wasm = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmInterpreter, commit), "wasm_bundle") + artifact_wasm_aot = os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle") + if force_regenerate or not os.path.exists(artifact_wasm_wasm) or not os.path.exists(artifact_wasm_aot): + provision_wasm_command = [ "make", "-C", os.path.join("src", "mono", "wasm"), "provision-wasm" ] - RunCommand(provision_wasm, verbose=True).run(os.path.join(repo_path)) + RunCommand(provision_wasm_command, verbose=True).run(os.path.join(repo_path)) os.environ["EMSDK_PATH"] = os.path.join(repo_path, 'src', 'mono', 'wasm', 'emsdk') build_runtime_dependency(parsed_args, repo_path, "mono+libs", os_override="browser", arch_override="wasm", additional_args=[f'/p:AotHostArchitecture={parsed_args.architecture}', f'/p:AotHostOS={parsed_args.os}']) - src_dir = os.path.join(repo_path, "artifacts", "BrowserWasm", "staging", "dotnet-latest") - dest_dir = os.path.join(repo_path, "artifacts", "bin", "wasm", "dotnet") - copy_directory_contents(src_dir, dest_dir) - src_dir = os.path.join(repo_path, "artifacts", "BrowserWasm", "staging", "built-nugets") - dest_dir = os.path.join(repo_path, "artifacts", "bin", "wasm") - copy_directory_contents(src_dir, dest_dir) - src_file = os.path.join(repo_path, "src", "mono", "wasm", "test-main.js") - dest_dir = os.path.join(repo_path, "artifacts", "bin", "wasm", "wasm-data") - dest_file = os.path.join(dest_dir, "test-main.js") - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) - shutil.copy2(src_file, dest_file) + src_dir_dotnet_latest = os.path.join(repo_path, "artifacts", "BrowserWasm", "staging", "dotnet-latest") + dest_dir_wasm_dotnet = os.path.join(repo_path, "artifacts", "bin", "wasm", "dotnet") + copy_directory_contents(src_dir_dotnet_latest, dest_dir_wasm_dotnet) + src_dir_built_nugets = os.path.join(repo_path, "artifacts", "BrowserWasm", "staging", "built-nugets") + dest_dir_bin_wasm = os.path.join(repo_path, "artifacts", "bin", "wasm") + copy_directory_contents(src_dir_built_nugets, dest_dir_bin_wasm) + src_file_test_main = os.path.join(repo_path, "src", "mono", "wasm", "test-main.js") + dest_dir_wasm_data = os.path.join(repo_path, "artifacts", "bin", "wasm", "wasm-data") + dest_file_test_main = os.path.join(dest_dir_wasm_data, "test-main.js") + if not os.path.exists(dest_dir_wasm_data): + os.makedirs(dest_dir_wasm_data) + shutil.copy2(src_file_test_main, dest_file_test_main) # Store the dotnet_mono in the artifact storage path - dotnet_wasm_path = os.path.join(repo_path, "artifacts", "bin", "wasm") - shutil.rmtree(dest_dir_wasm_wasm, ignore_errors=True) - copy_directory_contents(dotnet_wasm_path, dest_dir_wasm_wasm) - shutil.rmtree(dest_dir_wasm_aot, ignore_errors=True) - copy_directory_contents(dotnet_wasm_path, dest_dir_wasm_aot) + src_dir_dotnet_wasm = os.path.join(repo_path, "artifacts", "bin", "wasm") + shutil.rmtree(artifact_wasm_wasm, ignore_errors=True) + copy_directory_contents(src_dir_dotnet_wasm, artifact_wasm_wasm) + shutil.rmtree(artifact_wasm_aot, ignore_errors=True) + copy_directory_contents(src_dir_dotnet_wasm, artifact_wasm_aot) # Add wasm-tools to dotnet instance RunCommand([os.path.join(parsed_args.dotnet_dir_path, f'dotnet{".exe" if is_windows(parsed_args) else ""}'), "workload", "install", "wasm-tools"], verbose=True).run() else: - getLogger().info(f"wasm_bundle already exists in {dest_dir_wasm_wasm} and {dest_dir_wasm_aot}. Skipping generation.") + getLogger().info(f"wasm_bundle already exists in {artifact_wasm_wasm} and {artifact_wasm_aot}. Skipping generation.") getLogger().info(f"Finished generating dependencies for {' '.join(map(str, parsed_args.run_type_names))} run types in {repo_path} and stored in {parsed_args.artifact_storage_path}.") def generate_combined_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunType, all_commits: list[str]) -> list: getLogger().info(f"Generating benchmark_ci.py arguments for {specific_run_type.name} run type using artifacts in {parsed_args.artifact_storage_path}.") - benchmark_ci_args = [] bdn_args_unescaped = [] - benchmark_ci_args += ['--architecture', parsed_args.architecture] - benchmark_ci_args += ['--frameworks', parsed_args.framework] - benchmark_ci_args += ['--filter', parsed_args.filter] - benchmark_ci_args += ['--dotnet-path', parsed_args.dotnet_dir_path] - benchmark_ci_args += ['--csproj', parsed_args.csproj] - benchmark_ci_args += ['--incremental', "no"] - benchmark_ci_args += ['--bdn-artifacts', os.path.join(parsed_args.artifact_storage_path, f"BenchmarkDotNet.Artifacts.{specific_run_type.name}.{start_time.strftime('%y%m%d_%H%M%S')}")] # We don't include the commit hash in the artifact path because we are combining multiple runs into one + benchmark_ci_args = [ + '--architecture', parsed_args.architecture, + '--frameworks', parsed_args.framework, + '--filter', parsed_args.filter, + '--dotnet-path', parsed_args.dotnet_dir_path, + '--csproj', parsed_args.csproj, + '--incremental', "no", + '--bdn-artifacts', os.path.join(parsed_args.artifact_storage_path, f"BenchmarkDotNet.Artifacts.{specific_run_type.name}.{start_time.strftime('%y%m%d_%H%M%S')}") # We don't include the commit hash in the artifact path because we are combining multiple runs into on + ] if specific_run_type == RunType.CoreRun: bdn_args_unescaped += [ - '--anyCategories', 'Libraries', 'Runtime', - '--logBuildOutput', - '--generateBinLog' - ] - - bdn_args_unescaped += [ '--corerun' ] + '--anyCategories', 'Libraries', 'Runtime', + '--logBuildOutput', + '--generateBinLog' + ] + bdn_args_unescaped += ['--corerun'] for commit in all_commits: - bdn_args_unescaped += [ os.path.join(get_run_artifact_path(parsed_args, RunType.CoreRun, commit), "Core_Root", f'corerun{".exe" if is_windows(parsed_args) else ""}') ] + bdn_args_unescaped += [os.path.join(get_run_artifact_path(parsed_args, RunType.CoreRun, commit), "Core_Root", f'corerun{".exe" if is_windows(parsed_args) else ""}')] elif specific_run_type == RunType.MonoAOTLLVM: raise NotImplementedError("MonoAOTLLVM is not yet implemented.") elif specific_run_type == RunType.MonoInterpreter: bdn_args_unescaped += [ - '--anyCategories', 'Libraries', 'Runtime', - '--category-exclusion-filter', 'NoInterpreter', 'NoMono', - '--logBuildOutput', - '--generateBinLog' - ] - bdn_args_unescaped += [ '--corerun' ] + '--anyCategories', 'Libraries', 'Runtime', + '--category-exclusion-filter', 'NoInterpreter', 'NoMono', + '--logBuildOutput', + '--generateBinLog' + ] + bdn_args_unescaped += ['--corerun'] for commit in all_commits: # We can force only one capture because the artifact_paths include the commit hash which is what we get the corerun from. corerun_capture = glob.glob(os.path.join(get_run_artifact_path(parsed_args, RunType.MonoInterpreter, commit), "dotnet_mono", "shared", "Microsoft.NETCore.App", "*", f'corerun{".exe" if is_windows(parsed_args) else ""}')) @@ -288,18 +287,17 @@ def generate_combined_benchmark_ci_args(parsed_args: Namespace, specific_run_typ raise Exception(f"Found multiple corerun in {get_run_artifact_path(parsed_args, RunType.MonoInterpreter, commit)}") else: corerun_path = corerun_capture[0] - bdn_args_unescaped += [ corerun_path ] - + bdn_args_unescaped += [corerun_path] bdn_args_unescaped += ['--envVars', 'MONO_ENV_OPTIONS:--interpreter'] elif specific_run_type == RunType.MonoJIT: - bdn_args_unescaped += [ - '--anyCategories', 'Libraries', 'Runtime', - '--category-exclusion-filter', 'NoInterpreter', 'NoMono', - '--logBuildOutput', - '--generateBinLog' - ] - bdn_args_unescaped += [ '--corerun' ] + bdn_args_unescaped += [ + '--anyCategories', 'Libraries', 'Runtime', + '--category-exclusion-filter', 'NoInterpreter', 'NoMono', + '--logBuildOutput', + '--generateBinLog' + ] + bdn_args_unescaped += ['--corerun'] for commit in all_commits: # We can force only one capture because the artifact_paths include the commit hash which is what we get the corerun from. corerun_capture = glob.glob(os.path.join(get_run_artifact_path(parsed_args, RunType.MonoJIT, commit), "dotnet_mono", "shared", "Microsoft.NETCore.App", "*", f'corerun{".exe" if is_windows(parsed_args) else ""}')) @@ -309,7 +307,7 @@ def generate_combined_benchmark_ci_args(parsed_args: Namespace, specific_run_typ raise Exception(f"Found multiple corerun in {get_run_artifact_path(parsed_args, RunType.MonoJIT, commit)}") else: corerun_path = corerun_capture[0] - bdn_args_unescaped += [ corerun_path ] + bdn_args_unescaped += [corerun_path] # for commit in all_commits: There is not a way to run multiple Wasm's at once via CI, instead will split single run vs multi-run scenarios elif specific_run_type == RunType.WasmInterpreter: @@ -326,34 +324,35 @@ def generate_combined_benchmark_ci_args(parsed_args: Namespace, specific_run_typ def generate_single_benchmark_ci_args(parsed_args: Namespace, specific_run_type: RunType, commit: str) -> list: getLogger().info(f"Generating benchmark_ci.py arguments for {specific_run_type.name} run type using artifacts in {parsed_args.artifact_storage_path}.") - benchmark_ci_args = [] bdn_args_unescaped = [] - benchmark_ci_args += ['--architecture', parsed_args.architecture] - benchmark_ci_args += ['--frameworks', parsed_args.framework] - benchmark_ci_args += ['--filter', parsed_args.filter] - benchmark_ci_args += ['--dotnet-path', parsed_args.dotnet_dir_path] - benchmark_ci_args += ['--csproj', parsed_args.csproj] - benchmark_ci_args += ['--incremental', "no"] - benchmark_ci_args += ['--bdn-artifacts', os.path.join(parsed_args.artifact_storage_path, f"BenchmarkDotNet.Artifacts.{specific_run_type.name}.{commit}.{start_time.strftime('%y%m%d_%H%M%S')}")] # We add the commit hash to the artifact path because we are only running one commit at a time and they would clobber if running more than one commit perf type + benchmark_ci_args = [ + '--architecture', parsed_args.architecture, + '--frameworks', parsed_args.framework, + '--filter', parsed_args.filter, + '--dotnet-path', parsed_args.dotnet_dir_path, + '--csproj', parsed_args.csproj, + '--incremental', "no", + '--bdn-artifacts', os.path.join(parsed_args.artifact_storage_path, f"BenchmarkDotNet.Artifacts.{specific_run_type.name}.{commit}.{start_time.strftime('%y%m%d_%H%M%S')}") # We add the commit hash to the artifact path because we are only running one commit at a time and they would clobber if running more than one commit perf type + ] if specific_run_type == RunType.CoreRun: bdn_args_unescaped += [ - '--anyCategories', 'Libraries', 'Runtime', - '--logBuildOutput', - '--generateBinLog' - ] - bdn_args_unescaped += [ '--corerun', os.path.join(get_run_artifact_path(parsed_args, RunType.CoreRun, commit), "Core_Root", f'corerun{".exe" if is_windows(parsed_args) else ""}') ] + '--anyCategories', 'Libraries', 'Runtime', + '--logBuildOutput', + '--generateBinLog', + '--corerun', os.path.join(get_run_artifact_path(parsed_args, RunType.CoreRun, commit), "Core_Root", f'corerun{".exe" if is_windows(parsed_args) else ""}') + ] elif specific_run_type == RunType.MonoAOTLLVM: raise NotImplementedError("MonoAOTLLVM is not yet implemented.") elif specific_run_type == RunType.MonoInterpreter: bdn_args_unescaped += [ - '--anyCategories', 'Libraries', 'Runtime', - '--category-exclusion-filter', 'NoInterpreter', 'NoMono', - '--logBuildOutput', - '--generateBinLog' - ] + '--anyCategories', 'Libraries', 'Runtime', + '--category-exclusion-filter', 'NoInterpreter', 'NoMono', + '--logBuildOutput', + '--generateBinLog' + ] # We can force only one capture because the artifact_paths include the commit hash which is what we get the corerun from. corerun_capture = glob.glob(os.path.join(get_run_artifact_path(parsed_args, RunType.MonoInterpreter, commit), "dotnet_mono", "shared", "Microsoft.NETCore.App", "*", f'corerun{".exe" if is_windows(parsed_args) else ""}')) @@ -363,16 +362,18 @@ def generate_single_benchmark_ci_args(parsed_args: Namespace, specific_run_type: raise Exception(f"Found multiple corerun in {get_run_artifact_path(parsed_args, RunType.MonoInterpreter, commit)}") else: corerun_path = corerun_capture[0] - bdn_args_unescaped += [ '--corerun', corerun_path ] - bdn_args_unescaped += ['--envVars', 'MONO_ENV_OPTIONS:--interpreter'] + bdn_args_unescaped += [ + '--corerun', corerun_path, + '--envVars', 'MONO_ENV_OPTIONS:--interpreter' + ] elif specific_run_type == RunType.MonoJIT: - bdn_args_unescaped += [ - '--anyCategories', 'Libraries', 'Runtime', - '--category-exclusion-filter', 'NoInterpreter', 'NoMono', - '--logBuildOutput', - '--generateBinLog' - ] + bdn_args_unescaped += [ + '--anyCategories', 'Libraries', 'Runtime', + '--category-exclusion-filter', 'NoInterpreter', 'NoMono', + '--logBuildOutput', + '--generateBinLog' + ] # We can force only one capture because the artifact_paths include the commit hash which is what we get the corerun from. corerun_capture = glob.glob(os.path.join(get_run_artifact_path(parsed_args, RunType.MonoJIT, commit), "dotnet_mono", "shared", "Microsoft.NETCore.App", "*", f'corerun{".exe" if is_windows(parsed_args) else ""}')) @@ -382,11 +383,11 @@ def generate_single_benchmark_ci_args(parsed_args: Namespace, specific_run_type: raise Exception(f"Found multiple corerun in {get_run_artifact_path(parsed_args, RunType.MonoJIT, commit)}") else: corerun_path = corerun_capture[0] - bdn_args_unescaped += [ '--corerun', corerun_path ] + bdn_args_unescaped += ['--corerun', corerun_path] # for commit in all_commits: There is not a way to run multiple Wasm's at once via CI, instead will split single run vs multi-run scenarios elif specific_run_type == RunType.WasmInterpreter: - benchmark_ci_args += [ '--wasm' ] + benchmark_ci_args += ['--wasm'] bdn_args_unescaped += [ '--anyCategories', 'Libraries', 'Runtime', '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono', @@ -398,7 +399,7 @@ def generate_single_benchmark_ci_args(parsed_args: Namespace, specific_run_type: ] elif specific_run_type == RunType.WasmAOT: - benchmark_ci_args += [ '--wasm' ] + benchmark_ci_args += ['--wasm'] bdn_args_unescaped += [ '--anyCategories', 'Libraries', 'Runtime', '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono', From 491b3d196ceeaf90d702e45b7271ab569f5668b2 Mon Sep 17 00:00:00 2001 From: Parker Bibus Date: Tue, 22 Aug 2023 13:50:26 -0700 Subject: [PATCH 19/19] Fix alignment missed in previous commit. --- scripts/benchmarks_local.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/scripts/benchmarks_local.py b/scripts/benchmarks_local.py index e010f81674c..65809dac1c1 100644 --- a/scripts/benchmarks_local.py +++ b/scripts/benchmarks_local.py @@ -389,27 +389,27 @@ def generate_single_benchmark_ci_args(parsed_args: Namespace, specific_run_type: elif specific_run_type == RunType.WasmInterpreter: benchmark_ci_args += ['--wasm'] bdn_args_unescaped += [ - '--anyCategories', 'Libraries', 'Runtime', - '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono', - '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmInterpreter, commit), "wasm_bundle", "wasm-data"), - '--wasmEngine', parsed_args.wasm_engine_path, - '--wasmArgs', '\"--experimental-wasm-eh --expose_wasm --module\"', - '--logBuildOutput', - '--generateBinLog' - ] + '--anyCategories', 'Libraries', 'Runtime', + '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono', + '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmInterpreter, commit), "wasm_bundle", "wasm-data"), + '--wasmEngine', parsed_args.wasm_engine_path, + '--wasmArgs', '\"--experimental-wasm-eh --expose_wasm --module\"', + '--logBuildOutput', + '--generateBinLog' + ] elif specific_run_type == RunType.WasmAOT: benchmark_ci_args += ['--wasm'] bdn_args_unescaped += [ - '--anyCategories', 'Libraries', 'Runtime', - '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono', - '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "wasm-data"), - '--wasmEngine', parsed_args.wasm_engine_path, - '--wasmArgs', '\"--experimental-wasm-eh --expose_wasm --module\"', - '--aotcompilermode', 'wasm', - '--logBuildOutput', - '--generateBinLog' - ] + '--anyCategories', 'Libraries', 'Runtime', + '--category-exclusion-filter', 'NoInterpreter', 'NoWASM', 'NoMono', + '--wasmDataDir', os.path.join(get_run_artifact_path(parsed_args, RunType.WasmAOT, commit), "wasm_bundle", "wasm-data"), + '--wasmEngine', parsed_args.wasm_engine_path, + '--wasmArgs', '\"--experimental-wasm-eh --expose_wasm --module\"', + '--aotcompilermode', 'wasm', + '--logBuildOutput', + '--generateBinLog' + ] if parsed_args.bdn_arguments: bdn_args_unescaped += [parsed_args.bdn_arguments]