From 10928968556f39d9343967141ea1bb65d4bf739f Mon Sep 17 00:00:00 2001 From: Jeremy Date: Fri, 15 Dec 2023 17:04:37 -0500 Subject: [PATCH 1/2] Organize the turnkey benchmark help menu Signed-off-by: Jeremy --- src/turnkeyml/cli/cli.py | 183 ++++++++++++++++++++++----------------- 1 file changed, 105 insertions(+), 78 deletions(-) diff --git a/src/turnkeyml/cli/cli.py b/src/turnkeyml/cli/cli.py index ef05573..669dd9c 100644 --- a/src/turnkeyml/cli/cli.py +++ b/src/turnkeyml/cli/cli.py @@ -99,7 +99,9 @@ def check_extension(choices, file_name): return file_name benchmark_parser = subparsers.add_parser( - "benchmark", help="Benchmark the performance of one or more models" + "benchmark", + help="Benchmark the performance of one or more models", + description="Analyze, build, and then benchmark the model(s) within input file(s).", ) benchmark_parser.set_defaults(func=benchmark_command) @@ -110,40 +112,31 @@ def check_extension(choices, file_name): type=lambda file: check_extension(("py", "onnx", "txt"), file), ) - slurm_or_processes_group = benchmark_parser.add_mutually_exclusive_group() - - slurm_or_processes_group.add_argument( - "--use-slurm", - dest="use_slurm", - help="Execute on Slurm instead of using local compute resources", - action="store_true", + toolchain_select_group = benchmark_parser.add_argument_group( + "Select which phase(s) of the toolchain to run (default is to run analyze, build, and benchmark)" ) - slurm_or_processes_group.add_argument( - "--process-isolation", - dest="process_isolation", - help="Isolate evaluating each input into a separate process", + toolchain_select_group.add_argument( + "-a", + "--analyze-only", + dest="analyze_only", + help="Stop this command after the analyze phase", action="store_true", ) - benchmark_parser.add_argument( - "--lean-cache", - dest="lean_cache", - help="Delete all build artifacts except for log files when the command completes", + toolchain_select_group.add_argument( + "-b", + "--build-only", + dest="build_only", + help="Stop this command after the analyze and build phases", action="store_true", ) - benchmark_parser.add_argument( - "-d", - "--cache-dir", - dest="cache_dir", - help="Build cache directory where the resulting build directories will " - f"be stored (defaults to {filesystem.DEFAULT_CACHE_DIR})", - required=False, - default=filesystem.DEFAULT_CACHE_DIR, + analyze_group = benchmark_parser.add_argument_group( + "Options that specifically apply to the `analyze` phase of the toolflow" ) - benchmark_parser.add_argument( + analyze_group.add_argument( "--labels", dest="labels", help="Only benchmark the scripts that have the provided labels", @@ -151,27 +144,27 @@ def check_extension(choices, file_name): default=[], ) - benchmark_parser.add_argument( - "--sequence", - choices=SUPPORTED_SEQUENCES.keys(), - dest="sequence", - help="Name of a build sequence that will define the model-to-model transformations, " - "used to build the models. Each runtime has a default sequence that it uses.", - required=False, - default=None, + analyze_group.add_argument( + "--script-args", + dest="script_args", + type=str, + help="Arguments to pass into the target script(s)", ) - benchmark_parser.add_argument( - "--rebuild", - choices=build.REBUILD_OPTIONS, - dest="rebuild", - help=f"Sets the cache rebuild policy (defaults to {build.DEFAULT_REBUILD_POLICY})", - required=False, - default=build.DEFAULT_REBUILD_POLICY, + analyze_group.add_argument( + "--max-depth", + dest="max_depth", + type=int, + default=0, + help="Maximum depth to analyze within the model structure of the target script(s)", + ) + + both_build_benchmark_group = benchmark_parser.add_argument_group( + "Options that apply to both the `build` and `benchmark` phases of the toolflow" ) benchmark_default_device = "x86" - benchmark_parser.add_argument( + both_build_benchmark_group.add_argument( "--device", choices=SUPPORTED_DEVICES, dest="device", @@ -181,7 +174,7 @@ def check_extension(choices, file_name): default=benchmark_default_device, ) - benchmark_parser.add_argument( + both_build_benchmark_group.add_argument( "--runtime", choices=SUPPORTED_RUNTIMES.keys(), dest="runtime", @@ -193,46 +186,47 @@ def check_extension(choices, file_name): default=None, ) - benchmark_parser.add_argument( - "--iterations", - dest="iterations", - type=int, - default=100, - help="Number of execution iterations of the model to capture\ - the benchmarking performance (e.g., mean latency)", + both_build_benchmark_group.add_argument( + "-d", + "--cache-dir", + dest="cache_dir", + help="Build cache directory where the resulting build directories will " + f"be stored (defaults to {filesystem.DEFAULT_CACHE_DIR})", + required=False, + default=filesystem.DEFAULT_CACHE_DIR, ) - benchmark_parser.add_argument( - "--analyze-only", - dest="analyze_only", - help="Stop this command after the analysis phase", + both_build_benchmark_group.add_argument( + "--lean-cache", + dest="lean_cache", + help="Delete all build artifacts except for log files when the command completes", action="store_true", ) - benchmark_parser.add_argument( - "-b", - "--build-only", - dest="build_only", - help="Stop this command after the build phase", - action="store_true", + build_group = benchmark_parser.add_argument_group( + "Options that apply specifically to the `build` phase of the toolflow" ) - benchmark_parser.add_argument( - "--script-args", - dest="script_args", - type=str, - help="Arguments to pass into the target script(s)", + build_group.add_argument( + "--sequence", + choices=SUPPORTED_SEQUENCES.keys(), + dest="sequence", + help="Name of a build sequence that will define the model-to-model transformations, " + "used to build the models. Each runtime has a default sequence that it uses.", + required=False, + default=None, ) - benchmark_parser.add_argument( - "--max-depth", - dest="max_depth", - type=int, - default=0, - help="Maximum depth to analyze within the model structure of the target script(s)", + build_group.add_argument( + "--rebuild", + choices=build.REBUILD_OPTIONS, + dest="rebuild", + help=f"Sets the cache rebuild policy (defaults to {build.DEFAULT_REBUILD_POLICY})", + required=False, + default=build.DEFAULT_REBUILD_POLICY, ) - benchmark_parser.add_argument( + build_group.add_argument( "--onnx-opset", dest="onnx_opset", type=int, @@ -241,16 +235,20 @@ def check_extension(choices, file_name): "Not applicable when input model is already a .onnx file.", ) - benchmark_parser.add_argument( - "--timeout", + benchmark_group = benchmark_parser.add_argument_group( + "Options that apply specifically to the `benchmark` phase of the toolflow" + ) + + benchmark_group.add_argument( + "--iterations", + dest="iterations", type=int, - default=None, - help="Build timeout, in seconds, after which a build will be canceled " - f"(default={DEFAULT_TIMEOUT_SECONDS}). Only " - "applies when --process-isolation or --use-slurm is also used.", + default=100, + help="Number of execution iterations of the model to capture\ + the benchmarking performance (e.g., mean latency)", ) - benchmark_parser.add_argument( + benchmark_group.add_argument( "--rt-args", dest="rt_args", type=str, @@ -258,6 +256,35 @@ def check_extension(choices, file_name): help="Optional arguments provided to the runtime being used", ) + all_toolflows_group = benchmark_parser.add_argument_group( + "Options that apply to all toolflows" + ) + + slurm_or_processes_group = all_toolflows_group.add_mutually_exclusive_group() + + slurm_or_processes_group.add_argument( + "--use-slurm", + dest="use_slurm", + help="Execute on Slurm instead of using local compute resources", + action="store_true", + ) + + slurm_or_processes_group.add_argument( + "--process-isolation", + dest="process_isolation", + help="Isolate evaluating each input into a separate process", + action="store_true", + ) + + all_toolflows_group.add_argument( + "--timeout", + type=int, + default=None, + help="Build timeout, in seconds, after which a build will be canceled " + f"(default={DEFAULT_TIMEOUT_SECONDS}). Only " + "applies when --process-isolation or --use-slurm is also used.", + ) + ####################################### # Subparser for the "cache" command ####################################### From ccd1de7e1818bf3eb5bbe9670c5fbb9ee85333b1 Mon Sep 17 00:00:00 2001 From: Jeremy Date: Fri, 15 Dec 2023 17:10:31 -0500 Subject: [PATCH 2/2] Defence against Krishna Signed-off-by: Jeremy --- src/turnkeyml/cli/cli.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/turnkeyml/cli/cli.py b/src/turnkeyml/cli/cli.py index 669dd9c..c32b6ea 100644 --- a/src/turnkeyml/cli/cli.py +++ b/src/turnkeyml/cli/cli.py @@ -113,7 +113,8 @@ def check_extension(choices, file_name): ) toolchain_select_group = benchmark_parser.add_argument_group( - "Select which phase(s) of the toolchain to run (default is to run analyze, build, and benchmark)" + "Select which phase(s) of the toolchain to run " + "(default is to run analyze, build, and benchmark)" ) toolchain_select_group.add_argument(