From 63514421ec937ca3ba0051078704a2b72be003fc Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Mon, 4 Dec 2023 16:22:42 -0500 Subject: [PATCH 1/4] Better naming for Stats methods and members Signed-off-by: Jeremy Fowers --- docs/contribute.md | 2 +- .../turnkeyml_plugin_example_rt/runtime.py | 4 +- src/turnkeyml/analyze/script.py | 60 ++++++++++--------- src/turnkeyml/build/export.py | 36 +++++++---- src/turnkeyml/build/hummingbird.py | 5 +- src/turnkeyml/build/ignition.py | 4 +- src/turnkeyml/build/stage.py | 6 +- src/turnkeyml/build_api.py | 6 +- src/turnkeyml/cli/report.py | 4 +- src/turnkeyml/common/build.py | 23 +++---- src/turnkeyml/common/filesystem.py | 39 +++++------- src/turnkeyml/model_api.py | 6 +- src/turnkeyml/run/tensorrt/runtime.py | 7 ++- test/cli.py | 2 +- test/helpers/common.py | 13 ++-- 15 files changed, 113 insertions(+), 104 deletions(-) diff --git a/docs/contribute.md b/docs/contribute.md index d1e88dc..fa6596a 100644 --- a/docs/contribute.md +++ b/docs/contribute.md @@ -87,7 +87,7 @@ To add a runtime to a plugin: - `"RuntimeClass": `, where `` is a unique name for a Python class that inherits `BaseRT` and implements the runtime. - For example, `"RuntimeClass": ExampleRT` implements the `example` runtime. - The interface for the runtime class is defined in [Runtime Class](#runtime-class) below. - - (Optional) `"status_stats": List[str]`: a list of keys from the build stats that should be printed out at the end of benchmarking in the CLI's `Status` output. These keys, and corresponding values, must be set in the runtime class using `self.stats.add_build_stat(key, value)`. + - (Optional) `"status_stats": List[str]`: a list of keys from the build stats that should be printed out at the end of benchmarking in the CLI's `Status` output. These keys, and corresponding values, must be set in the runtime class using `self.stats.save_model_eval_stat(key, value)`. - (Optional) `"requirement_check": Callable`: a callable that runs before each benchmark. This may be used to check whether the device selected is available and functional before each benchmarking run. Exceptions raised during this callable will halt the benchmark of all selected files. 1. Populate the package with the following files (see [Plugin Directory Layout](#plugin-directory-layout)): diff --git a/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py b/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py index 7846a2a..2dc117d 100644 --- a/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py +++ b/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py @@ -51,8 +51,8 @@ def benchmark(self) -> MeasuredPerformance: # Assign values to the stats that will be printed # out by the CLI when status is reported - self.stats.add_build_stat("magic_perf_points", 42) - self.stats.add_build_stat("super_runtime_points", 100) + self.stats.save_model_eval_stat("magic_perf_points", 42) + self.stats.save_model_eval_stat("super_runtime_points", 100) return MeasuredPerformance( mean_latency=self.mean_latency, diff --git a/src/turnkeyml/analyze/script.py b/src/turnkeyml/analyze/script.py index 27c13ef..10e3540 100644 --- a/src/turnkeyml/analyze/script.py +++ b/src/turnkeyml/analyze/script.py @@ -157,37 +157,37 @@ def explore_invocation( invocation_info.stats_keys = [] # Create an ID for the build stats by combining the device and runtime. - # We don't need more info in the stats_id because changes to benchmark_model() + # We don't need more info in the evaluation_id because changes to benchmark_model() # arguments (e.g., sequence) will trigger a rebuild, which is intended to replace the # build stats so long as the device and runtime have not changed. - stats_id = f"{tracer_args.device}_{selected_runtime}" + evaluation_id = f"{tracer_args.device}_{selected_runtime}" stats = fs.Stats( tracer_args.cache_dir, build_name, - stats_id, + evaluation_id, ) invocation_info.stats = stats # Stats that apply to the model, regardless of build - stats.save_stat( + stats.save_model_stat( fs.Keys.HASH, model_info.hash, ) - stats.save_stat( + stats.save_model_stat( fs.Keys.MODEL_NAME, tracer_args.script_name, ) - stats.save_stat( + stats.save_model_stat( fs.Keys.PARAMETERS, model_info.params, ) if fs.Keys.AUTHOR in tracer_args.labels: - stats.save_stat(fs.Keys.AUTHOR, tracer_args.labels[fs.Keys.AUTHOR][0]) + stats.save_model_stat(fs.Keys.AUTHOR, tracer_args.labels[fs.Keys.AUTHOR][0]) if fs.Keys.CLASS in tracer_args.labels: - stats.save_stat(fs.Keys.CLASS, tracer_args.labels[fs.Keys.CLASS][0]) + stats.save_model_stat(fs.Keys.CLASS, tracer_args.labels[fs.Keys.CLASS][0]) if fs.Keys.TASK in tracer_args.labels: - stats.save_stat(fs.Keys.TASK, tracer_args.labels[fs.Keys.TASK][0]) + stats.save_model_stat(fs.Keys.TASK, tracer_args.labels[fs.Keys.TASK][0]) # If the input script is a built-in TurnkeyML model, make a note of # which one @@ -205,18 +205,18 @@ def explore_invocation( fs.MODELS_DIR, f"https://github.com/onnx/turnkeyml/tree/{git_hash}/models", ).replace("\\", "/") - stats.save_stat(fs.Keys.MODEL_SCRIPT, relative_path) + stats.save_model_stat(fs.Keys.MODEL_SCRIPT, relative_path) # Build-specific stats - stats.add_build_stat( + stats.save_model_eval_stat( fs.Keys.DEVICE_TYPE, tracer_args.device, ) - stats.add_build_stat( + stats.save_model_eval_stat( fs.Keys.RUNTIME, selected_runtime, ) - stats.add_build_stat( + stats.save_model_eval_stat( fs.Keys.ITERATIONS, tracer_args.iterations, ) @@ -235,12 +235,14 @@ def explore_invocation( # we will try to catch the exception and note it in the stats. # If a concluded build still has a status of "running", this means # there was an uncaught exception. - stats.add_build_stat(fs.Keys.BENCHMARK_STATUS, fs.BenchmarkStatus.RUNNING) + stats.save_model_eval_stat( + fs.Keys.BENCHMARK_STATUS, fs.BenchmarkStatus.RUNNING + ) perf = benchmark_model( model_info.model, inputs, - stats_id=stats_id, + evaluation_id=evaluation_id, device=tracer_args.device, runtime=selected_runtime, build_name=build_name, @@ -265,7 +267,7 @@ def explore_invocation( invocation_info.status_message = f"Build Error: {e}" invocation_info.status_message_color = printing.Colors.WARNING - stats.add_build_stat(fs.Keys.BENCHMARK_STATUS, fs.BenchmarkStatus.FAILED) + stats.save_model_eval_stat(fs.Keys.BENCHMARK_STATUS, fs.BenchmarkStatus.FAILED) _store_traceback(invocation_info) @@ -277,14 +279,14 @@ def explore_invocation( ) invocation_info.status_message_color = printing.Colors.WARNING - stats.add_build_stat(fs.Keys.BENCHMARK_STATUS, fs.BenchmarkStatus.KILLED) + stats.save_model_eval_stat(fs.Keys.BENCHMARK_STATUS, fs.BenchmarkStatus.KILLED) except exp.ArgError as e: # ArgError indicates that some argument to benchmark_model() was # illegal. In that case we want to halt execution so that users can # fix their arguments. - stats.add_build_stat(fs.Keys.BENCHMARK_STATUS, fs.BenchmarkStatus.FAILED) + stats.save_model_eval_stat(fs.Keys.BENCHMARK_STATUS, fs.BenchmarkStatus.FAILED) raise e @@ -292,7 +294,7 @@ def explore_invocation( invocation_info.status_message = f"Error: {e}." invocation_info.status_message_color = printing.Colors.WARNING - stats.add_build_stat(fs.Keys.BENCHMARK_STATUS, fs.BenchmarkStatus.FAILED) + stats.save_model_eval_stat(fs.Keys.BENCHMARK_STATUS, fs.BenchmarkStatus.FAILED) _store_traceback(invocation_info) @@ -302,19 +304,21 @@ def explore_invocation( invocation_info.status_message = f"Unknown turnkey error: {e}" invocation_info.status_message_color = printing.Colors.WARNING - stats.add_build_stat(fs.Keys.BENCHMARK_STATUS, fs.BenchmarkStatus.FAILED) + stats.save_model_eval_stat(fs.Keys.BENCHMARK_STATUS, fs.BenchmarkStatus.FAILED) _store_traceback(invocation_info) else: # If there was no exception then we consider the build to be a success - stats.add_build_stat(fs.Keys.BENCHMARK_STATUS, fs.BenchmarkStatus.SUCCESSFUL) + stats.save_model_eval_stat( + fs.Keys.BENCHMARK_STATUS, fs.BenchmarkStatus.SUCCESSFUL + ) finally: # Ensure that stdout/stderr is not being forwarded before updating status util.stop_logger_forward() system_info = build.get_system_info() - stats.save_stat( + stats.save_model_stat( fs.Keys.SYSTEM_INFO, system_info, ) @@ -326,11 +330,11 @@ def explore_invocation( # ONNX stats that we want to save into the build's turnkey_stats.yaml file # so that they can be easily accessed by the report command later - if fs.Keys.ONNX_FILE in stats.build_stats.keys(): + if fs.Keys.ONNX_FILE in stats.evaluation_stats.keys(): # Just in case the ONNX file was generated on a different machine: # strip the state's cache dir, then prepend the current cache dir final_onnx_file = fs.rebase_cache_dir( - stats.build_stats[fs.Keys.ONNX_FILE], + stats.evaluation_stats[fs.Keys.ONNX_FILE], build_name, tracer_args.cache_dir, ) @@ -339,22 +343,22 @@ def explore_invocation( onnx_model_info = util.populate_onnx_model_info(final_onnx_file) onnx_input_dimensions = util.onnx_input_dimensions(final_onnx_file) - stats.save_stat( + stats.save_model_stat( fs.Keys.ONNX_OPS_COUNTER, onnx_ops_counter, ) - stats.save_stat( + stats.save_model_stat( fs.Keys.ONNX_MODEL_INFO, onnx_model_info, ) - stats.save_stat( + stats.save_model_stat( fs.Keys.ONNX_INPUT_DIMENSIONS, onnx_input_dimensions, ) if perf: for key, value in vars(perf).items(): - stats.add_build_stat( + stats.save_model_eval_stat( key=key, value=value, ) diff --git a/src/turnkeyml/build/export.py b/src/turnkeyml/build/export.py index 4a9f157..93a0a8b 100644 --- a/src/turnkeyml/build/export.py +++ b/src/turnkeyml/build/export.py @@ -189,8 +189,10 @@ def fire(self, state: build.State): if check_model(output_path, success_msg, fail_msg): state.intermediate_results = [output_path] - stats = fs.Stats(state.cache_dir, state.config.build_name, state.stats_id) - stats.add_build_stat( + stats = fs.Stats( + state.cache_dir, state.config.build_name, state.evaluation_id + ) + stats.save_model_eval_stat( fs.Keys.ONNX_FILE, output_path, ) @@ -315,8 +317,10 @@ def fire(self, state: build.State): if check_model(output_path, success_msg, fail_msg): state.intermediate_results = [output_path] - stats = fs.Stats(state.cache_dir, state.config.build_name, state.stats_id) - stats.add_build_stat( + stats = fs.Stats( + state.cache_dir, state.config.build_name, state.evaluation_id + ) + stats.save_model_eval_stat( fs.Keys.ONNX_FILE, output_path, ) @@ -436,8 +440,10 @@ def fire(self, state: build.State): if check_model(output_path, success_msg, fail_msg): state.intermediate_results = [output_path] - stats = fs.Stats(state.cache_dir, state.config.build_name, state.stats_id) - stats.add_build_stat( + stats = fs.Stats( + state.cache_dir, state.config.build_name, state.evaluation_id + ) + stats.save_model_eval_stat( fs.Keys.ONNX_FILE, output_path, ) @@ -500,8 +506,10 @@ def fire(self, state: build.State): if check_model(output_path, success_msg, fail_msg): state.intermediate_results = [output_path] - stats = fs.Stats(state.cache_dir, state.config.build_name, state.stats_id) - stats.add_build_stat( + stats = fs.Stats( + state.cache_dir, state.config.build_name, state.evaluation_id + ) + stats.save_model_eval_stat( fs.Keys.ONNX_FILE, output_path, ) @@ -605,8 +613,10 @@ def fire(self, state: build.State): if check_model(output_path, success_msg, fail_msg): state.intermediate_results = [output_path] - stats = fs.Stats(state.cache_dir, state.config.build_name, state.stats_id) - stats.add_build_stat( + stats = fs.Stats( + state.cache_dir, state.config.build_name, state.evaluation_id + ) + stats.save_model_eval_stat( fs.Keys.ONNX_FILE, output_path, ) @@ -657,8 +667,10 @@ def fire(self, state: build.State): if check_model(output_path, success_msg, fail_msg): state.intermediate_results = [output_path] - stats = fs.Stats(state.cache_dir, state.config.build_name, state.stats_id) - stats.add_build_stat( + stats = fs.Stats( + state.cache_dir, state.config.build_name, state.evaluation_id + ) + stats.save_model_eval_stat( fs.Keys.ONNX_FILE, output_path, ) diff --git a/src/turnkeyml/build/hummingbird.py b/src/turnkeyml/build/hummingbird.py index 21d5f1a..5540482 100644 --- a/src/turnkeyml/build/hummingbird.py +++ b/src/turnkeyml/build/hummingbird.py @@ -217,9 +217,8 @@ def fire(self, state: build.State): np.save(state.original_inputs_file, state.inputs) state.intermediate_results = [output_path] - stats = fs.Stats(state.cache_dir, state.config.build_name) - stats.add_sub_stat( - state.stats_id, + stats = fs.Stats(state.cache_dir, state.config.build_name, state.evaluation_id) + stats.save_model_eval_stat( fs.Keys.ONNX_FILE, output_path, ) diff --git a/src/turnkeyml/build/ignition.py b/src/turnkeyml/build/ignition.py index 18da5aa..00ed81b 100644 --- a/src/turnkeyml/build/ignition.py +++ b/src/turnkeyml/build/ignition.py @@ -251,7 +251,7 @@ def _rebuild_if_needed( def load_or_make_state( config: build.Config, - stats_id: str, + evaluation_id: str, cache_dir: str, rebuild: str, model_type: build.ModelType, @@ -276,7 +276,7 @@ def load_or_make_state( "inputs": inputs, "monitor": monitor, "rebuild": rebuild, - "stats_id": stats_id, + "evaluation_id": evaluation_id, "cache_dir": cache_dir, "config": config, "model_type": model_type, diff --git a/src/turnkeyml/build/stage.py b/src/turnkeyml/build/stage.py index cd2ec53..0267995 100644 --- a/src/turnkeyml/build/stage.py +++ b/src/turnkeyml/build/stage.py @@ -273,8 +273,8 @@ def launch(self, state: build.State) -> build.State: raise exp.Error(msg) # Collect telemetry for the build - stats = fs.Stats(state.cache_dir, state.config.build_name, state.stats_id) - stats.add_build_stat( + stats = fs.Stats(state.cache_dir, state.config.build_name, state.evaluation_id) + stats.save_model_eval_stat( fs.Keys.ALL_BUILD_STAGES, self.get_names(), ) @@ -292,7 +292,7 @@ def launch(self, state: build.State) -> build.State: # Collect telemetry about the stage execution_time = time.time() - start_time - stats.add_build_sub_stat( + stats.save_model_eval_sub_stat( parent_key=fs.Keys.COMPLETED_BUILD_STAGES, key=stage.unique_name, value=execution_time, diff --git a/src/turnkeyml/build_api.py b/src/turnkeyml/build_api.py index 5cf681b..d9f8a0f 100644 --- a/src/turnkeyml/build_api.py +++ b/src/turnkeyml/build_api.py @@ -12,7 +12,7 @@ def build_model( model: build.UnionValidModelInstanceTypes = None, inputs: Optional[Dict[str, Any]] = None, build_name: Optional[str] = None, - stats_id: Optional[str] = "build", + evaluation_id: Optional[str] = "build", cache_dir: str = filesystem.DEFAULT_CACHE_DIR, monitor: Optional[bool] = None, rebuild: Optional[str] = None, @@ -32,7 +32,7 @@ def build_model( build_name: Unique name for the model that will be used to store the ONNX file and build state on disk. Defaults to the name of the file that calls build_model(). - stats_id: Unique name for build statistics that should persist across multiple + evaluation_id: Unique name for evaluation statistics that should persist across multiple builds of the same model. cache_dir: Directory to use as the cache for this build. Output files from this build will be stored at cache_dir/build_name/ @@ -102,7 +102,7 @@ def build_model( # Get the state of the model from the cache if a valid build is available state = ignition.load_or_make_state( config=config, - stats_id=stats_id, + evaluation_id=evaluation_id, cache_dir=parsed_cache_dir, rebuild=rebuild or build.DEFAULT_REBUILD_POLICY, model_type=model_type, diff --git a/src/turnkeyml/cli/report.py b/src/turnkeyml/cli/report.py index 6f778f7..efc5eea 100644 --- a/src/turnkeyml/cli/report.py +++ b/src/turnkeyml/cli/report.py @@ -68,12 +68,12 @@ def summary_spreadsheets(args) -> None: model_stats = yaml.load(stream, Loader=yaml.FullLoader) # create a separate dict for each build - for build in model_stats[fs.Keys.BUILDS].values(): + for build in model_stats[fs.Keys.EVALUATIONS].values(): build_stats = {} # Copy all of the stats for the model that are common across builds for key, value in model_stats.items(): - if key != fs.Keys.BUILDS: + if key != fs.Keys.EVALUATIONS: build_stats[key] = value # Copy the build-specific stats diff --git a/src/turnkeyml/common/build.py b/src/turnkeyml/common/build.py index a224d1a..553d14f 100644 --- a/src/turnkeyml/common/build.py +++ b/src/turnkeyml/common/build.py @@ -222,7 +222,7 @@ class State: monitor: bool = False rebuild: str = "" cache_dir: str = "" - stats_id: str = "" + evaluation_id: str = "" # User-provided args that will not be saved as part of state.yaml model: UnionValidModelInstanceTypes = None @@ -524,7 +524,7 @@ def get_system_info(): # Get OS Version try: info_dict["OS Version"] = platform.platform() - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except info_dict["Error OS Version"] = str(e) if os_type == "Windows": @@ -537,7 +537,7 @@ def get_system_info(): .strip() ) info_dict["Processor"] = proc_info - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except info_dict["Error Processor"] = str(e) # Get OEM System Information @@ -549,7 +549,7 @@ def get_system_info(): .strip() ) info_dict["OEM System"] = oem_info - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except info_dict["Error OEM System"] = str(e) # Get Physical Memory in GB @@ -564,7 +564,7 @@ def get_system_info(): ) mem_info_gb = round(int(mem_info_bytes) / (1024**3), 2) info_dict["Physical Memory"] = f"{mem_info_gb} GB" - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except info_dict["Error Physical Memory"] = str(e) elif os_type == "Linux": @@ -586,7 +586,7 @@ def get_system_info(): .strip() ) info_dict["OEM System"] = oem_info - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except info_dict["Error OEM System (WSL)"] = str(e) else: @@ -602,7 +602,7 @@ def get_system_info(): .replace("\n", " ") ) info_dict["OEM System"] = oem_info - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except info_dict["Error OEM System"] = str(e) # Get CPU Information @@ -612,7 +612,7 @@ def get_system_info(): if "Model name:" in line: info_dict["Processor"] = line.split(":")[1].strip() break - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except info_dict["Error Processor"] = str(e) # Get Memory Information @@ -625,7 +625,7 @@ def get_system_info(): ) mem_info_gb = round(int(mem_info) / 1024, 2) info_dict["Memory Info"] = f"{mem_info_gb} GB" - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except info_dict["Error Memory Info"] = str(e) else: @@ -635,9 +635,10 @@ def get_system_info(): try: installed_packages = pkg_resources.working_set info_dict["Python Packages"] = [ - f"{i.key}=={i.version}" for i in installed_packages # pylint: disable=not-an-iterable + f"{i.key}=={i.version}" + for i in installed_packages # pylint: disable=not-an-iterable ] - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except info_dict["Error Python Packages"] = str(e) return info_dict diff --git a/src/turnkeyml/common/filesystem.py b/src/turnkeyml/common/filesystem.py index 082ecef..41c1204 100644 --- a/src/turnkeyml/common/filesystem.py +++ b/src/turnkeyml/common/filesystem.py @@ -331,8 +331,8 @@ class Keys: DEVICE_TYPE = "device_type" # Name of the model MODEL_NAME = "model_name" - # References the per-build stats section - BUILDS = "builds" + # References the per-evaluation stats section + EVALUATIONS = "evaluations" # Author of the model AUTHOR = "author" # Class type of the model @@ -357,15 +357,15 @@ class BenchmarkStatus: class Stats: - def __init__(self, cache_dir: str, build_name: str, stats_id: str = None): + def __init__(self, cache_dir: str, build_name: str, evaluation_id: str = None): output_dir = build.output_dir(cache_dir, build_name) self.file = os.path.join(output_dir, "turnkey_stats.yaml") - self.stats_id = stats_id + self.evaluation_id = evaluation_id os.makedirs(output_dir, exist_ok=True) if not os.path.exists(self.file): - initial = {Keys.BUILDS: {}} + initial = {Keys.EVALUATIONS: {}} _save_yaml(initial, self.file) @property @@ -389,7 +389,7 @@ def _set_key(self, dict, keys: List["str"], value): self._set_key(dict[keys[0]], keys[1:], value) - def save_stat(self, key: str, value): + def save_model_stat(self, key: str, value): """ Save statistics to an yaml file in the build directory """ @@ -400,36 +400,25 @@ def save_stat(self, key: str, value): _save_yaml(stats_dict, self.file) - def add_sub_stat(self, parent_key: str, key: str, value): - """ - Save nested statistics to an yaml file in the build directory - - stats[parent_key][key] = value - """ - - stats_dict = self.stats - - self._set_key(stats_dict, [parent_key, key], value) - - _save_yaml(stats_dict, self.file) - - def add_build_stat(self, key: str, value): + def save_model_eval_stat(self, key: str, value): stats_dict = self.stats - self._set_key(stats_dict, [Keys.BUILDS, self.stats_id, key], value) + self._set_key(stats_dict, [Keys.EVALUATIONS, self.evaluation_id, key], value) _save_yaml(stats_dict, self.file) - def add_build_sub_stat(self, parent_key: str, key: str, value): + def save_model_eval_sub_stat(self, parent_key: str, key: str, value): stats_dict = self.stats - self._set_key(stats_dict, [Keys.BUILDS, self.stats_id, parent_key, key], value) + self._set_key( + stats_dict, [Keys.EVALUATIONS, self.evaluation_id, parent_key, key], value + ) _save_yaml(stats_dict, self.file) @property - def build_stats(self): - return self.stats[Keys.BUILDS][self.stats_id] + def evaluation_stats(self): + return self.stats[Keys.EVALUATIONS][self.evaluation_id] def print_cache_dir(_=None): diff --git a/src/turnkeyml/model_api.py b/src/turnkeyml/model_api.py index 6b1213b..d7cb155 100644 --- a/src/turnkeyml/model_api.py +++ b/src/turnkeyml/model_api.py @@ -21,7 +21,7 @@ def benchmark_model( inputs: Dict[str, Any], build_name: str, iterations: int = 100, - stats_id: str = "build", + evaluation_id: str = "build", cache_dir: str = filesystem.DEFAULT_CACHE_DIR, device: str = "x86", runtime: Optional[str] = None, @@ -88,7 +88,7 @@ def benchmark_model( build_model( model=model, inputs=inputs, - stats_id=stats_id, + evaluation_id=evaluation_id, build_name=build_name, cache_dir=cache_dir, rebuild=rebuild, @@ -105,7 +105,7 @@ def benchmark_model( rt_args_to_use = rt_args printing.log_info(f"Benchmarking on {device}...") - stats = filesystem.Stats(cache_dir, build_name, stats_id) + stats = filesystem.Stats(cache_dir, build_name, evaluation_id) model_handle = runtime_info["RuntimeClass"]( cache_dir=cache_dir, build_name=build_name, diff --git a/src/turnkeyml/run/tensorrt/runtime.py b/src/turnkeyml/run/tensorrt/runtime.py index b6ef9a9..df1270c 100644 --- a/src/turnkeyml/run/tensorrt/runtime.py +++ b/src/turnkeyml/run/tensorrt/runtime.py @@ -13,6 +13,7 @@ average_power_and_utilization, ) + def _get_nvidia_driver_version(): try: output = subprocess.check_output(["nvidia-smi"], text=True) @@ -23,10 +24,12 @@ def _get_nvidia_driver_version(): # Extract and return the driver version return line.split(":")[1].strip().split()[0] - except Exception as e: # pylint: disable=broad-except + except Exception as e: # pylint: disable=broad-except return str(e) return "Driver not found" + + class TensorRT(BaseRT): def __init__( self, @@ -88,7 +91,7 @@ def _execute( # Add the GPU driver version to the stats file before execution gpu_driver_version = _get_nvidia_driver_version() - self.stats.add_build_stat("gpu_driver_version", gpu_driver_version) + self.stats.save_model_eval_stat("gpu_driver_version", gpu_driver_version) power_thread.start() run( diff --git a/test/cli.py b/test/cli.py index e6ce708..bf7e916 100644 --- a/test/cli.py +++ b/test/cli.py @@ -149,7 +149,7 @@ def assert_success_of_builds( stats = filesystem.Stats( build_state.cache_dir, build_state.config.build_name, - build_state.stats_id, + build_state.evaluation_id, ) assert build_state.build_status == build.Status.SUCCESSFUL_BUILD script_build_found = True diff --git a/test/helpers/common.py b/test/helpers/common.py index d5b402e..42904db 100644 --- a/test/helpers/common.py +++ b/test/helpers/common.py @@ -1,4 +1,3 @@ - import os import shutil from typing import Dict @@ -101,13 +100,13 @@ def forward(self, x): } -def create_test_dir(key:str, test_scripts: Dict = None): +def create_test_dir(key: str, test_scripts: Dict = None): # Define paths to be used base_dir = os.path.dirname(os.path.abspath(__file__)) cache_dir = os.path.join(base_dir, "generated", f"{key}_cache_dir") corpus_dir = os.path.join(base_dir, "generated", f"test_corpus") - - # Delete folders if they exist and + + # Delete folders if they exist and if os.path.isdir(cache_dir): shutil.rmtree(cache_dir) if os.path.isdir(corpus_dir): @@ -124,9 +123,11 @@ def create_test_dir(key:str, test_scripts: Dict = None): return cache_dir, corpus_dir + def strip_dot_py(test_script_file: str) -> str: return test_script_file.split(".")[0] + def get_stats_and_state( test_script: str, cache_dir: str, @@ -141,8 +142,8 @@ def get_stats_and_state( stats = filesystem.Stats( build_state.cache_dir, build_state.config.build_name, - build_state.stats_id, + build_state.evaluation_id, ) return stats.build_stats, build_state - raise Exception(f"Stats not found for {test_script}") \ No newline at end of file + raise Exception(f"Stats not found for {test_script}") From 47ade1ff2e256fa7b1067d44a918b9165e17211f Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Mon, 4 Dec 2023 16:33:53 -0500 Subject: [PATCH 2/4] One last find and replace Signed-off-by: Jeremy Fowers --- src/turnkeyml/analyze/status.py | 2 +- src/turnkeyml/cli/report.py | 34 ++++++++++++++++----------------- test/cli.py | 6 +++--- test/helpers/common.py | 2 +- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/turnkeyml/analyze/status.py b/src/turnkeyml/analyze/status.py index c94542e..fae91f9 100644 --- a/src/turnkeyml/analyze/status.py +++ b/src/turnkeyml/analyze/status.py @@ -198,7 +198,7 @@ def print_invocation( if unique_invocation.stats_keys is not None: for key in unique_invocation.stats_keys: nice_key = _pretty_print_key(key) - value = unique_invocation.stats.build_stats[key] + value = unique_invocation.stats.evaluation_stats[key] printing.logn(f"{ident}\t\t\t{nice_key}:\t{value}") print() else: diff --git a/src/turnkeyml/cli/report.py b/src/turnkeyml/cli/report.py index efc5eea..ec49a9f 100644 --- a/src/turnkeyml/cli/report.py +++ b/src/turnkeyml/cli/report.py @@ -47,7 +47,7 @@ def summary_spreadsheets(args) -> None: Path(report_dir).mkdir(parents=True, exist_ok=True) report: List[Dict] = [] - all_build_stats = [] + all_evaluation_stats = [] # Add results from all user-provided cache folders for cache_dir in cache_dirs: @@ -69,12 +69,12 @@ def summary_spreadsheets(args) -> None: # create a separate dict for each build for build in model_stats[fs.Keys.EVALUATIONS].values(): - build_stats = {} + evaluation_stats = {} # Copy all of the stats for the model that are common across builds for key, value in model_stats.items(): if key != fs.Keys.EVALUATIONS: - build_stats[key] = value + evaluation_stats[key] = value # Copy the build-specific stats for key, value in build.items(): @@ -82,7 +82,7 @@ def summary_spreadsheets(args) -> None: # to make analysis easier if key == fs.Keys.COMPLETED_BUILD_STAGES: for subkey, subvalue in value.items(): - build_stats[subkey] = subvalue + evaluation_stats[subkey] = subvalue # If a build is still marked as "running" at reporting time, it # must have been killed by a time out, out-of-memory (OOM), or some @@ -93,30 +93,30 @@ def summary_spreadsheets(args) -> None: ): value = fs.BenchmarkStatus.KILLED - build_stats[key] = value + evaluation_stats[key] = value - all_build_stats.append(build_stats) + all_evaluation_stats.append(evaluation_stats) except yaml.scanner.ScannerError: continue # Scan the build stats to determine the set of columns for the CSV file. # The CSV will have one column for every key in any build stats dict. column_headers = [] - for build_stats in all_build_stats: + for evaluation_stats in all_evaluation_stats: # Add any key that isn't already in column_headers - for header in build_stats.keys(): + for header in evaluation_stats.keys(): if header not in column_headers: column_headers.append(header) # Add each build to the report - for build_stats in all_build_stats: + for evaluation_stats in all_evaluation_stats: # Start with a dictionary where all of the values are "-". If a build # has a value for each key we will fill it in, and otherwise the "-" # will indicate that no value was available result = {k: "-" for k in column_headers} for key in column_headers: - result[key] = _good_get(build_stats, key) + result[key] = _good_get(evaluation_stats, key) report.append(result) @@ -133,13 +133,13 @@ def summary_spreadsheets(args) -> None: # Save the unique errors and counts to a file errors = [] - for build_stats in all_build_stats: + for evaluation_stats in all_evaluation_stats: if ( - "compilation_error" in build_stats.keys() - and "compilation_error_id" in build_stats.keys() + "compilation_error" in evaluation_stats.keys() + and "compilation_error_id" in evaluation_stats.keys() ): - error = build_stats["compilation_error"] - id = build_stats["compilation_error_id"] + error = evaluation_stats["compilation_error"] + id = evaluation_stats["compilation_error_id"] if id != "": unique_error = True for reported_error in errors: @@ -148,13 +148,13 @@ def summary_spreadsheets(args) -> None: reported_error["count"] = reported_error["count"] + 1 reported_error["models_impacted"] = reported_error[ "models_impacted" - ] + [build_stats["model_name"]] + ] + [evaluation_stats["model_name"]] if unique_error: reported_error = { "id": id, "count": 1, - "models_impacted": [build_stats["model_name"]], + "models_impacted": [evaluation_stats["model_name"]], "example": error, } errors.append(reported_error) diff --git a/test/cli.py b/test/cli.py index bf7e916..5f128ea 100644 --- a/test/cli.py +++ b/test/cli.py @@ -161,11 +161,11 @@ def assert_success_of_builds( ), f"{build_state.info.__dict__[info_property[0]]} == {info_property[1]}" if check_perf: - assert stats.build_stats["mean_latency"] > 0 - assert stats.build_stats["throughput"] > 0 + assert stats.evaluation_stats["mean_latency"] > 0 + assert stats.evaluation_stats["throughput"] > 0 if check_iteration_count: - iterations = stats.build_stats["iterations"] + iterations = stats.evaluation_stats["iterations"] assert iterations == check_iteration_count if check_opset: diff --git a/test/helpers/common.py b/test/helpers/common.py index 42904db..4a86782 100644 --- a/test/helpers/common.py +++ b/test/helpers/common.py @@ -144,6 +144,6 @@ def get_stats_and_state( build_state.config.build_name, build_state.evaluation_id, ) - return stats.build_stats, build_state + return stats.evaluation_stats, build_state raise Exception(f"Stats not found for {test_script}") From afd81b9ab886c50f3ac12e55219237f5ded7cc88 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Mon, 4 Dec 2023 17:00:34 -0500 Subject: [PATCH 3/4] I forgot to hit save Signed-off-by: Jeremy Fowers --- src/turnkeyml/build/export.py | 57 ----------------------------------- 1 file changed, 57 deletions(-) diff --git a/src/turnkeyml/build/export.py b/src/turnkeyml/build/export.py index 4b23b15..6d32f05 100644 --- a/src/turnkeyml/build/export.py +++ b/src/turnkeyml/build/export.py @@ -622,63 +622,6 @@ def fire(self, state: build.State): return state -<<<<<<< HEAD -class QuantizeONNXModel(stage.Stage): - """ - Stage that takes an ONNX model and a dataset of quantization samples as inputs, - and performs static post-training quantization to the model to int8 precision. - - Expected inputs: - - state.model is a path to the ONNX model - - state.quantization_dataset is a dataset that is used for static quantization - - Outputs: - - A *_quantized.onnx file => the quantized onnx model. - """ - - def __init__(self): - super().__init__( - unique_name="quantize_onnx", - monitor_message="Quantizing ONNX model", - ) - - def fire(self, state: build.State): - input_path = state.intermediate_results[0] - output_path = quantized_onnx_file(state) - - quant_helpers.quantize( - input_file=input_path, - data=state.quantization_samples, - output_file=output_path, - ) - - # Check that the converted model is still valid - success_msg = "\tSuccess quantizing ONNX model to int8" - fail_msg = "\tFailed quantizing ONNX model to int8" - - if check_model(output_path, success_msg, fail_msg): - state.intermediate_results = [output_path] - - stats = fs.Stats( - state.cache_dir, state.config.build_name, state.evaluation_id - ) - stats.save_model_eval_stat( - fs.Keys.ONNX_FILE, - output_path, - ) - else: - msg = f""" - Attempted to use {state.quantization_dataset} to statically quantize - model to int8 datatype, however this operation was not successful. - More information may be available in the log file at **{self.logfile_path}** - """ - raise exp.StageError(msg) - - return state - - -======= ->>>>>>> canary class SuccessStage(stage.Stage): """ Stage that sets state.build_status = build.Status.SUCCESSFUL_BUILD, From 99d2ae1ff67ad4b0ee5b287ee698cf0b1c23458b Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Tue, 5 Dec 2023 09:24:03 -0500 Subject: [PATCH 4/4] Fix rename bugs Signed-off-by: Jeremy Fowers --- src/turnkeyml/analyze/script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/turnkeyml/analyze/script.py b/src/turnkeyml/analyze/script.py index 2d2bb0c..cebf799 100644 --- a/src/turnkeyml/analyze/script.py +++ b/src/turnkeyml/analyze/script.py @@ -178,14 +178,14 @@ def explore_invocation( model_info.params, ) if model_info.model_type != build.ModelType.ONNX_FILE: - stats.save_stat(fs.Keys.CLASS, type(model_info.model).__name__) + stats.save_model_stat(fs.Keys.CLASS, type(model_info.model).__name__) if fs.Keys.AUTHOR in tracer_args.labels: stats.save_model_stat(fs.Keys.AUTHOR, tracer_args.labels[fs.Keys.AUTHOR][0]) if fs.Keys.TASK in tracer_args.labels: stats.save_model_stat(fs.Keys.TASK, tracer_args.labels[fs.Keys.TASK][0]) # Save all of the lables in one place - stats.save_stat(fs.Keys.LABELS, tracer_args.labels) + stats.save_model_stat(fs.Keys.LABELS, tracer_args.labels) # If the input script is a built-in TurnkeyML model, make a note of # which one