From ba89e19b6af809d53aa2b61462c11799cd27c58b Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Fri, 23 Aug 2024 15:14:37 -0400 Subject: [PATCH 01/22] Split testing into benchmark vs non-benchmark --- .github/workflows/test_turnkey.yml | 1 + test/benchmark.py | 695 +++++++++++++++++++++++++++++ test/cli.py | 360 ++------------- test/unit.py | 13 + 4 files changed, 736 insertions(+), 333 deletions(-) create mode 100644 test/benchmark.py diff --git a/.github/workflows/test_turnkey.yml b/.github/workflows/test_turnkey.yml index f7ad30d..173bd38 100644 --- a/.github/workflows/test_turnkey.yml +++ b/.github/workflows/test_turnkey.yml @@ -70,6 +70,7 @@ jobs: cd test/ python cli.py python analysis.py + python benchmark.py - name: Test example plugins shell: bash -el {0} run: | diff --git a/test/benchmark.py b/test/benchmark.py new file mode 100644 index 0000000..77e64b8 --- /dev/null +++ b/test/benchmark.py @@ -0,0 +1,695 @@ +""" +Tests focused on the benchmkarking functionality of turnkey CLI +""" + +import os +import shutil +import glob +import csv +from typing import List, Tuple, Any, Union, Optional +import unittest +from unittest.mock import patch +import sys +import io +from contextlib import redirect_stdout +import onnx +import platform +import torch +from turnkeyml.cli.cli import main as turnkeycli +import turnkeyml.tools.report as report +import turnkeyml.common.filesystem as fs +import turnkeyml.common.build as build +import turnkeyml.common.exceptions as exceptions +import turnkeyml.common.onnx_helpers as onnx_helpers +import turnkeyml.common.test_helpers as common +from turnkeyml.state import load_state + + +def bash(cmd: str) -> List[str]: + """ + Emulate behavior of bash terminal when listing files + """ + return glob.glob(cmd) + + +def flatten(lst: List[Union[str, List[str]]]) -> List[str]: + """ + Flatten List[Union[str, List[str]]] into a List[str] + """ + flattened = [] + for element in lst: + if isinstance(element, list): + flattened.extend(element) + else: + flattened.append(element) + return flattened + + +def assert_success_of_builds( + test_script_files: List[str], + cache_dir: str, + info_property: Tuple[str, Any] = None, + check_perf: bool = False, + check_opset: Optional[int] = None, + check_iteration_count: Optional[int] = None, + check_onnx_file_count: Optional[int] = None, +) -> int: + # Figure out the build name by surveying the build cache + # for a build that includes test_script_name in the name + builds = fs.get_all(cache_dir) + builds_found = 0 + + for test_script in test_script_files: + test_script_name = common.strip_dot_py(test_script) + script_build_found = False + + for build_state_file in builds: + if test_script_name in build_state_file: + build_state = load_state(state_path=build_state_file) + stats = fs.Stats( + build_state.cache_dir, + build_state.build_name, + ) + assert build_state.build_status == build.FunctionStatus.SUCCESSFUL + script_build_found = True + builds_found += 1 + + if info_property is not None: + assert ( + build_state.info.__dict__[info_property[0]] == info_property[1] + ), f"{build_state.info.__dict__[info_property[0]]} == {info_property[1]}" + + if check_perf: + assert stats.stats["mean_latency"] > 0 + assert stats.stats["throughput"] > 0 + + if check_iteration_count: + iterations = stats.stats["iterations"] + assert iterations == check_iteration_count + + if check_opset: + onnx_model = onnx.load(build_state.results) + model_opset = getattr(onnx_model.opset_import[0], "version", None) + assert model_opset == check_opset + + if check_onnx_file_count: + onnx_dir = onnx_helpers.onnx_dir(build_state) + assert len(os.listdir(onnx_dir)) == check_onnx_file_count + + assert script_build_found + + # Returns the total number of builds found + return builds_found + + +class SmallPytorchModel(torch.nn.Module): + def __init__(self): + super(SmallPytorchModel, self).__init__() + self.fc = torch.nn.Linear(10, 5) + + def forward(self, x): + output = self.fc(x) + return output + + +# Define pytorch model and inputs +pytorch_model = SmallPytorchModel() +inputs = {"x": torch.rand(10)} +inputs_2 = {"x": torch.rand(5)} +input_tensor = torch.rand(10) + + +class Testing(unittest.TestCase): + def setUp(self) -> None: + fs.rmdir(cache_dir) + fs.rmdir(new_cache_dir) + + return super().setUp() + + # TODO: Investigate why this test is failing only on Windows CI failing + @unittest.skipIf(platform.system() == "Windows", "Windows CI only failure") + def test_001_cli_benchmark(self): + # Test the first model in the corpus + test_script = list(common.test_scripts_dot_py.keys())[0] + + testargs = [ + "turnkey", + "-i", + os.path.join(corpus_dir, test_script), + "--cache-dir", + cache_dir, + "discover", + "export-pytorch", + "optimize-ort", + "benchmark", + ] + with patch.object(sys, "argv", testargs): + turnkeycli() + + assert_success_of_builds([test_script], cache_dir, None, check_perf=True) + + def test_002_runtimes(self): + # Attempt to benchmark using an invalid runtime + with self.assertRaises(exceptions.ArgError): + testargs = [ + "turnkey", + "-i", + bash(f"{corpus_dir}/linear.py"), + "--cache-dir", + cache_dir, + "discover", + "export-pytorch", + "optimize-ort", + "benchmark", + "--device", + "x86", + "--runtime", + "trt", + ] + with patch.object(sys, "argv", flatten(testargs)): + turnkeycli() + + # Benchmark with Pytorch + testargs = [ + "turnkey", + "-i", + bash(f"{corpus_dir}/linear.py"), + "--cache-dir", + cache_dir, + "discover", + "benchmark", + "--device", + "x86", + "--runtime", + "torch-eager", + ] + with patch.object(sys, "argv", flatten(testargs)): + turnkeycli() + + # Benchmark with Onnx Runtime + testargs = [ + "turnkey", + "-i", + bash(f"{corpus_dir}/linear.py"), + "--cache-dir", + cache_dir, + "discover", + "export-pytorch", + "optimize-ort", + "benchmark", + "--device", + "x86", + "--runtime", + "ort", + ] + with patch.object(sys, "argv", flatten(testargs)): + turnkeycli() + + def test_003_cli_iteration_count(self): + # Test the first model in the corpus + test_script = list(common.test_scripts_dot_py.keys())[0] + + test_iterations = 123 + testargs = [ + "turnkey", + "-i", + os.path.join(corpus_dir, test_script), + "--cache-dir", + cache_dir, + "discover", + "export-pytorch", + "optimize-ort", + "benchmark", + "--iterations", + str(test_iterations), + ] + with patch.object(sys, "argv", testargs): + turnkeycli() + + assert_success_of_builds( + [test_script], + cache_dir, + None, + check_perf=True, + check_iteration_count=test_iterations, + ) + + def test_004_cli_process_isolation(self): + # Test the first model in the corpus + test_script = list(common.test_scripts_dot_py.keys())[0] + + with redirect_stdout(io.StringIO()) as f: + testargs = [ + "turnkey", + "-i", + os.path.join(corpus_dir, test_script), + "--cache-dir", + cache_dir, + "--process-isolation", + "discover", + "export-pytorch", + "--opset", + "17", + "optimize-ort", + "benchmark", + ] + with patch.object(sys, "argv", testargs): + turnkeycli() + + assert_success_of_builds([test_script], cache_dir, None, check_perf=True) + + def test_005_cli_export_only(self): + # Test the first model in the corpus + test_script = list(common.test_scripts_dot_py.keys())[0] + + testargs = [ + "turnkey", + "-i", + os.path.join(corpus_dir, test_script), + "--cache-dir", + cache_dir, + "discover", + "export-pytorch", + "benchmark", + ] + with patch.object(sys, "argv", testargs): + turnkeycli() + + assert_success_of_builds([test_script], cache_dir, check_onnx_file_count=1) + + def test_006_cli_onnx_model(self): + """ + Manually export an ONNX file, then feed it into the CLI + """ + build_name = "receive_onnx" + onnx_file = os.path.join(corpus_dir, f"{build_name}.onnx") + + # Create ONNX file + torch.onnx.export( + pytorch_model, + input_tensor, + onnx_file, + opset_version=build.DEFAULT_ONNX_OPSET, + input_names=["input"], + output_names=["output"], + ) + + testargs = [ + "turnkey", + "-i", + onnx_file, + "--cache-dir", + cache_dir, + "load-onnx", + "benchmark", + ] + with patch.object(sys, "argv", testargs): + turnkeycli() + + assert_success_of_builds([build_name], cache_dir) + + def test_007_cli_onnx_model_opset(self): + """ + Manually export an ONNX file with a non-defualt opset, then feed it into the CLI + """ + build_name = "receive_onnx_opset" + onnx_file = os.path.join(corpus_dir, f"{build_name}.onnx") + user_opset = build.MINIMUM_ONNX_OPSET + + # Make sure we are using an non-default ONNX opset + assert user_opset != build.DEFAULT_ONNX_OPSET + + # Create ONNX file + torch.onnx.export( + pytorch_model, + input_tensor, + onnx_file, + opset_version=user_opset, + input_names=["input"], + output_names=["output"], + ) + + testargs = [ + "turnkey", + "-i", + onnx_file, + "--cache-dir", + cache_dir, + "load-onnx", + "benchmark", + ] + with patch.object(sys, "argv", testargs): + turnkeycli() + + assert_success_of_builds([build_name], cache_dir) + + def test_008_cli_timeout(self): + """ + Make sure that the --timeout option and its associated reporting features work. + + timeout.py is designed to take a long time to export, which gives us the + opportunity to kill it with a timeout. + + NOTE: this test can become flakey if: + - exporting timeout.py takes less time than the timeout + - the timeout kills the process before it has a chance to create a stats.yaml file + """ + + testargs = [ + "turnkey", + "-i", + os.path.join(extras_dir, "timeout.py"), + "--cache-dir", + cache_dir, + "--process-isolation", + "--timeout", + "10", + "discover", + "export-pytorch", + "optimize-ort", + "benchmark", + ] + with patch.object(sys, "argv", flatten(testargs)): + turnkeycli() + + testargs = [ + "turnkey", + "report", + "--input-caches", + cache_dir, + ] + with patch.object(sys, "argv", testargs): + turnkeycli() + + # Read generated CSV file and make sure the build was killed by the timeout + summary_csv_path = report.get_report_name() + with open(summary_csv_path, "r", encoding="utf8") as summary_csv: + summary = list(csv.DictReader(summary_csv)) + + # Check the summary for "killed", but also accept the edge case that + # the build timed out before the stats.yaml was created + try: + timeout_summary = summary[0] + + assert timeout_summary["build_status"] == "timeout", timeout_summary[ + "build_status" + ] + except IndexError: + # Edge case where the CSV is empty because the build timed out before + # the stats.yaml was created, which in turn means the CSV is empty + pass + except KeyError: + # Edge case where the CSV only contains a key for "error_log" + assert "timeout" in timeout_summary["error_log"] + + def test_009_cli_report(self): + # NOTE: this is not a unit test, it relies on other command + # If this test is failing, make sure the following tests are passing: + # - test_cli_corpus + + test_scripts = common.test_scripts_dot_py.keys() + + # Benchmark the test corpus so we have builds to report + testargs = [ + "turnkey", + "-i", + bash(f"{corpus_dir}/*.py"), + "--cache-dir", + cache_dir, + "discover", + "export-pytorch", + "optimize-ort", + "benchmark", + ] + with patch.object(sys, "argv", flatten(testargs)): + turnkeycli() + + testargs = [ + "turnkey", + "report", + "--input-caches", + cache_dir, + ] + with patch.object(sys, "argv", testargs): + turnkeycli() + + # Read generated CSV file + summary_csv_path = report.get_report_name() + with open(summary_csv_path, "r", encoding="utf8") as summary_csv: + summary = list(csv.DictReader(summary_csv)) + + # Check if csv file contains all expected rows and columns + expected_cols = [ + "model_name", + "author", + "class", + "parameters", + "hash", + "runtime", + "device_type", + "device", + "mean_latency", + "throughput", + "selected_sequence_of_tools", + ] + linear_summary = summary[1] + assert len(summary) == len(test_scripts) + for elem in expected_cols: + assert ( + elem in linear_summary + ), f"Couldn't find expected key {elem} in results spreadsheet" + + # Check whether all rows we expect to be populated are actually populated + assert ( + linear_summary["model_name"] == "linear2" + ), f"Wrong model name found {linear_summary['model_name']}" + assert ( + linear_summary["author"] == "turnkey" + ), f"Wrong author name found {linear_summary['author']}" + assert ( + linear_summary["class"] == "TwoLayerModel" + ), f"Wrong class found {linear_summary['model_class']}" + assert ( + linear_summary["hash"] == "80b93950" + ), f"Wrong hash found {linear_summary['hash']}" + assert ( + linear_summary["runtime"] == "ort" + ), f"Wrong runtime found {linear_summary['runtime']}" + assert ( + linear_summary["device_type"] == "x86" + ), f"Wrong device type found {linear_summary['device_type']}" + assert ( + float(linear_summary["mean_latency"]) > 0 + ), f"latency must be >0, got {linear_summary['x86_latency']}" + assert ( + float(linear_summary["throughput"]) > 100 + ), f"throughput must be >100, got {linear_summary['throughput']}" + + # Make sure the report.get_dict() API works + result_dict = report.get_dict( + summary_csv_path, + [ + "selected_sequence_of_tools", + "tool_duration:discover", + "tool_duration:export-pytorch", + "tool_duration:optimize-ort", + "tool_status:discover", + "tool_status:export-pytorch", + "tool_status:optimize-ort", + ], + ) + for result in result_dict.values(): + # All of the models should have exported to ONNX and optimized the ONNX model + for tool in ["export-pytorch", "optimize-ort"]: + assert tool in result["selected_sequence_of_tools"] + duration = result[f"tool_duration:{tool}"] + status = result[f"tool_status:{tool}"] + assert ( + status == "successful" + ), f"Unexpected status {status} for tool '{tool}'" + try: + assert ( + float(duration) > 0 + ), f"Tool {tool} has invalid duration '{duration}'" + except ValueError: + # Catch the case where the value is not numeric + assert False, f"Tool {tool} has invalid duration {duration}" + + def test_010_cli_cache_benchmark(self): + + test_scripts = common.test_scripts_dot_py.keys() + + # Build the test corpus so we have builds to benchmark + testargs = [ + "turnkey", + "-i", + bash(f"{corpus_dir}/*.py"), + "--cache-dir", + cache_dir, + "discover", + "export-pytorch", + "optimize-ort", + ] + with patch.object(sys, "argv", flatten(testargs)): + turnkeycli() + + # Benchmark the single model from cache directory + selected_build = fs.get_available_builds(cache_dir)[-1] + state_file_path = os.path.join( + cache_dir, selected_build, f"{selected_build}_state.yaml" + ) + + testargs = [ + "turnkey", + "--cache-dir", + cache_dir, + "-i", + state_file_path, + "load-build", + "benchmark", + ] + with patch.object(sys, "argv", flatten(testargs)): + turnkeycli() + + # Make sure the benchmark happened + test_script = selected_build + ".py" + assert_success_of_builds([test_script], cache_dir, check_perf=True) + + # Benchmark the cache directory + testargs = [ + "turnkey", + "--cache-dir", + cache_dir, + "-i", + os.path.join(cache_dir, "*", "*_state.yaml"), + "load-build", + "benchmark", + ] + with patch.object(sys, "argv", flatten(testargs)): + turnkeycli() + + # Make sure the benchmarks happened + assert_success_of_builds(test_scripts, cache_dir, check_perf=True) + + def test_011_cli_cache_move(self): + + test_script = list(common.test_scripts_dot_py.keys())[0] + + # Build a model into the default cache location + testargs = [ + "turnkey", + "-i", + os.path.join(corpus_dir, test_script), + "--cache-dir", + cache_dir, + "discover", + "export-pytorch", + ] + with patch.object(sys, "argv", flatten(testargs)): + turnkeycli() + + # Move the cache to a new location + shutil.move(cache_dir, new_cache_dir) + + # Get the build state file in its new location + selected_build = fs.get_available_builds(new_cache_dir)[-1] + state_file_path = os.path.join( + new_cache_dir, selected_build, f"{selected_build}_state.yaml" + ) + + # Benchmark the cached build in its new location + testargs = [ + "turnkey", + "-i", + state_file_path, + "load-build", + "benchmark", + ] + with patch.object(sys, "argv", flatten(testargs)): + turnkeycli() + + # Make sure the benchmark happened + test_script = selected_build + ".py" + assert_success_of_builds([test_script], new_cache_dir, check_perf=True) + + +if __name__ == "__main__": + # Create a cache directory a directory with test models + cache_dir, corpus_dir = common.create_test_dir("benchmark") + new_cache_dir = f"{cache_dir}2" + + extras_dot_py = { + "compiled.py": """ +# labels: name::linear author::selftest test_group::selftest task::test +import torch + +torch.manual_seed(0) + + +class LinearTestModel(torch.nn.Module): + def __init__(self, input_features, output_features): + super(LinearTestModel, self).__init__() + self.fc = torch.nn.Linear(input_features, output_features) + + def forward(self, x): + output = self.fc(x) + return output + + +input_features = 10 +output_features = 10 + +# Compiled model +model = LinearTestModel(input_features, output_features) +model = torch.compile(model) +inputs = {"x": torch.rand(input_features)} +model(**inputs) + +# Non-compiled model +model2 = LinearTestModel(input_features * 2, output_features) +inputs2 = {"x": torch.rand(input_features * 2)} +model2(**inputs2) + """, + "selected_models.txt": f""" + {os.path.join(corpus_dir,"linear.py")} + {os.path.join(corpus_dir,"linear2.py")} + """, + "timeout.py": """ +# labels: name::timeout author::turnkey license::mit test_group::a task::test +import torch + +torch.manual_seed(0) + + +class LinearTestModel(torch.nn.Module): + def __init__(self, input_features, output_features): + super(LinearTestModel, self).__init__() + self.fc = torch.nn.Linear(input_features, output_features) + + def forward(self, x): + output = self.fc(x) + return output + + +input_features = 500000 +output_features = 1000 + +# Model and input configurations +model = LinearTestModel(input_features, output_features) +inputs = {"x": torch.rand(input_features)} + +output = model(**inputs) + + """, + } + + extras_dir = os.path.join(corpus_dir, "extras") + os.makedirs(extras_dir, exist_ok=True) + + for key, value in extras_dot_py.items(): + file_path = os.path.join(extras_dir, key) + + with open(file_path, "w", encoding="utf") as f: + f.write(value) + + unittest.main() diff --git a/test/cli.py b/test/cli.py index 0f9345d..ff92351 100644 --- a/test/cli.py +++ b/test/cli.py @@ -11,7 +11,6 @@ from unittest.mock import patch import sys import io -from pathlib import Path from contextlib import redirect_stdout import yaml import onnx @@ -23,7 +22,6 @@ import turnkeyml.common.build as build import turnkeyml.common.exceptions as exceptions import turnkeyml.common.onnx_helpers as onnx_helpers -from turnkeyml.cli.parser_helpers import decode_args, encode_args import turnkeyml.common.test_helpers as common from turnkeyml.state import load_state @@ -408,191 +406,9 @@ def test_008_cli_turnkey_args(self): assert_success_of_builds([test_script], cache_dir) - # TODO: Investigate why this test is failing only on Windows CI failing - @unittest.skipIf(platform.system() == "Windows", "Windows CI only failure") - def test_009_cli_benchmark(self): - # Test the first model in the corpus - test_script = list(common.test_scripts_dot_py.keys())[0] - - testargs = [ - "turnkey", - "-i", - os.path.join(corpus_dir, test_script), - "--cache-dir", - cache_dir, - "discover", - "export-pytorch", - "optimize-ort", - "benchmark", - ] - with patch.object(sys, "argv", testargs): - turnkeycli() - - assert_success_of_builds([test_script], cache_dir, None, check_perf=True) - - # TODO: Investigate why this test is non-deterministically failing - @unittest.skip("Flaky test") - def test_010_cli_labels(self): - # Only build models labels with test_group::a - testargs = [ - "turnkey", - "-i", - bash(f"{corpus_dir}/*.py"), - "--labels", - "test_group::a", - "--build-only", - "--cache-dir", - cache_dir, - "discover", - "export-pytorch", - "optimize-ort", - "benchmark", - ] - with patch.object(sys, "argv", flatten(testargs)): - turnkeycli() - - state_files = [Path(p).stem for p in fs.get_all(cache_dir)] - assert state_files == ["linear_d5b1df11_state"] - - # Delete the builds - testargs = [ - "turnkey", - "--cache-dir", - cache_dir, - "cache", - "--delete", - "--all", - ] - with patch.object(sys, "argv", testargs): - turnkeycli() - - assert fs.get_all(cache_dir) == [] - - # Only build models labels with test_group::a and test_group::b - testargs = [ - "turnkey", - "-i", - bash(f"{corpus_dir}/*.py"), - "--labels", - "test_group::a,b", - "--cache-dir", - cache_dir, - "discover", - "export-pytorch", - "optimize-ort", - ] - with patch.object(sys, "argv", flatten(testargs)): - turnkeycli() - - state_files = [Path(p).stem for p in fs.get_all(cache_dir)] - assert state_files == ["linear_d5b1df11_state", "linear2_80b93950_state"] - - @unittest.skip("Needs re-implementation") - def test_011_report_on_failed_build(self): - testargs = [ - "turnkey", - bash(f"{corpus_dir}/linear.py"), - "--cache-dir", - cache_dir, - "discover", - "export-pytorch", - "optimize-ort", - "benchmark", - "--device", - "reimplement_me", - ] - with patch.object(sys, "argv", flatten(testargs)): - turnkeycli() - - # Ensure test failed - build_state = load_state(state_path=fs.get_all(cache_dir)[0]) - assert build_state.build_status != build.FunctionStatus.SUCCESSFUL - - # Generate report - testargs = [ - "turnkey", - "report", - "--input-caches", - cache_dir, - ] - with patch.object(sys, "argv", testargs): - turnkeycli() - - # Read generated CSV file - summary_csv_path = report.get_report_name() - summary = None - with open(summary_csv_path, "r", encoding="utf8") as summary_csv: - summary = list(csv.DictReader(summary_csv)) - - # Ensure parameters and hash have been saved despite crash - assert ( - len(summary) == 1 - ), "Report must contain only one row, but got {len(summary)}" - assert ( - summary[0]["params"] == "110" - ), "Wrong number of parameters found in report" - assert summary[0]["hash"] == "d5b1df11", "Wrong hash found in report" - - def test_012_runtimes(self): - # Attempt to benchmark using an invalid runtime - with self.assertRaises(exceptions.ArgError): - testargs = [ - "turnkey", - "-i", - bash(f"{corpus_dir}/linear.py"), - "--cache-dir", - cache_dir, - "discover", - "export-pytorch", - "optimize-ort", - "benchmark", - "--device", - "x86", - "--runtime", - "trt", - ] - with patch.object(sys, "argv", flatten(testargs)): - turnkeycli() - - # Benchmark with Pytorch - testargs = [ - "turnkey", - "-i", - bash(f"{corpus_dir}/linear.py"), - "--cache-dir", - cache_dir, - "discover", - "benchmark", - "--device", - "x86", - "--runtime", - "torch-eager", - ] - with patch.object(sys, "argv", flatten(testargs)): - turnkeycli() - - # Benchmark with Onnx Runtime - testargs = [ - "turnkey", - "-i", - bash(f"{corpus_dir}/linear.py"), - "--cache-dir", - cache_dir, - "discover", - "export-pytorch", - "optimize-ort", - "benchmark", - "--device", - "x86", - "--runtime", - "ort", - ] - with patch.object(sys, "argv", flatten(testargs)): - turnkeycli() - # TODO: Investigate why this test is only failing on Windows CI @unittest.skipIf(platform.system() == "Windows", "Windows CI only failure") - def test_013_cli_onnx_opset(self): + def test_08_cli_onnx_opset(self): # Test the first model in the corpus test_script = list(common.test_scripts_dot_py.keys())[0] @@ -610,45 +426,15 @@ def test_013_cli_onnx_opset(self): "--opset", str(user_opset), "optimize-ort", - "benchmark", ] with patch.object(sys, "argv", testargs): turnkeycli() assert_success_of_builds( - [test_script], cache_dir, None, check_perf=True, check_opset=user_opset + [test_script], cache_dir, None, check_perf=False, check_opset=user_opset ) - def test_014_cli_iteration_count(self): - # Test the first model in the corpus - test_script = list(common.test_scripts_dot_py.keys())[0] - - test_iterations = 123 - testargs = [ - "turnkey", - "-i", - os.path.join(corpus_dir, test_script), - "--cache-dir", - cache_dir, - "discover", - "export-pytorch", - "optimize-ort", - "benchmark", - "--iterations", - str(test_iterations), - ] - with patch.object(sys, "argv", testargs): - turnkeycli() - - assert_success_of_builds( - [test_script], - cache_dir, - None, - check_perf=True, - check_iteration_count=test_iterations, - ) - - def test_015_cli_process_isolation(self): + def test_09_cli_process_isolation(self): # Test the first model in the corpus test_script = list(common.test_scripts_dot_py.keys())[0] @@ -665,7 +451,6 @@ def test_015_cli_process_isolation(self): "--opset", "17", "optimize-ort", - "benchmark", ] with patch.object(sys, "argv", testargs): turnkeycli() @@ -677,7 +462,7 @@ def test_015_cli_process_isolation(self): "Skipping, as torch.compile is not supported on Windows" "Revisit when torch.compile for Windows is supported", ) - def test_016_skip_compiled(self): + def test_010_skip_compiled(self): test_script = "compiled.py" testargs = [ "turnkey", @@ -688,7 +473,6 @@ def test_016_skip_compiled(self): "discover", "export-pytorch", "optimize-ort", - "benchmark", ] with patch.object(sys, "argv", testargs): turnkeycli() @@ -699,7 +483,7 @@ def test_016_skip_compiled(self): # One of those is compiled and should be skipped. assert builds_found == 1 - def test_017_invalid_file_type(self): + def test_011_invalid_file_type(self): # Ensure that we get an error when running turnkey with invalid input_files with self.assertRaises(SystemExit): testargs = [ @@ -709,12 +493,11 @@ def test_017_invalid_file_type(self): "discover", "export-pytorch", "optimize-ort", - "benchmark", ] with patch.object(sys, "argv", flatten(testargs)): turnkeycli() - def test_018_cli_export_only(self): + def test_012_cli_export_only(self): # Test the first model in the corpus test_script = list(common.test_scripts_dot_py.keys())[0] @@ -726,14 +509,13 @@ def test_018_cli_export_only(self): cache_dir, "discover", "export-pytorch", - "benchmark", ] with patch.object(sys, "argv", testargs): turnkeycli() assert_success_of_builds([test_script], cache_dir, check_onnx_file_count=1) - def test_019_cli_onnx_model(self): + def test_013_cli_onnx_model(self): """ Manually export an ONNX file, then feed it into the CLI """ @@ -757,14 +539,15 @@ def test_019_cli_onnx_model(self): "--cache-dir", cache_dir, "load-onnx", - "benchmark", + "convert-fp16", + "optimize-onnx", ] with patch.object(sys, "argv", testargs): turnkeycli() assert_success_of_builds([build_name], cache_dir) - def test_020_cli_onnx_model_opset(self): + def test_014_cli_onnx_model_opset(self): """ Manually export an ONNX file with a non-defualt opset, then feed it into the CLI """ @@ -792,27 +575,16 @@ def test_020_cli_onnx_model_opset(self): "--cache-dir", cache_dir, "load-onnx", - "benchmark", + "convert-fp16", + "optimize-onnx", ] with patch.object(sys, "argv", testargs): turnkeycli() assert_success_of_builds([build_name], cache_dir) - def test_021_args_encode_decode(self): - """ - Test the encoding and decoding of arguments that follow the - ["arg1::[value1,value2]","arg2::value1","flag_arg"]' format - """ - encoded_value = ["arg1::[value1,value2]", "arg2::value1", "flag_arg"] - decoded_value = decode_args(encoded_value) - reencoded_value = encode_args(decoded_value) - assert ( - reencoded_value == encoded_value - ), f"input: {encoded_value}, decoded: {decoded_value}, reencoded_value: {reencoded_value}" - - def test_022_benchmark_non_existent_file(self): - # Ensure we get an error when benchmarking a non existent file + def test_015_non_existent_file(self): + # Ensure we get an error when loading a non existent file with self.assertRaises(exceptions.ArgError): filename = "thou_shall_not_exist.py" with redirect_stdout(io.StringIO()) as f: @@ -823,13 +595,12 @@ def test_022_benchmark_non_existent_file(self): "discover", "export-pytorch", "optimize-ort", - "benchmark", ] with patch.object(sys, "argv", testargs): turnkeycli() - def test_023_benchmark_non_existent_file_prefix(self): - # Ensure we get an error when benchmarking a non existent file + def test_016_non_existent_file_prefix(self): + # Ensure we get an error when loading a non existent file with self.assertRaises(exceptions.ArgError): file_prefix = "non_existent_prefix_*.py" with redirect_stdout(io.StringIO()) as f: @@ -840,12 +611,11 @@ def test_023_benchmark_non_existent_file_prefix(self): "discover", "export-pytorch", "optimize-ort", - "benchmark", ] with patch.object(sys, "argv", testargs): turnkeycli() - def test_024_input_text_file(self): + def test_017_input_text_file(self): """ Ensure that we can intake .txt files """ @@ -868,12 +638,12 @@ def test_024_input_text_file(self): builds_found == 3 ), f"Expected 3 builds (1 for linear.py, 2 for linear2.py), but got {builds_found}." - def test_025_cli_timeout(self): + def test_018_cli_timeout(self): """ Make sure that the --timeout option and its associated reporting features work. timeout.py is designed to take a long time to export, which gives us the - oportunity to kill it with a timeout. + opportunity to kill it with a timeout. NOTE: this test can become flakey if: - exporting timeout.py takes less time than the timeout @@ -892,7 +662,6 @@ def test_025_cli_timeout(self): "discover", "export-pytorch", "optimize-ort", - "benchmark", ] with patch.object(sys, "argv", flatten(testargs)): turnkeycli() @@ -927,14 +696,14 @@ def test_025_cli_timeout(self): # Edge case where the CSV only contains a key for "error_log" assert "timeout" in timeout_summary["error_log"] - def test_026_cli_report(self): + def test_019_cli_report(self): # NOTE: this is not a unit test, it relies on other command # If this test is failing, make sure the following tests are passing: # - test_cli_corpus test_scripts = common.test_scripts_dot_py.keys() - # Benchmark the test corpus so we have builds to report + # Build the test corpus so we have builds to report testargs = [ "turnkey", "-i", @@ -944,7 +713,6 @@ def test_026_cli_report(self): "discover", "export-pytorch", "optimize-ort", - "benchmark", ] with patch.object(sys, "argv", flatten(testargs)): turnkeycli() @@ -970,11 +738,6 @@ def test_026_cli_report(self): "class", "parameters", "hash", - "runtime", - "device_type", - "device", - "mean_latency", - "throughput", "selected_sequence_of_tools", ] linear_summary = summary[1] @@ -997,18 +760,6 @@ def test_026_cli_report(self): assert ( linear_summary["hash"] == "80b93950" ), f"Wrong hash found {linear_summary['hash']}" - assert ( - linear_summary["runtime"] == "ort" - ), f"Wrong runtime found {linear_summary['runtime']}" - assert ( - linear_summary["device_type"] == "x86" - ), f"Wrong device type found {linear_summary['device_type']}" - assert ( - float(linear_summary["mean_latency"]) > 0 - ), f"latency must be >0, got {linear_summary['x86_latency']}" - assert ( - float(linear_summary["throughput"]) > 100 - ), f"throughput must be >100, got {linear_summary['throughput']}" # Make sure the report.get_dict() API works result_dict = report.get_dict( @@ -1040,63 +791,7 @@ def test_026_cli_report(self): # Catch the case where the value is not numeric assert False, f"Tool {tool} has invalid duration {duration}" - def test_027_cli_cache_benchmark(self): - - test_scripts = common.test_scripts_dot_py.keys() - - # Build the test corpus so we have builds to benchmark - testargs = [ - "turnkey", - "-i", - bash(f"{corpus_dir}/*.py"), - "--cache-dir", - cache_dir, - "discover", - "export-pytorch", - "optimize-ort", - ] - with patch.object(sys, "argv", flatten(testargs)): - turnkeycli() - - # Benchmark the single model from cache directory - selected_build = fs.get_available_builds(cache_dir)[-1] - state_file_path = os.path.join( - cache_dir, selected_build, f"{selected_build}_state.yaml" - ) - - testargs = [ - "turnkey", - "--cache-dir", - cache_dir, - "-i", - state_file_path, - "load-build", - "benchmark", - ] - with patch.object(sys, "argv", flatten(testargs)): - turnkeycli() - - # Make sure the benchmark happened - test_script = selected_build + ".py" - assert_success_of_builds([test_script], cache_dir, check_perf=True) - - # Benchmark the cache directory - testargs = [ - "turnkey", - "--cache-dir", - cache_dir, - "-i", - os.path.join(cache_dir, "*", "*_state.yaml"), - "load-build", - "benchmark", - ] - with patch.object(sys, "argv", flatten(testargs)): - turnkeycli() - - # Make sure the benchmarks happened - assert_success_of_builds(test_scripts, cache_dir, check_perf=True) - - def test_028_cli_onnx_verify(self): + def test_020_cli_onnx_verify(self): # Test the first model in the corpus test_script = list(common.test_scripts_dot_py.keys())[0] @@ -1116,7 +811,7 @@ def test_028_cli_onnx_verify(self): assert_success_of_builds([test_script], cache_dir) - def test_029_cli_fp16_convert(self): + def test_021_cli_fp16_convert(self): # Test the first model in the corpus test_script = list(common.test_scripts_dot_py.keys())[0] @@ -1136,7 +831,7 @@ def test_029_cli_fp16_convert(self): assert_success_of_builds([test_script], cache_dir) - def test_030_cli_cache_move(self): + def test_022_cli_cache_move(self): test_script = list(common.test_scripts_dot_py.keys())[0] @@ -1162,20 +857,19 @@ def test_030_cli_cache_move(self): new_cache_dir, selected_build, f"{selected_build}_state.yaml" ) - # Benchmark the cached build in its new location + # Build the cached build in its new location testargs = [ "turnkey", "-i", state_file_path, "load-build", - "benchmark", + "optimize-ort", ] with patch.object(sys, "argv", flatten(testargs)): turnkeycli() - # Make sure the benchmark happened test_script = selected_build + ".py" - assert_success_of_builds([test_script], new_cache_dir, check_perf=True) + assert_success_of_builds([test_script], new_cache_dir, check_perf=False) if __name__ == "__main__": diff --git a/test/unit.py b/test/unit.py index 49c909f..2fd2f1f 100644 --- a/test/unit.py +++ b/test/unit.py @@ -9,6 +9,7 @@ import turnkeyml.common.build as build import turnkeyml.run.performance as performance import turnkeyml.run.plugin_helpers as plugin_helpers +from turnkeyml.cli.parser_helpers import decode_args, encode_args class Testing(unittest.TestCase): @@ -153,6 +154,18 @@ def test_003_subprocess_logger(self): assert outside_stdout_msg in log_contents assert outside_stderr_msg in log_contents + def test_021_args_encode_decode(self): + """ + Test the encoding and decoding of arguments that follow the + ["arg1::[value1,value2]","arg2::value1","flag_arg"]' format + """ + encoded_value = ["arg1::[value1,value2]", "arg2::value1", "flag_arg"] + decoded_value = decode_args(encoded_value) + reencoded_value = encode_args(decoded_value) + assert ( + reencoded_value == encoded_value + ), f"input: {encoded_value}, decoded: {decoded_value}, reencoded_value: {reencoded_value}" + if __name__ == "__main__": unittest.main() From daa04f7fbab54b195f50d1a738ddd286e6f3d213 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Fri, 23 Aug 2024 15:22:28 -0400 Subject: [PATCH 02/22] deduplicate --- src/turnkeyml/common/test_helpers.py | 66 +++++++++++++++++++++++++++ test/benchmark.py | 67 +--------------------------- test/cli.py | 67 +--------------------------- 3 files changed, 68 insertions(+), 132 deletions(-) diff --git a/src/turnkeyml/common/test_helpers.py b/src/turnkeyml/common/test_helpers.py index 1f80b2b..b29e3ac 100644 --- a/src/turnkeyml/common/test_helpers.py +++ b/src/turnkeyml/common/test_helpers.py @@ -100,6 +100,72 @@ def forward(self, x): } +extras_dot_py = { + "compiled.py": """ +# labels: name::linear author::selftest test_group::selftest task::test +import torch + +torch.manual_seed(0) + + +class LinearTestModel(torch.nn.Module): + def __init__(self, input_features, output_features): + super(LinearTestModel, self).__init__() + self.fc = torch.nn.Linear(input_features, output_features) + + def forward(self, x): + output = self.fc(x) + return output + + +input_features = 10 +output_features = 10 + +# Compiled model +model = LinearTestModel(input_features, output_features) +model = torch.compile(model) +inputs = {"x": torch.rand(input_features)} +model(**inputs) + +# Non-compiled model +model2 = LinearTestModel(input_features * 2, output_features) +inputs2 = {"x": torch.rand(input_features * 2)} +model2(**inputs2) + """, + "selected_models.txt": f""" + {os.path.join(corpus_dir,"linear.py")} + {os.path.join(corpus_dir,"linear2.py")} + """, + "timeout.py": """ +# labels: name::timeout author::turnkey license::mit test_group::a task::test +import torch + +torch.manual_seed(0) + + +class LinearTestModel(torch.nn.Module): + def __init__(self, input_features, output_features): + super(LinearTestModel, self).__init__() + self.fc = torch.nn.Linear(input_features, output_features) + + def forward(self, x): + output = self.fc(x) + return output + + +input_features = 500000 +output_features = 1000 + +# Model and input configurations +model = LinearTestModel(input_features, output_features) +inputs = {"x": torch.rand(input_features)} + +output = model(**inputs) + + """, +} + + def create_test_dir( key: str, test_scripts: Dict = None, diff --git a/test/benchmark.py b/test/benchmark.py index 77e64b8..fccb1cd 100644 --- a/test/benchmark.py +++ b/test/benchmark.py @@ -618,75 +618,10 @@ def test_011_cli_cache_move(self): cache_dir, corpus_dir = common.create_test_dir("benchmark") new_cache_dir = f"{cache_dir}2" - extras_dot_py = { - "compiled.py": """ -# labels: name::linear author::selftest test_group::selftest task::test -import torch - -torch.manual_seed(0) - - -class LinearTestModel(torch.nn.Module): - def __init__(self, input_features, output_features): - super(LinearTestModel, self).__init__() - self.fc = torch.nn.Linear(input_features, output_features) - - def forward(self, x): - output = self.fc(x) - return output - - -input_features = 10 -output_features = 10 - -# Compiled model -model = LinearTestModel(input_features, output_features) -model = torch.compile(model) -inputs = {"x": torch.rand(input_features)} -model(**inputs) - -# Non-compiled model -model2 = LinearTestModel(input_features * 2, output_features) -inputs2 = {"x": torch.rand(input_features * 2)} -model2(**inputs2) - """, - "selected_models.txt": f""" - {os.path.join(corpus_dir,"linear.py")} - {os.path.join(corpus_dir,"linear2.py")} - """, - "timeout.py": """ -# labels: name::timeout author::turnkey license::mit test_group::a task::test -import torch - -torch.manual_seed(0) - - -class LinearTestModel(torch.nn.Module): - def __init__(self, input_features, output_features): - super(LinearTestModel, self).__init__() - self.fc = torch.nn.Linear(input_features, output_features) - - def forward(self, x): - output = self.fc(x) - return output - - -input_features = 500000 -output_features = 1000 - -# Model and input configurations -model = LinearTestModel(input_features, output_features) -inputs = {"x": torch.rand(input_features)} - -output = model(**inputs) - - """, - } - extras_dir = os.path.join(corpus_dir, "extras") os.makedirs(extras_dir, exist_ok=True) - for key, value in extras_dot_py.items(): + for key, value in common.extras_dot_py.items(): file_path = os.path.join(extras_dir, key) with open(file_path, "w", encoding="utf") as f: diff --git a/test/cli.py b/test/cli.py index ff92351..b06ff15 100644 --- a/test/cli.py +++ b/test/cli.py @@ -877,75 +877,10 @@ def test_022_cli_cache_move(self): cache_dir, corpus_dir = common.create_test_dir("cli") new_cache_dir = f"{cache_dir}2" - extras_dot_py = { - "compiled.py": """ -# labels: name::linear author::selftest test_group::selftest task::test -import torch - -torch.manual_seed(0) - - -class LinearTestModel(torch.nn.Module): - def __init__(self, input_features, output_features): - super(LinearTestModel, self).__init__() - self.fc = torch.nn.Linear(input_features, output_features) - - def forward(self, x): - output = self.fc(x) - return output - - -input_features = 10 -output_features = 10 - -# Compiled model -model = LinearTestModel(input_features, output_features) -model = torch.compile(model) -inputs = {"x": torch.rand(input_features)} -model(**inputs) - -# Non-compiled model -model2 = LinearTestModel(input_features * 2, output_features) -inputs2 = {"x": torch.rand(input_features * 2)} -model2(**inputs2) - """, - "selected_models.txt": f""" - {os.path.join(corpus_dir,"linear.py")} - {os.path.join(corpus_dir,"linear2.py")} - """, - "timeout.py": """ -# labels: name::timeout author::turnkey license::mit test_group::a task::test -import torch - -torch.manual_seed(0) - - -class LinearTestModel(torch.nn.Module): - def __init__(self, input_features, output_features): - super(LinearTestModel, self).__init__() - self.fc = torch.nn.Linear(input_features, output_features) - - def forward(self, x): - output = self.fc(x) - return output - - -input_features = 500000 -output_features = 1000 - -# Model and input configurations -model = LinearTestModel(input_features, output_features) -inputs = {"x": torch.rand(input_features)} - -output = model(**inputs) - - """, - } - extras_dir = os.path.join(corpus_dir, "extras") os.makedirs(extras_dir, exist_ok=True) - for key, value in extras_dot_py.items(): + for key, value in common.extras_dot_py.items(): file_path = os.path.join(extras_dir, key) with open(file_path, "w", encoding="utf") as f: From e0d13070578367cd4e3e59d5d808edf7fb4b40ac Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Fri, 23 Aug 2024 15:39:05 -0400 Subject: [PATCH 03/22] fix --- src/turnkeyml/common/test_helpers.py | 11 ++++++----- src/turnkeyml/version.py | 2 +- test/benchmark.py | 2 +- test/cli.py | 2 +- 4 files changed, 9 insertions(+), 8 deletions(-) diff --git a/src/turnkeyml/common/test_helpers.py b/src/turnkeyml/common/test_helpers.py index b29e3ac..9db3a40 100644 --- a/src/turnkeyml/common/test_helpers.py +++ b/src/turnkeyml/common/test_helpers.py @@ -100,8 +100,9 @@ def forward(self, x): } -extras_dot_py = { - "compiled.py": """ +def extras_python(corpus_dir: str): + return { + "compiled.py": """ # labels: name::linear author::selftest test_group::selftest task::test import torch @@ -132,11 +133,11 @@ def forward(self, x): inputs2 = {"x": torch.rand(input_features * 2)} model2(**inputs2) """, - "selected_models.txt": f""" + "selected_models.txt": f""" {os.path.join(corpus_dir,"linear.py")} {os.path.join(corpus_dir,"linear2.py")} """, - "timeout.py": """ + "timeout.py": """ # labels: name::timeout author::turnkey license::mit test_group::a task::test import torch @@ -163,7 +164,7 @@ def forward(self, x): output = model(**inputs) """, -} + } def create_test_dir( diff --git a/src/turnkeyml/version.py b/src/turnkeyml/version.py index c11769e..35c154a 100644 --- a/src/turnkeyml/version.py +++ b/src/turnkeyml/version.py @@ -1 +1 @@ -__version__ = "3.0.7" +__version__ = "3.0.8" diff --git a/test/benchmark.py b/test/benchmark.py index fccb1cd..607b33d 100644 --- a/test/benchmark.py +++ b/test/benchmark.py @@ -621,7 +621,7 @@ def test_011_cli_cache_move(self): extras_dir = os.path.join(corpus_dir, "extras") os.makedirs(extras_dir, exist_ok=True) - for key, value in common.extras_dot_py.items(): + for key, value in common.extras_python(corpus_dir).items(): file_path = os.path.join(extras_dir, key) with open(file_path, "w", encoding="utf") as f: diff --git a/test/cli.py b/test/cli.py index b06ff15..a4e32be 100644 --- a/test/cli.py +++ b/test/cli.py @@ -880,7 +880,7 @@ def test_022_cli_cache_move(self): extras_dir = os.path.join(corpus_dir, "extras") os.makedirs(extras_dir, exist_ok=True) - for key, value in common.extras_dot_py.items(): + for key, value in common.extras_python(corpus_dir).items(): file_path = os.path.join(extras_dir, key) with open(file_path, "w", encoding="utf") as f: From 83e4d17938470c3cf85839377bb724133a068f36 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Fri, 23 Aug 2024 16:03:55 -0400 Subject: [PATCH 04/22] fix tests --- test/cli.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/cli.py b/test/cli.py index a4e32be..ab25637 100644 --- a/test/cli.py +++ b/test/cli.py @@ -455,7 +455,7 @@ def test_09_cli_process_isolation(self): with patch.object(sys, "argv", testargs): turnkeycli() - assert_success_of_builds([test_script], cache_dir, None, check_perf=True) + assert_success_of_builds([test_script], cache_dir, None, check_perf=False) @unittest.skipIf( platform.system() == "Windows", @@ -540,7 +540,7 @@ def test_013_cli_onnx_model(self): cache_dir, "load-onnx", "convert-fp16", - "optimize-onnx", + "optimize-ort", ] with patch.object(sys, "argv", testargs): turnkeycli() @@ -549,7 +549,7 @@ def test_013_cli_onnx_model(self): def test_014_cli_onnx_model_opset(self): """ - Manually export an ONNX file with a non-defualt opset, then feed it into the CLI + Manually export an ONNX file with a non-default opset, then feed it into the CLI """ build_name = "receive_onnx_opset" onnx_file = os.path.join(corpus_dir, f"{build_name}.onnx") @@ -576,7 +576,7 @@ def test_014_cli_onnx_model_opset(self): cache_dir, "load-onnx", "convert-fp16", - "optimize-onnx", + "optimize-ort", ] with patch.object(sys, "argv", testargs): turnkeycli() From 93548a11fa401b513f087ddad435cd0d10ca5a5b Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Fri, 23 Aug 2024 16:35:49 -0400 Subject: [PATCH 05/22] fix tests --- src/turnkeyml/common/test_helpers.py | 62 ++++++++++++++++++++++++++- test/benchmark.py | 63 +--------------------------- test/cli.py | 63 +--------------------------- 3 files changed, 65 insertions(+), 123 deletions(-) diff --git a/src/turnkeyml/common/test_helpers.py b/src/turnkeyml/common/test_helpers.py index 9db3a40..816bdc5 100644 --- a/src/turnkeyml/common/test_helpers.py +++ b/src/turnkeyml/common/test_helpers.py @@ -1,7 +1,10 @@ import os import shutil -from typing import Dict +from typing import List, Tuple, Any, Dict, Optional +import onnx import turnkeyml.common.filesystem as fs +import turnkeyml.common.build as build +import turnkeyml.common.onnx_helpers as onnx_helpers from turnkeyml.state import load_state @@ -216,3 +219,60 @@ def get_stats_and_state( return stats.stats, build_state raise Exception(f"Stats not found for {test_script}") + + +def assert_success_of_builds( + test_script_files: List[str], + cache_dir: str, + info_property: Tuple[str, Any] = None, + check_perf: bool = False, + check_opset: Optional[int] = None, + check_iteration_count: Optional[int] = None, + check_onnx_file_count: Optional[int] = None, +) -> int: + # Figure out the build name by surveying the build cache + # for a build that includes test_script_name in the name + builds = fs.get_all(cache_dir) + builds_found = 0 + + for test_script in test_script_files: + test_script_name = strip_dot_py(test_script) + script_build_found = False + + for build_state_file in builds: + if test_script_name in build_state_file: + build_state = load_state(state_path=build_state_file) + stats = fs.Stats( + build_state.cache_dir, + build_state.build_name, + ) + assert build_state.build_status == build.FunctionStatus.SUCCESSFUL + script_build_found = True + builds_found += 1 + + if info_property is not None: + assert ( + build_state.info.__dict__[info_property[0]] == info_property[1] + ), f"{build_state.info.__dict__[info_property[0]]} == {info_property[1]}" + + if check_perf: + assert stats.stats["mean_latency"] > 0 + assert stats.stats["throughput"] > 0 + + if check_iteration_count: + iterations = stats.stats["iterations"] + assert iterations == check_iteration_count + + if check_opset: + onnx_model = onnx.load(build_state.results) + model_opset = getattr(onnx_model.opset_import[0], "version", None) + assert model_opset == check_opset + + if check_onnx_file_count: + onnx_dir = onnx_helpers.onnx_dir(build_state) + assert len(os.listdir(onnx_dir)) == check_onnx_file_count + + assert script_build_found + + # Returns the total number of builds found + return builds_found diff --git a/test/benchmark.py b/test/benchmark.py index 607b33d..cc61f68 100644 --- a/test/benchmark.py +++ b/test/benchmark.py @@ -6,13 +6,12 @@ import shutil import glob import csv -from typing import List, Tuple, Any, Union, Optional +from typing import List, Union import unittest from unittest.mock import patch import sys import io from contextlib import redirect_stdout -import onnx import platform import torch from turnkeyml.cli.cli import main as turnkeycli @@ -20,9 +19,8 @@ import turnkeyml.common.filesystem as fs import turnkeyml.common.build as build import turnkeyml.common.exceptions as exceptions -import turnkeyml.common.onnx_helpers as onnx_helpers import turnkeyml.common.test_helpers as common -from turnkeyml.state import load_state +from turnkeyml.common.test_helpers import assert_success_of_builds def bash(cmd: str) -> List[str]: @@ -45,63 +43,6 @@ def flatten(lst: List[Union[str, List[str]]]) -> List[str]: return flattened -def assert_success_of_builds( - test_script_files: List[str], - cache_dir: str, - info_property: Tuple[str, Any] = None, - check_perf: bool = False, - check_opset: Optional[int] = None, - check_iteration_count: Optional[int] = None, - check_onnx_file_count: Optional[int] = None, -) -> int: - # Figure out the build name by surveying the build cache - # for a build that includes test_script_name in the name - builds = fs.get_all(cache_dir) - builds_found = 0 - - for test_script in test_script_files: - test_script_name = common.strip_dot_py(test_script) - script_build_found = False - - for build_state_file in builds: - if test_script_name in build_state_file: - build_state = load_state(state_path=build_state_file) - stats = fs.Stats( - build_state.cache_dir, - build_state.build_name, - ) - assert build_state.build_status == build.FunctionStatus.SUCCESSFUL - script_build_found = True - builds_found += 1 - - if info_property is not None: - assert ( - build_state.info.__dict__[info_property[0]] == info_property[1] - ), f"{build_state.info.__dict__[info_property[0]]} == {info_property[1]}" - - if check_perf: - assert stats.stats["mean_latency"] > 0 - assert stats.stats["throughput"] > 0 - - if check_iteration_count: - iterations = stats.stats["iterations"] - assert iterations == check_iteration_count - - if check_opset: - onnx_model = onnx.load(build_state.results) - model_opset = getattr(onnx_model.opset_import[0], "version", None) - assert model_opset == check_opset - - if check_onnx_file_count: - onnx_dir = onnx_helpers.onnx_dir(build_state) - assert len(os.listdir(onnx_dir)) == check_onnx_file_count - - assert script_build_found - - # Returns the total number of builds found - return builds_found - - class SmallPytorchModel(torch.nn.Module): def __init__(self): super(SmallPytorchModel, self).__init__() diff --git a/test/cli.py b/test/cli.py index ab25637..c662f15 100644 --- a/test/cli.py +++ b/test/cli.py @@ -6,14 +6,13 @@ import shutil import glob import csv -from typing import List, Tuple, Any, Union, Optional +from typing import List, Union import unittest from unittest.mock import patch import sys import io from contextlib import redirect_stdout import yaml -import onnx import platform import torch from turnkeyml.cli.cli import main as turnkeycli @@ -21,9 +20,8 @@ import turnkeyml.common.filesystem as fs import turnkeyml.common.build as build import turnkeyml.common.exceptions as exceptions -import turnkeyml.common.onnx_helpers as onnx_helpers import turnkeyml.common.test_helpers as common -from turnkeyml.state import load_state +from turnkeyml.common.test_helpers import assert_success_of_builds def bash(cmd: str) -> List[str]: @@ -46,63 +44,6 @@ def flatten(lst: List[Union[str, List[str]]]) -> List[str]: return flattened -def assert_success_of_builds( - test_script_files: List[str], - cache_dir: str, - info_property: Tuple[str, Any] = None, - check_perf: bool = False, - check_opset: Optional[int] = None, - check_iteration_count: Optional[int] = None, - check_onnx_file_count: Optional[int] = None, -) -> int: - # Figure out the build name by surveying the build cache - # for a build that includes test_script_name in the name - builds = fs.get_all(cache_dir) - builds_found = 0 - - for test_script in test_script_files: - test_script_name = common.strip_dot_py(test_script) - script_build_found = False - - for build_state_file in builds: - if test_script_name in build_state_file: - build_state = load_state(state_path=build_state_file) - stats = fs.Stats( - build_state.cache_dir, - build_state.build_name, - ) - assert build_state.build_status == build.FunctionStatus.SUCCESSFUL - script_build_found = True - builds_found += 1 - - if info_property is not None: - assert ( - build_state.info.__dict__[info_property[0]] == info_property[1] - ), f"{build_state.info.__dict__[info_property[0]]} == {info_property[1]}" - - if check_perf: - assert stats.stats["mean_latency"] > 0 - assert stats.stats["throughput"] > 0 - - if check_iteration_count: - iterations = stats.stats["iterations"] - assert iterations == check_iteration_count - - if check_opset: - onnx_model = onnx.load(build_state.results) - model_opset = getattr(onnx_model.opset_import[0], "version", None) - assert model_opset == check_opset - - if check_onnx_file_count: - onnx_dir = onnx_helpers.onnx_dir(build_state) - assert len(os.listdir(onnx_dir)) == check_onnx_file_count - - assert script_build_found - - # Returns the total number of builds found - return builds_found - - class SmallPytorchModel(torch.nn.Module): def __init__(self): super(SmallPytorchModel, self).__init__() From 98da43367b6f81264f864c185a01b3b6c2dd9503 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Fri, 23 Aug 2024 17:18:18 -0400 Subject: [PATCH 06/22] fixes --- test/benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/benchmark.py b/test/benchmark.py index cc61f68..5f0f63a 100644 --- a/test/benchmark.py +++ b/test/benchmark.py @@ -1,5 +1,5 @@ """ -Tests focused on the benchmkarking functionality of turnkey CLI +Tests focused on the benchmarking functionality of turnkey CLI """ import os From efc5cb8013511286849a118929c9ce353b15c940 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Mon, 26 Aug 2024 10:25:48 -0400 Subject: [PATCH 07/22] create devices plugin --- .github/workflows/test_turnkey.yml | 2 +- .../src/turnkeyml_plugin_devices/common}/run/__init__.py | 0 .../devices/src/turnkeyml_plugin_devices/common}/run/basert.py | 0 .../src/turnkeyml_plugin_devices/common}/run/benchmark_model.py | 0 .../devices/src/turnkeyml_plugin_devices/common}/run/devices.py | 0 .../src/turnkeyml_plugin_devices/common}/run/performance.py | 0 .../src/turnkeyml_plugin_devices/common}/run/plugin_helpers.py | 0 .../devices/src/turnkeyml_plugin_devices}/onnxrt/__init__.py | 0 .../devices/src/turnkeyml_plugin_devices}/onnxrt/execute.py | 0 .../devices/src/turnkeyml_plugin_devices}/onnxrt/runtime.py | 0 .../src/turnkeyml_plugin_devices}/onnxrt/within_conda.py | 0 .../devices/src/turnkeyml_plugin_devices}/tensorrt/__init__.py | 0 .../devices/src/turnkeyml_plugin_devices}/tensorrt/execute.py | 0 .../devices/src/turnkeyml_plugin_devices}/tensorrt/runtime.py | 0 .../devices/src/turnkeyml_plugin_devices}/torchrt/__init__.py | 0 .../devices/src/turnkeyml_plugin_devices}/torchrt/runtime.py | 0 {test => plugins/devices/test}/benchmark.py | 0 17 files changed, 1 insertion(+), 1 deletion(-) rename {src/turnkeyml => plugins/devices/src/turnkeyml_plugin_devices/common}/run/__init__.py (100%) rename {src/turnkeyml => plugins/devices/src/turnkeyml_plugin_devices/common}/run/basert.py (100%) rename {src/turnkeyml => plugins/devices/src/turnkeyml_plugin_devices/common}/run/benchmark_model.py (100%) rename {src/turnkeyml => plugins/devices/src/turnkeyml_plugin_devices/common}/run/devices.py (100%) rename {src/turnkeyml => plugins/devices/src/turnkeyml_plugin_devices/common}/run/performance.py (100%) rename {src/turnkeyml => plugins/devices/src/turnkeyml_plugin_devices/common}/run/plugin_helpers.py (100%) rename {src/turnkeyml/run => plugins/devices/src/turnkeyml_plugin_devices}/onnxrt/__init__.py (100%) rename {src/turnkeyml/run => plugins/devices/src/turnkeyml_plugin_devices}/onnxrt/execute.py (100%) rename {src/turnkeyml/run => plugins/devices/src/turnkeyml_plugin_devices}/onnxrt/runtime.py (100%) rename {src/turnkeyml/run => plugins/devices/src/turnkeyml_plugin_devices}/onnxrt/within_conda.py (100%) rename {src/turnkeyml/run => plugins/devices/src/turnkeyml_plugin_devices}/tensorrt/__init__.py (100%) rename {src/turnkeyml/run => plugins/devices/src/turnkeyml_plugin_devices}/tensorrt/execute.py (100%) rename {src/turnkeyml/run => plugins/devices/src/turnkeyml_plugin_devices}/tensorrt/runtime.py (100%) rename {src/turnkeyml/run => plugins/devices/src/turnkeyml_plugin_devices}/torchrt/__init__.py (100%) rename {src/turnkeyml/run => plugins/devices/src/turnkeyml_plugin_devices}/torchrt/runtime.py (100%) rename {test => plugins/devices/test}/benchmark.py (100%) diff --git a/.github/workflows/test_turnkey.yml b/.github/workflows/test_turnkey.yml index 173bd38..16a3175 100644 --- a/.github/workflows/test_turnkey.yml +++ b/.github/workflows/test_turnkey.yml @@ -36,6 +36,7 @@ jobs: conda install pylint pip install pytest pip install -e . + pip install -e plugins/devices pip install transformers timm python -m pip check - name: Lint with PyLint @@ -70,7 +71,6 @@ jobs: cd test/ python cli.py python analysis.py - python benchmark.py - name: Test example plugins shell: bash -el {0} run: | diff --git a/src/turnkeyml/run/__init__.py b/plugins/devices/src/turnkeyml_plugin_devices/common/run/__init__.py similarity index 100% rename from src/turnkeyml/run/__init__.py rename to plugins/devices/src/turnkeyml_plugin_devices/common/run/__init__.py diff --git a/src/turnkeyml/run/basert.py b/plugins/devices/src/turnkeyml_plugin_devices/common/run/basert.py similarity index 100% rename from src/turnkeyml/run/basert.py rename to plugins/devices/src/turnkeyml_plugin_devices/common/run/basert.py diff --git a/src/turnkeyml/run/benchmark_model.py b/plugins/devices/src/turnkeyml_plugin_devices/common/run/benchmark_model.py similarity index 100% rename from src/turnkeyml/run/benchmark_model.py rename to plugins/devices/src/turnkeyml_plugin_devices/common/run/benchmark_model.py diff --git a/src/turnkeyml/run/devices.py b/plugins/devices/src/turnkeyml_plugin_devices/common/run/devices.py similarity index 100% rename from src/turnkeyml/run/devices.py rename to plugins/devices/src/turnkeyml_plugin_devices/common/run/devices.py diff --git a/src/turnkeyml/run/performance.py b/plugins/devices/src/turnkeyml_plugin_devices/common/run/performance.py similarity index 100% rename from src/turnkeyml/run/performance.py rename to plugins/devices/src/turnkeyml_plugin_devices/common/run/performance.py diff --git a/src/turnkeyml/run/plugin_helpers.py b/plugins/devices/src/turnkeyml_plugin_devices/common/run/plugin_helpers.py similarity index 100% rename from src/turnkeyml/run/plugin_helpers.py rename to plugins/devices/src/turnkeyml_plugin_devices/common/run/plugin_helpers.py diff --git a/src/turnkeyml/run/onnxrt/__init__.py b/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/__init__.py similarity index 100% rename from src/turnkeyml/run/onnxrt/__init__.py rename to plugins/devices/src/turnkeyml_plugin_devices/onnxrt/__init__.py diff --git a/src/turnkeyml/run/onnxrt/execute.py b/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/execute.py similarity index 100% rename from src/turnkeyml/run/onnxrt/execute.py rename to plugins/devices/src/turnkeyml_plugin_devices/onnxrt/execute.py diff --git a/src/turnkeyml/run/onnxrt/runtime.py b/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/runtime.py similarity index 100% rename from src/turnkeyml/run/onnxrt/runtime.py rename to plugins/devices/src/turnkeyml_plugin_devices/onnxrt/runtime.py diff --git a/src/turnkeyml/run/onnxrt/within_conda.py b/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/within_conda.py similarity index 100% rename from src/turnkeyml/run/onnxrt/within_conda.py rename to plugins/devices/src/turnkeyml_plugin_devices/onnxrt/within_conda.py diff --git a/src/turnkeyml/run/tensorrt/__init__.py b/plugins/devices/src/turnkeyml_plugin_devices/tensorrt/__init__.py similarity index 100% rename from src/turnkeyml/run/tensorrt/__init__.py rename to plugins/devices/src/turnkeyml_plugin_devices/tensorrt/__init__.py diff --git a/src/turnkeyml/run/tensorrt/execute.py b/plugins/devices/src/turnkeyml_plugin_devices/tensorrt/execute.py similarity index 100% rename from src/turnkeyml/run/tensorrt/execute.py rename to plugins/devices/src/turnkeyml_plugin_devices/tensorrt/execute.py diff --git a/src/turnkeyml/run/tensorrt/runtime.py b/plugins/devices/src/turnkeyml_plugin_devices/tensorrt/runtime.py similarity index 100% rename from src/turnkeyml/run/tensorrt/runtime.py rename to plugins/devices/src/turnkeyml_plugin_devices/tensorrt/runtime.py diff --git a/src/turnkeyml/run/torchrt/__init__.py b/plugins/devices/src/turnkeyml_plugin_devices/torchrt/__init__.py similarity index 100% rename from src/turnkeyml/run/torchrt/__init__.py rename to plugins/devices/src/turnkeyml_plugin_devices/torchrt/__init__.py diff --git a/src/turnkeyml/run/torchrt/runtime.py b/plugins/devices/src/turnkeyml_plugin_devices/torchrt/runtime.py similarity index 100% rename from src/turnkeyml/run/torchrt/runtime.py rename to plugins/devices/src/turnkeyml_plugin_devices/torchrt/runtime.py diff --git a/test/benchmark.py b/plugins/devices/test/benchmark.py similarity index 100% rename from test/benchmark.py rename to plugins/devices/test/benchmark.py From 4733678e20260279634e488bc0350cefc239fd07 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Mon, 26 Aug 2024 10:29:22 -0400 Subject: [PATCH 08/22] fix paths --- .../common/run/basert.py | 4 +- .../common/run/benchmark_model.py | 21 ++-- .../common/run/devices.py | 106 +++++++++--------- .../onnxrt/__init__.py | 10 -- .../onnxrt/execute.py | 2 +- .../onnxrt/runtime.py | 10 +- .../tensorrt/__init__.py | 11 -- .../tensorrt/runtime.py | 8 +- .../torchrt/__init__.py | 14 --- .../torchrt/runtime.py | 6 +- 10 files changed, 83 insertions(+), 109 deletions(-) diff --git a/plugins/devices/src/turnkeyml_plugin_devices/common/run/basert.py b/plugins/devices/src/turnkeyml_plugin_devices/common/run/basert.py index 42722f9..7a72d4f 100644 --- a/plugins/devices/src/turnkeyml_plugin_devices/common/run/basert.py +++ b/plugins/devices/src/turnkeyml_plugin_devices/common/run/basert.py @@ -7,7 +7,7 @@ from abc import ABC, abstractmethod import torch import numpy as np -from turnkeyml.run.performance import MeasuredPerformance, Device +from turnkeyml_plugin_devices.common.run.performance import MeasuredPerformance, Device import turnkeyml.common.build as build import turnkeyml.common.exceptions as exp import turnkeyml.common.filesystem as fs @@ -87,7 +87,7 @@ def __init__( # Validate runtime is supported if runtime not in runtimes_supported: raise ValueError( - f"'runtime' argument {runtime} passed to TensorRT, which only " + f"'runtime' argument {runtime} passed to runtime, which only " f"supports runtimes: {runtimes_supported}" ) diff --git a/plugins/devices/src/turnkeyml_plugin_devices/common/run/benchmark_model.py b/plugins/devices/src/turnkeyml_plugin_devices/common/run/benchmark_model.py index 66e6094..8efa25a 100644 --- a/plugins/devices/src/turnkeyml_plugin_devices/common/run/benchmark_model.py +++ b/plugins/devices/src/turnkeyml_plugin_devices/common/run/benchmark_model.py @@ -4,13 +4,12 @@ import turnkeyml.common.filesystem as fs from turnkeyml.tools import Tool from turnkeyml.state import State -from turnkeyml.run.devices import ( - SUPPORTED_RUNTIMES, - SUPPORTED_DEVICES, +import turnkeyml.cli.parser_helpers as parser_helpers +from turnkeyml_plugin_devices.common.run.devices import ( + runtime_plugins, apply_default_runtime, ) -import turnkeyml.cli.parser_helpers as parser_helpers -from turnkeyml.run.performance import Device, parse_device +from turnkeyml_plugin_devices.common.run.performance import Device, parse_device default_iterations = 100 benchmark_default_device = "x86" @@ -35,6 +34,8 @@ def __init__(self): @staticmethod def parser(add_help: bool = True) -> argparse.ArgumentParser: + supported_devices, supported_runtimes, _ = runtime_plugins() + parser = __class__.helpful_parser( short_description="Benchmark a model", add_help=add_help, @@ -42,7 +43,7 @@ def parser(add_help: bool = True) -> argparse.ArgumentParser: parser.add_argument( "--device", - choices=SUPPORTED_DEVICES, + choices=supported_devices, dest="device", help="Type of hardware device to be used for the benchmark " f'(defaults to "{benchmark_default_device}")', @@ -51,7 +52,7 @@ def parser(add_help: bool = True) -> argparse.ArgumentParser: parser.add_argument( "--runtime", - choices=SUPPORTED_RUNTIMES.keys(), + choices=supported_runtimes.keys(), dest="runtime", help="Software runtime that will be used to collect the benchmark. " "Must be compatible with the selected device. " @@ -101,11 +102,13 @@ def run( rt_args: Optional[str] = None, ): + _, supported_runtimes, _ = runtime_plugins() + selected_runtime = apply_default_runtime(device, runtime) # Get the default part and config by providing the Device class with # the supported devices by the runtime - runtime_supported_devices = SUPPORTED_RUNTIMES[selected_runtime][ + runtime_supported_devices = supported_runtimes[selected_runtime][ "supported_devices" ] specific_device = str(Device(device, runtime_supported_devices)) @@ -116,7 +119,7 @@ def run( rt_args_to_use = rt_args try: - runtime_info = SUPPORTED_RUNTIMES[selected_runtime] + runtime_info = supported_runtimes[selected_runtime] except KeyError as e: # User should never get this far without hitting an actionable error message, # but let's raise an exception just in case. diff --git a/plugins/devices/src/turnkeyml_plugin_devices/common/run/devices.py b/plugins/devices/src/turnkeyml_plugin_devices/common/run/devices.py index 3ec62b1..80d831a 100644 --- a/plugins/devices/src/turnkeyml_plugin_devices/common/run/devices.py +++ b/plugins/devices/src/turnkeyml_plugin_devices/common/run/devices.py @@ -1,13 +1,13 @@ from typing import Optional from typing import List, Dict, Tuple -import turnkeyml.run.onnxrt as onnxrt -import turnkeyml.run.tensorrt as tensorrt -import turnkeyml.run.torchrt as torchrt import turnkeyml.common.plugins as plugins from turnkeyml.sequence import Sequence import turnkeyml.common.exceptions as exp +DEFAULT_RUNTIME = 0 + + def supported_devices_list(data: Dict, parent_key: str = "") -> List: """Recursive function to generate all device::part::config pairs""" result = [] @@ -22,56 +22,56 @@ def supported_devices_list(data: Dict, parent_key: str = "") -> List: return result -discovered_plugins = plugins.discover() +def runtime_plugins(): + + plugin_modules = plugins.discover().values() + + supported_runtimes = {} + + for module in plugin_modules: + if "runtimes" in module.implements.keys(): + for runtime_name, runtime_info in module.implements["runtimes"].items(): + if runtime_name in supported_runtimes: + raise ValueError( + f"Your turnkeyml installation has two runtimes named '{runtime_name}' " + "installed. You must uninstall one of your plugins that includes " + f"{runtime_name}. Your imported runtime plugins are: {supported_runtimes}\n" + f"This error was thrown while trying to import {module}" + ) + if isinstance(runtime_info["supported_devices"], set): + runtime_info["supported_devices"] = { + item: {} for item in runtime_info["supported_devices"] + } + supported_runtimes[runtime_name] = runtime_info + + # Get the list of supported devices by checking which devices each runtime supports + supported_devices = [] + for runtime_info in supported_runtimes.values(): + supported_devices.extend( + supported_devices_list(runtime_info["supported_devices"]) + ) + supported_devices = list(set(supported_devices)) + + # Organizing the supported devices that are in the "long form". + # Those are not the elements used for setting the defaults. + # This is useful for nicely showing the list of supported devices as part of the help menu. + supported_devices.sort() -DEFAULT_RUNTIME = 0 + # Create a map of devices to the runtimes that support them + device_runtime_map = {key: [] for key in supported_devices} + for device in supported_devices: + for runtime_name, runtime_info in supported_runtimes.items(): + if device in supported_devices_list(runtime_info["supported_devices"]): + device_runtime_map[device].append(runtime_name) -# Note: order matters here. We append the discovered_plugins after builtin so -# that the default runtime for each device will come from a builtin, whenever -# available. -builtin_runtimes = [onnxrt, tensorrt, torchrt] -plugin_modules = builtin_runtimes + list(discovered_plugins.values()) - -SUPPORTED_RUNTIMES = {} - -for module in plugin_modules: - if "runtimes" in module.implements.keys(): - for runtime_name, runtime_info in module.implements["runtimes"].items(): - if runtime_name in SUPPORTED_RUNTIMES: - raise ValueError( - f"Your turnkeyml installation has two runtimes named '{runtime_name}' " - "installed. You must uninstall one of your plugins that includes " - f"{runtime_name}. Your imported runtime plugins are: {SUPPORTED_RUNTIMES}\n" - f"This error was thrown while trying to import {module}" - ) - if isinstance(runtime_info["supported_devices"], set): - runtime_info["supported_devices"] = { - item: {} for item in runtime_info["supported_devices"] - } - SUPPORTED_RUNTIMES[runtime_name] = runtime_info - -# Get the list of supported devices by checking which devices each runtime supports -SUPPORTED_DEVICES = [] -for runtime_info in SUPPORTED_RUNTIMES.values(): - SUPPORTED_DEVICES.extend(supported_devices_list(runtime_info["supported_devices"])) -SUPPORTED_DEVICES = list(set(SUPPORTED_DEVICES)) - -# Organizing the supported devices that are in the "long form". -# Those are not the elements used for setting the defaults. -# This is useful for nicely showing the list of supported devices as part of the help menu. -SUPPORTED_DEVICES.sort() - -# Create a map of devices to the runtimes that support them -DEVICE_RUNTIME_MAP = {key: [] for key in SUPPORTED_DEVICES} -for device in SUPPORTED_DEVICES: - for runtime_name, runtime_info in SUPPORTED_RUNTIMES.items(): - if device in supported_devices_list(runtime_info["supported_devices"]): - DEVICE_RUNTIME_MAP[device].append(runtime_name) + return supported_devices, supported_runtimes, device_runtime_map def apply_default_runtime(device: str, runtime: Optional[str] = None): + _, _, device_runtime_map = runtime_plugins() + if runtime is None: - return DEVICE_RUNTIME_MAP[str(device)][DEFAULT_RUNTIME] + return device_runtime_map[str(device)][DEFAULT_RUNTIME] else: return runtime @@ -84,26 +84,28 @@ def _check_suggestion(value: str): def select_runtime(device: str, runtime: Optional[str]) -> Tuple[str, str, Sequence]: + supported_devices, supported_runtimes, device_runtime_map = runtime_plugins() + # Convert to str in case its an instance of Device device_str = str(device) selected_runtime = apply_default_runtime(device_str, runtime) # Validate device and runtime selections - if device_str not in SUPPORTED_DEVICES: + if device_str not in supported_devices: raise exp.ArgError( f"Device argument '{device_str}' is not one of the available " - f"supported devices {SUPPORTED_DEVICES}\n" + f"supported devices {supported_devices}\n" f"{_check_suggestion(device_str)}" ) - if selected_runtime not in DEVICE_RUNTIME_MAP[device_str]: + if selected_runtime not in device_runtime_map[device_str]: raise exp.ArgError( f"Runtime argument '{selected_runtime}' is not one of the available " - f"runtimes supported for device '{device_str}': {DEVICE_RUNTIME_MAP[device_str]}\n" + f"runtimes supported for device '{device_str}': {device_runtime_map[device_str]}\n" f"{_check_suggestion(selected_runtime)}" ) # Get the plugin module for the selected runtime - runtime_info = SUPPORTED_RUNTIMES[selected_runtime] + runtime_info = supported_runtimes[selected_runtime] return selected_runtime, runtime_info diff --git a/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/__init__.py b/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/__init__.py index d163fbd..e69de29 100644 --- a/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/__init__.py +++ b/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/__init__.py @@ -1,10 +0,0 @@ -from .runtime import OnnxRT - -implements = { - "runtimes": { - "ort": { - "RuntimeClass": OnnxRT, - "supported_devices": {"x86"}, - } - } -} diff --git a/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/execute.py b/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/execute.py index ba946a9..794adb1 100644 --- a/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/execute.py +++ b/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/execute.py @@ -10,7 +10,7 @@ from statistics import mean import platform import onnxruntime -import turnkeyml.run.plugin_helpers as plugin_helpers +import turnkeyml_plugin_devices.common.run.plugin_helpers as plugin_helpers # Use the same ORT version of the base environment in the new conda env ORT_VERSION = onnxruntime.__version__ diff --git a/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/runtime.py b/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/runtime.py index f3782e6..c570a87 100644 --- a/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/runtime.py +++ b/plugins/devices/src/turnkeyml_plugin_devices/onnxrt/runtime.py @@ -1,16 +1,18 @@ import platform import os import numpy as np -from turnkeyml.run.basert import BaseRT + import turnkeyml.common.exceptions as exp -from turnkeyml.run.onnxrt.execute import ORT_VERSION from turnkeyml.common.filesystem import Stats -from turnkeyml.run.onnxrt.execute import ( + +from turnkeyml_plugin_devices.common.run.basert import BaseRT +from turnkeyml_plugin_devices.onnxrt.execute import ORT_VERSION +from turnkeyml_plugin_devices.onnxrt.execute import ( create_conda_env, execute_benchmark, get_cpu_specs, ) -import turnkeyml.run.plugin_helpers as plugin_helpers +import turnkeyml_plugin_devices.common.run.plugin_helpers as plugin_helpers class OnnxRT(BaseRT): diff --git a/plugins/devices/src/turnkeyml_plugin_devices/tensorrt/__init__.py b/plugins/devices/src/turnkeyml_plugin_devices/tensorrt/__init__.py index bbdb447..e69de29 100644 --- a/plugins/devices/src/turnkeyml_plugin_devices/tensorrt/__init__.py +++ b/plugins/devices/src/turnkeyml_plugin_devices/tensorrt/__init__.py @@ -1,11 +0,0 @@ -from .runtime import TensorRT - - -implements = { - "runtimes": { - "trt": { - "RuntimeClass": TensorRT, - "supported_devices": {"nvidia"}, - } - } -} diff --git a/plugins/devices/src/turnkeyml_plugin_devices/tensorrt/runtime.py b/plugins/devices/src/turnkeyml_plugin_devices/tensorrt/runtime.py index e99c18d..e334c23 100644 --- a/plugins/devices/src/turnkeyml_plugin_devices/tensorrt/runtime.py +++ b/plugins/devices/src/turnkeyml_plugin_devices/tensorrt/runtime.py @@ -3,11 +3,13 @@ import threading import json import numpy as np -from turnkeyml.run.tensorrt.execute import TRT_VERSION -from turnkeyml.run.basert import BaseRT + import turnkeyml.common.exceptions as exp from turnkeyml.common.filesystem import Stats -from turnkeyml.run.tensorrt.execute import ( + +from turnkeyml_plugin_devices.tensorrt.execute import TRT_VERSION +from turnkeyml_plugin_devices.common.run.basert import BaseRT +from turnkeyml_plugin_devices.tensorrt.execute import ( measure_power, run, average_power_and_utilization, diff --git a/plugins/devices/src/turnkeyml_plugin_devices/torchrt/__init__.py b/plugins/devices/src/turnkeyml_plugin_devices/torchrt/__init__.py index c6e746d..e69de29 100644 --- a/plugins/devices/src/turnkeyml_plugin_devices/torchrt/__init__.py +++ b/plugins/devices/src/turnkeyml_plugin_devices/torchrt/__init__.py @@ -1,14 +0,0 @@ -from .runtime import TorchRT - -implements = { - "runtimes": { - "torch-eager": { - "RuntimeClass": TorchRT, - "supported_devices": {"x86"}, - }, - "torch-compiled": { - "RuntimeClass": TorchRT, - "supported_devices": {"x86"}, - }, - } -} diff --git a/plugins/devices/src/turnkeyml_plugin_devices/torchrt/runtime.py b/plugins/devices/src/turnkeyml_plugin_devices/torchrt/runtime.py index 7673194..ec38401 100644 --- a/plugins/devices/src/turnkeyml_plugin_devices/torchrt/runtime.py +++ b/plugins/devices/src/turnkeyml_plugin_devices/torchrt/runtime.py @@ -6,9 +6,9 @@ from packaging import version import torch import numpy as np -from turnkeyml.run.basert import BaseRT -from turnkeyml.run.performance import MeasuredPerformance -from turnkeyml.run.onnxrt.execute import get_cpu_specs +from turnkeyml_plugin_devices.common.run.basert import BaseRT +from turnkeyml_plugin_devices.common.run.performance import MeasuredPerformance +from turnkeyml_plugin_devices.onnxrt.execute import get_cpu_specs import turnkeyml.common.build as build import turnkeyml.common.exceptions as exp import turnkeyml.common.filesystem as fs From 097bd287b763a8c22630407ad1a5d3f0d89b4361 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Mon, 26 Aug 2024 14:03:59 -0400 Subject: [PATCH 09/22] Add new files --- .github/workflows/test_devices_plugin.yml | 57 ++++++ plugins/devices/setup.py | 53 +++++ .../src/turnkeyml_plugin_devices/__init__.py | 124 ++++++++++++ .../common/__init__.py | 0 .../turnkeyml_plugin_devices/common/util.py | 184 ++++++++++++++++++ 5 files changed, 418 insertions(+) create mode 100644 .github/workflows/test_devices_plugin.yml create mode 100644 plugins/devices/setup.py create mode 100644 plugins/devices/src/turnkeyml_plugin_devices/__init__.py create mode 100644 plugins/devices/src/turnkeyml_plugin_devices/common/__init__.py create mode 100644 plugins/devices/src/turnkeyml_plugin_devices/common/util.py diff --git a/.github/workflows/test_devices_plugin.yml b/.github/workflows/test_devices_plugin.yml new file mode 100644 index 0000000..565c51c --- /dev/null +++ b/.github/workflows/test_devices_plugin.yml @@ -0,0 +1,57 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python + +name: Lint and Test Devices Plugin + +on: + push: + branches: ["main"] + paths: + - plugins/devices/src/turnkeyml_plugin_devices/common/** + - plugins/devices/setup.py + - .github/workflows/test_devices_plugin.yml + pull_request: + branches: ["main"] + paths: + - plugins/devices/src/turnkeyml_plugin_devices/common/** + - plugins/devices/setup.py + - .github/workflows/test_devices_plugin.yml + +permissions: + contents: read + +jobs: + build-turnkey: + env: + TURNKEY_TRACEBACK: True + strategy: + matrix: + python-version: ["3.8", "3.11"] + os: [ubuntu-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - name: Set up Miniconda with 64-bit Python + uses: conda-incubator/setup-miniconda@v3 + with: + miniconda-version: "latest" + activate-environment: tkml + python-version: ${{ matrix.python-version }} + - name: Install dependencies + shell: bash -el {0} + run: | + python -m pip install --upgrade pip + conda install pylint + pip install pytest + pip install -e plugins/devices + pip install transformers timm + python -m pip check + - name: Lint with PyLint + shell: bash -el {0} + run: | + pylint plugins/devices/src/turnkeyml_plugin_devices/common --rcfile .pylintrc --disable E0401,E0203 + - name: Test with unittest + shell: bash -el {0} + run: | + python plugins/devices/test/benchmark.py + \ No newline at end of file diff --git a/plugins/devices/setup.py b/plugins/devices/setup.py new file mode 100644 index 0000000..6b8e203 --- /dev/null +++ b/plugins/devices/setup.py @@ -0,0 +1,53 @@ +import os +import json +import pathlib +from setuptools import setup + + +def get_specific_version(plugin_name: str, version_key: str) -> str: + with open( + os.path.join( + pathlib.Path(__file__).parent, + "src", + "turnkeyml_plugin_devices", + plugin_name, + "version.json", + ), + "r", + encoding="utf-8", + ) as file: + return json.load(file)[version_key] + + +setup( + name="turnkeyml_plugin_devices", + version="0.0.2", + package_dir={"": "src"}, + packages=[ + "turnkeyml_plugin_devices", + "turnkeyml_plugin_devices.common", + "turnkeyml_plugin_devices.common.run", + "turnkeyml_plugin_devices.onnxrt", + "turnkeyml_plugin_devices.tensorrt", + "turnkeyml_plugin_devices.torchrt", + ], + python_requires=">=3.8, <3.11", + install_requires=[ + # "turnkeyml==3.0.8", + "importlib_metadata", + "onnx_tool", + "numpy<2", + "gitpython", + "timm==0.9.10", + ], + include_package_data=True, + package_data={ + "turnkeyml_plugin_devices": [ + ] + }, + extras_require={ + "onnxrt": [], + "torchrt": [], + "tensorrt": [], + }, +) diff --git a/plugins/devices/src/turnkeyml_plugin_devices/__init__.py b/plugins/devices/src/turnkeyml_plugin_devices/__init__.py new file mode 100644 index 0000000..c8d6680 --- /dev/null +++ b/plugins/devices/src/turnkeyml_plugin_devices/__init__.py @@ -0,0 +1,124 @@ +import importlib.metadata +import os +from packaging.version import Version +from packaging.requirements import Requirement + +from .common.run.benchmark_model import Benchmark + + +# Determine which plugins are available to the user, based on whether +# that plugin's requirements are satisfied in the current environment +# +# We do this by the following method: +# 1. Get the list of extras (plugins) supported by this package +# 2. Get the specific requirements for each plugin +# 3. Check whether all specific requirements for a given plugin are installed +# in the current environment +# +# NOTE: users can set environment variable TURNKEY_PLUGIN_HELP to get +# information about which plugins are properly installed, and the +# reason why the other plugins are not considered to be properly +# installed. + +dist = importlib.metadata.distribution("turnkeyml_plugin_devices") +plugins_supported = dist.metadata.get_all("Provides-Extra") +dist_requirements = dist.metadata.get_all("Requires-Dist") +plugin_requirements = {plugin_name: [] for plugin_name in plugins_supported} + +if os.getenv("TURNKEY_PLUGIN_HELP") == "True": + print( + "Plugins supported by this package, when their requirements are installed:", + plugins_supported, + ) + print( + "Package requirements, including a mapping of requirements to plugins (extras):", + dist_requirements, + ) + + +for req in dist_requirements: + req_split = req.split("; extra == ") + if len(req_split) > 1: + plugin_name = req_split[1].replace('"', "") + plugin_name = plugin_name.replace("'", "") + plugin_requirements[plugin_name].append(req_split[0]) + +if os.getenv("TURNKEY_PLUGIN_HELP") == "True": + print("Requirements for each plugin:", plugin_requirements) + + +def plugin_installed(plugin_name, plugin_reqs) -> bool: + install_help = f"{plugin_name} is installed" + installed = True + for req in plugin_reqs: + try: + req = Requirement(req) + pkg, ver = req.name, req.specifier + installed_ver = Version(importlib.metadata.version(pkg)) + + if installed_ver not in ver: + installed = False + install_help = ( + f"{plugin_name} is not installed because, while {pkg} is installed, " + f"it has version {installed_ver} which does not match required version {ver}" + ) + except importlib.metadata.PackageNotFoundError: + installed = False + install_help = ( + f"{plugin_name} is not installed because " + f"requirement {pkg}{ver} is not installed." + ) + break + + if os.getenv("TURNKEY_PLUGIN_HELP") == "True": + print(install_help) + + return installed + + +installed_plugins = [ + name + for name in plugins_supported + if plugin_installed(name, plugin_requirements[name]) +] + +# Collect the total set of runtimes and tools installed by this plugin, +# given the available requirements in the environment, starting with +# those that are always available (e.g., Benchmark) +# Then, get the specific runtimes and tools made available by each of +# the installed plugins detected in the code block above +runtimes = {} +tools = [Benchmark] + +if "onnxrt" in installed_plugins: + from .onnxrt.runtime import OnnxRT + + runtimes["ort"] = { + "RuntimeClass": OnnxRT, + "supported_devices": {"x86"}, + } + +if "torchrt" in installed_plugins: + from .torchrt.runtime import TorchRT + + runtimes["torch-eager"] = { + "RuntimeClass": TorchRT, + "supported_devices": {"x86"}, + } + runtimes["torch-compiled"] = { + "RuntimeClass": TorchRT, + "supported_devices": {"x86"}, + } + +if "tensorrt" in installed_plugins: + from .tensorrt.runtime import TensorRT + + runtimes["trt"] = { + "RuntimeClass": TensorRT, + "supported_devices": {"nvidia"}, + } + +implements = { + "runtimes": runtimes, + "tools": tools, +} diff --git a/plugins/devices/src/turnkeyml_plugin_devices/common/__init__.py b/plugins/devices/src/turnkeyml_plugin_devices/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/plugins/devices/src/turnkeyml_plugin_devices/common/util.py b/plugins/devices/src/turnkeyml_plugin_devices/common/util.py new file mode 100644 index 0000000..5bbe14f --- /dev/null +++ b/plugins/devices/src/turnkeyml_plugin_devices/common/util.py @@ -0,0 +1,184 @@ +import json +import os +import subprocess +import pkgutil +import platform +import shutil +from typing import Tuple +import posixpath +import git + +import turnkeyml.common.exceptions as exp +from turnkeyml.common.printing import log_info + +import turnkeyml_plugin_devices.common.run.plugin_helpers as plugin_helpers + +PLUGIN_DIR = os.path.dirname(pkgutil.get_loader("turnkeyml_plugin_devices").path) + + +def version_path(plugin_name: str): + return os.path.join(PLUGIN_DIR, plugin_name, "version.json") + + +def get_versions(plugin_name: str): + """Get correct version number for runtime and dependencies""" + + path = version_path(plugin_name) + + with open(path, "r", encoding="utf-8") as file: + return json.load(file) + + +def get_version_key(plugin_name: str, version_key: str) -> str: + with open(version_path(plugin_name), "r", encoding="utf-8") as file: + versions = json.load(file) + + return versions[version_key] + + +def get_runtime_version(plugin_name: str) -> str: + return get_version_key(plugin_name, "runtime_version") + + +def get_deps_dir(plugin_name: str): + return os.path.join(PLUGIN_DIR, plugin_name, "deps") + + +def conda_and_env_path(conda_env_name: str) -> Tuple[str, str]: + conda_path = os.getenv("CONDA_EXE") + if conda_path is None: + raise EnvironmentError( + "CONDA_EXE environment variable not set." + "Make sure Conda is properly installed." + ) + + # Normalize the path for Windows + if platform.system() == "Windows": + conda_path = os.path.normpath(conda_path) + + env_path = os.path.join( + os.path.dirname(os.path.dirname(conda_path)), "envs", conda_env_name + ) + + return conda_path, env_path + + +def get_env_version_path(env_path) -> str: + return os.path.join(env_path, "version.json") + + +def env_up_to_date(plugin_name: str, env_path: str) -> bool: + + # Check if the local plugin version matches the conda environment version and part + env_version_path = get_env_version_path(env_path) + local_version = get_runtime_version(plugin_name) + if os.path.exists(env_version_path): + with open(env_version_path, "r", encoding="utf-8") as file: + env_versions = json.load(file) + if local_version == env_versions["runtime_version"]: + return True + + return False + + +def write_env_version(plugin_name: str, env_path: str): + # Attach a version to the conda env created + shutil.copy(version_path(plugin_name), get_env_version_path(env_path)) + + +def lfs_pull(plugin_name: str): + dependencies_dir = get_deps_dir(plugin_name) + + # For developers in editable mode, fetch the deps folder from git lfs + # NOTE: for non-editable-mode (ie, `pip install` with the `-e`): + # `git lfs pull -I PLUGIN_PATH` needs to be manually called prior to `pip install` + deps_size_bytes = sum( + os.path.getsize(os.path.join(dependencies_dir, f)) + for f in os.listdir(dependencies_dir) + if os.path.isfile(os.path.join(dependencies_dir, f)) + ) + + # LFS files have not been pulled if the deps folder is under 100KB + # because the folder will only have LFS pointers in it, which are about 100B each + plugin_path = posixpath.join( + "plugins", + "devices", + "src", + "turnkeyml_plugin_devices", + plugin_name, + ) + lfs_command = f'git lfs pull -I {plugin_path} -X " "' + + deps_folder_too_small_size_bytes = 100000 + # Only attempt this in a git repo; if this code is running somewhere else + # like site-packages then this will be skipped + with open(os.devnull, "w", encoding="utf-8") as d: + is_git_repo = not bool( + subprocess.call("git rev-parse", shell=True, stdout=d, stderr=d) + ) + + if is_git_repo and deps_size_bytes < deps_folder_too_small_size_bytes: + # Always run the LFS command from the git repo root + git_repo = git.Repo(__file__, search_parent_directories=True) + lfs_cwd = git_repo.git.rev_parse("--show-toplevel") + + print("Running:", lfs_command) + print("With cwd:", lfs_cwd) + subprocess.run( + lfs_command, + shell=True, + # check=False because the next code block will raise a more helpful + # exception if this subprocess doesn't work out + check=False, + cwd=lfs_cwd, + ) + + # If the deps size didn't change after the pull, that means the pull + # silently failed. Raise a helpful exception. + deps_size_bytes_post_pull = sum( + os.path.getsize(os.path.join(dependencies_dir, f)) + for f in os.listdir(dependencies_dir) + if os.path.isfile(os.path.join(dependencies_dir, f)) + ) + if deps_size_bytes_post_pull < deps_folder_too_small_size_bytes: + raise exp.EnvError( + "The vitisep dependencies have not been pulled from LFS " + "If you are building from source you can try running this command: " + f"`{lfs_command}`" + ) + + +def create_fresh_conda_env(conda_env_name: str, python_version: str, requirements=None): + conda_path, env_path = conda_and_env_path(conda_env_name) + + # Create new environment from scratch + log_info("Updating environment...") + if os.path.exists(env_path): + plugin_helpers.run_subprocess( + [ + conda_path, + "remove", + "--name", + conda_env_name, + "--all", + "-y", + ] + ) + + plugin_helpers.run_subprocess( + [ + conda_path, + "create", + "--name", + conda_env_name, + f"python={python_version}", + "-y", + ] + ) + + # Install requirements in the created environment + if requirements is not None: + for req in requirements: + plugin_helpers.run_subprocess( + [conda_path, "run", "--name", conda_env_name, "pip", "install", req] + ) From e9bbe9fbbc543ac9ed3ef9a6c6c76ddda0b4f260 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Mon, 26 Aug 2024 14:09:12 -0400 Subject: [PATCH 10/22] fixes --- plugins/devices/setup.py | 2 +- setup.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/plugins/devices/setup.py b/plugins/devices/setup.py index 6b8e203..72ac9c6 100644 --- a/plugins/devices/setup.py +++ b/plugins/devices/setup.py @@ -33,7 +33,7 @@ def get_specific_version(plugin_name: str, version_key: str) -> str: ], python_requires=">=3.8, <3.11", install_requires=[ - # "turnkeyml==3.0.8", + "turnkeyml==3.0.8", "importlib_metadata", "onnx_tool", "numpy<2", diff --git a/setup.py b/setup.py index 27c8399..87857ef 100644 --- a/setup.py +++ b/setup.py @@ -15,10 +15,6 @@ "turnkeyml.tools", "turnkeyml.tools.discovery", "turnkeyml.sequence", - "turnkeyml.run", - "turnkeyml.run.onnxrt", - "turnkeyml.run.tensorrt", - "turnkeyml.run.torchrt", "turnkeyml.cli", "turnkeyml.common", "turnkeyml_models", From 1db4701e1f9d10a000627b3c6664915a35eafce3 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Mon, 26 Aug 2024 14:53:39 -0400 Subject: [PATCH 11/22] fixes --- src/turnkeyml/common/test_helpers.py | 8 +------- src/turnkeyml/sequence/tool_plugins.py | 8 -------- test/cli.py | 4 ++-- 3 files changed, 3 insertions(+), 17 deletions(-) diff --git a/src/turnkeyml/common/test_helpers.py b/src/turnkeyml/common/test_helpers.py index 816bdc5..929920f 100644 --- a/src/turnkeyml/common/test_helpers.py +++ b/src/turnkeyml/common/test_helpers.py @@ -1,6 +1,6 @@ import os import shutil -from typing import List, Tuple, Any, Dict, Optional +from typing import List, Dict, Optional import onnx import turnkeyml.common.filesystem as fs import turnkeyml.common.build as build @@ -224,7 +224,6 @@ def get_stats_and_state( def assert_success_of_builds( test_script_files: List[str], cache_dir: str, - info_property: Tuple[str, Any] = None, check_perf: bool = False, check_opset: Optional[int] = None, check_iteration_count: Optional[int] = None, @@ -250,11 +249,6 @@ def assert_success_of_builds( script_build_found = True builds_found += 1 - if info_property is not None: - assert ( - build_state.info.__dict__[info_property[0]] == info_property[1] - ), f"{build_state.info.__dict__[info_property[0]]} == {info_property[1]}" - if check_perf: assert stats.stats["mean_latency"] > 0 assert stats.stats["throughput"] > 0 diff --git a/src/turnkeyml/sequence/tool_plugins.py b/src/turnkeyml/sequence/tool_plugins.py index 161c469..0553e60 100644 --- a/src/turnkeyml/sequence/tool_plugins.py +++ b/src/turnkeyml/sequence/tool_plugins.py @@ -42,12 +42,4 @@ def get_supported_tools(): supported_tools.append(tool_class) - # Give a "benchmark" tool installed by a plugin priority over - # a "benchmark" tool built into turnkeyml - tool_names = [tool.unique_name for tool in supported_tools] - if "benchmark" not in tool_names: - from turnkeyml.run.benchmark_model import Benchmark - - supported_tools.append(Benchmark) - return supported_tools diff --git a/test/cli.py b/test/cli.py index c662f15..463e733 100644 --- a/test/cli.py +++ b/test/cli.py @@ -372,7 +372,7 @@ def test_08_cli_onnx_opset(self): turnkeycli() assert_success_of_builds( - [test_script], cache_dir, None, check_perf=False, check_opset=user_opset + [test_script], cache_dir, check_perf=False, check_opset=user_opset ) def test_09_cli_process_isolation(self): @@ -396,7 +396,7 @@ def test_09_cli_process_isolation(self): with patch.object(sys, "argv", testargs): turnkeycli() - assert_success_of_builds([test_script], cache_dir, None, check_perf=False) + assert_success_of_builds([test_script], cache_dir, check_perf=False) @unittest.skipIf( platform.system() == "Windows", From 3b1290e9b4c9acdc549e502d967fad98ddc5d869 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Mon, 26 Aug 2024 15:13:56 -0400 Subject: [PATCH 12/22] fixes --- .github/workflows/test_devices_plugin.yml | 4 ++-- plugins/devices/setup.py | 2 +- plugins/devices/test/benchmark.py | 5 ++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test_devices_plugin.yml b/.github/workflows/test_devices_plugin.yml index 565c51c..5c5c433 100644 --- a/.github/workflows/test_devices_plugin.yml +++ b/.github/workflows/test_devices_plugin.yml @@ -21,7 +21,7 @@ permissions: contents: read jobs: - build-turnkey: + build-devices-plugin: env: TURNKEY_TRACEBACK: True strategy: @@ -49,7 +49,7 @@ jobs: - name: Lint with PyLint shell: bash -el {0} run: | - pylint plugins/devices/src/turnkeyml_plugin_devices/common --rcfile .pylintrc --disable E0401,E0203 + pylint plugins/devices/src --rcfile .pylintrc --disable E0401,E0203 - name: Test with unittest shell: bash -el {0} run: | diff --git a/plugins/devices/setup.py b/plugins/devices/setup.py index 72ac9c6..2d977af 100644 --- a/plugins/devices/setup.py +++ b/plugins/devices/setup.py @@ -31,7 +31,7 @@ def get_specific_version(plugin_name: str, version_key: str) -> str: "turnkeyml_plugin_devices.tensorrt", "turnkeyml_plugin_devices.torchrt", ], - python_requires=">=3.8, <3.11", + python_requires=">=3.8, <3.12", install_requires=[ "turnkeyml==3.0.8", "importlib_metadata", diff --git a/plugins/devices/test/benchmark.py b/plugins/devices/test/benchmark.py index 5f0f63a..04a91ef 100644 --- a/plugins/devices/test/benchmark.py +++ b/plugins/devices/test/benchmark.py @@ -87,7 +87,7 @@ def test_001_cli_benchmark(self): with patch.object(sys, "argv", testargs): turnkeycli() - assert_success_of_builds([test_script], cache_dir, None, check_perf=True) + assert_success_of_builds([test_script], cache_dir, check_perf=True) def test_002_runtimes(self): # Attempt to benchmark using an invalid runtime @@ -170,7 +170,6 @@ def test_003_cli_iteration_count(self): assert_success_of_builds( [test_script], cache_dir, - None, check_perf=True, check_iteration_count=test_iterations, ) @@ -197,7 +196,7 @@ def test_004_cli_process_isolation(self): with patch.object(sys, "argv", testargs): turnkeycli() - assert_success_of_builds([test_script], cache_dir, None, check_perf=True) + assert_success_of_builds([test_script], cache_dir, check_perf=True) def test_005_cli_export_only(self): # Test the first model in the corpus From fe6d00198c089408a094a826df04f963d5e8197c Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Mon, 26 Aug 2024 15:23:48 -0400 Subject: [PATCH 13/22] try it this way --- .github/workflows/test_devices_plugin.yml | 1 + plugins/devices/test/unit.py | 97 +++++++++++++++++++++++ test/unit.py | 87 +------------------- 3 files changed, 100 insertions(+), 85 deletions(-) create mode 100644 plugins/devices/test/unit.py diff --git a/.github/workflows/test_devices_plugin.yml b/.github/workflows/test_devices_plugin.yml index 5c5c433..4011159 100644 --- a/.github/workflows/test_devices_plugin.yml +++ b/.github/workflows/test_devices_plugin.yml @@ -53,5 +53,6 @@ jobs: - name: Test with unittest shell: bash -el {0} run: | + python plugins/devices/test/unit.py python plugins/devices/test/benchmark.py \ No newline at end of file diff --git a/plugins/devices/test/unit.py b/plugins/devices/test/unit.py new file mode 100644 index 0000000..48ef556 --- /dev/null +++ b/plugins/devices/test/unit.py @@ -0,0 +1,97 @@ +""" +Miscellaneous unit tests +""" + +import unittest +import os +import turnkeyml.common.build as build +import turnkeyml_plugin_devices.common.run.performance as performance +import turnkeyml_plugin_devices.common.run.plugin_helpers as plugin_helpers + + +class Testing(unittest.TestCase): + + def test_000_device_class(self): + family = "family" + part = "part" + config = "config" + device_str = f"{family}::{part}::{config}" + my_device = performance.Device(device_str) + + assert my_device.family == family + assert my_device.part == part + assert my_device.config == config + assert str(my_device) == device_str + + def test_001_subprocess_logger(self): + """ + Ensure the subprocess logger stores both stdout and stderr and is also fail-safe + """ + + # Initialize messages and commands used in the test + logfile_path = "unit_log_subprocess.txt" + inside_stdout_msg = "This stdout should be inside the log file" + inside_sterr_msg = "This stderr should be inside the log file" + outside_stdout_msg = "This stdout should be outside the log file" + outside_stderr_msg = "This stderr should be outside the log file" + traceback_error_msg = "Tracebacks should be inside the log file" + inside_stdout_cmd = f"print('{inside_stdout_msg}',file=sys.stdout)" + inside_sterr_cmd = f"print('{inside_sterr_msg}',file=sys.stderr)" + traceback_error_cmd = f"raise ValueError('{traceback_error_msg}')" + + # Perform basic test (no exceptions inside logger) + cmd = ["python", "-c", f"import sys\n{inside_stdout_cmd}\n{inside_sterr_cmd}"] + plugin_helpers.logged_subprocess(cmd=cmd, log_file_path=logfile_path) + + # Make sure we captured everything we intended to capture + with open(logfile_path, "r", encoding="utf-8") as file: + log_contents = file.read() + assert inside_stdout_msg in log_contents + assert inside_sterr_msg in log_contents + + # Perform test with exceptions inside the logger + cmd = [ + "python", + "-c", + f"import sys\n{inside_stdout_cmd}\n{inside_sterr_cmd}\n{traceback_error_cmd}", + ] + with self.assertRaises(plugin_helpers.CondaError): + plugin_helpers.logged_subprocess(cmd=cmd, log_file_path=logfile_path) + + # Make sure we captured everything we intended to capture + with open(logfile_path, "r", encoding="utf-8") as file: + log_contents = file.read() + assert inside_stdout_msg in log_contents + assert inside_sterr_msg in log_contents + assert traceback_error_msg in log_contents + + # Ensure subprocess correctly receives the environment + subprocess_env = os.environ.copy() + expected_env_var_value = "Expected Value" + subprocess_env["TEST_ENV_VAR"] = expected_env_var_value + cmd = ["python", "-c", f'import os\nprint(os.environ["TEST_ENV_VAR"])'] + plugin_helpers.logged_subprocess( + cmd=cmd, log_file_path=logfile_path, env=subprocess_env + ) + with open(logfile_path, "r", encoding="utf-8") as file: + log_contents = file.read() + assert expected_env_var_value in log_contents + + # Test log_to_std_streams + cmd = [ + "python", + "-c", + f'print("{outside_stdout_msg}")\nprint("{outside_stderr_msg}")', + ] + with build.Logger("", logfile_path): + plugin_helpers.logged_subprocess( + cmd=cmd, log_to_std_streams=True, log_to_file=False + ) + with open(logfile_path, "r", encoding="utf-8") as file: + log_contents = file.read() + assert outside_stdout_msg in log_contents + assert outside_stderr_msg in log_contents + + +if __name__ == "__main__": + unittest.main() diff --git a/test/unit.py b/test/unit.py index 2fd2f1f..dde0e9f 100644 --- a/test/unit.py +++ b/test/unit.py @@ -7,8 +7,6 @@ import sys import turnkeyml.common.filesystem as filesystem import turnkeyml.common.build as build -import turnkeyml.run.performance as performance -import turnkeyml.run.plugin_helpers as plugin_helpers from turnkeyml.cli.parser_helpers import decode_args, encode_args @@ -27,19 +25,7 @@ def test_000_models_dir(self): assert "transformers" in models assert "readme.md" in models - def test_001_device_class(self): - family = "family" - part = "part" - config = "config" - device_str = f"{family}::{part}::{config}" - my_device = performance.Device(device_str) - - assert my_device.family == family - assert my_device.part == part - assert my_device.config == config - assert str(my_device) == device_str - - def test_002_logger(self): + def test_001_logger(self): """ Ensure the logger stores both stdout and stderr and is also fail-safe """ @@ -85,76 +71,7 @@ def test_002_logger(self): assert outside_stdout_msg not in log_contents assert outside_stderr_msg not in log_contents - def test_003_subprocess_logger(self): - """ - Ensure the subprocess logger stores both stdout and stderr and is also fail-safe - """ - - # Initialize messages and commands used in the test - logfile_path = "unit_log_subprocess.txt" - inside_stdout_msg = "This stdout should be inside the log file" - inside_sterr_msg = "This stderr should be inside the log file" - outside_stdout_msg = "This stdout should be outside the log file" - outside_stderr_msg = "This stderr should be outside the log file" - traceback_error_msg = "Tracebacks should be inside the log file" - inside_stdout_cmd = f"print('{inside_stdout_msg}',file=sys.stdout)" - inside_sterr_cmd = f"print('{inside_sterr_msg}',file=sys.stderr)" - traceback_error_cmd = f"raise ValueError('{traceback_error_msg}')" - - # Perform basic test (no exceptions inside logger) - cmd = ["python", "-c", f"import sys\n{inside_stdout_cmd}\n{inside_sterr_cmd}"] - plugin_helpers.logged_subprocess(cmd=cmd, log_file_path=logfile_path) - - # Make sure we captured everything we intended to capture - with open(logfile_path, "r", encoding="utf-8") as file: - log_contents = file.read() - assert inside_stdout_msg in log_contents - assert inside_sterr_msg in log_contents - - # Perform test with exceptions inside the logger - cmd = [ - "python", - "-c", - f"import sys\n{inside_stdout_cmd}\n{inside_sterr_cmd}\n{traceback_error_cmd}", - ] - with self.assertRaises(plugin_helpers.CondaError): - plugin_helpers.logged_subprocess(cmd=cmd, log_file_path=logfile_path) - - # Make sure we captured everything we intended to capture - with open(logfile_path, "r", encoding="utf-8") as file: - log_contents = file.read() - assert inside_stdout_msg in log_contents - assert inside_sterr_msg in log_contents - assert traceback_error_msg in log_contents - - # Ensure subprocess correctly receives the environment - subprocess_env = os.environ.copy() - expected_env_var_value = "Expected Value" - subprocess_env["TEST_ENV_VAR"] = expected_env_var_value - cmd = ["python", "-c", f'import os\nprint(os.environ["TEST_ENV_VAR"])'] - plugin_helpers.logged_subprocess( - cmd=cmd, log_file_path=logfile_path, env=subprocess_env - ) - with open(logfile_path, "r", encoding="utf-8") as file: - log_contents = file.read() - assert expected_env_var_value in log_contents - - # Test log_to_std_streams - cmd = [ - "python", - "-c", - f'print("{outside_stdout_msg}")\nprint("{outside_stderr_msg}")', - ] - with build.Logger("", logfile_path): - plugin_helpers.logged_subprocess( - cmd=cmd, log_to_std_streams=True, log_to_file=False - ) - with open(logfile_path, "r", encoding="utf-8") as file: - log_contents = file.read() - assert outside_stdout_msg in log_contents - assert outside_stderr_msg in log_contents - - def test_021_args_encode_decode(self): + def test_002_args_encode_decode(self): """ Test the encoding and decoding of arguments that follow the ["arg1::[value1,value2]","arg2::value1","flag_arg"]' format From 95671c9d9f2801ccdeee5c937b42beda275d8116 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Mon, 26 Aug 2024 15:24:49 -0400 Subject: [PATCH 14/22] this too --- .github/workflows/test_devices_plugin.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test_devices_plugin.yml b/.github/workflows/test_devices_plugin.yml index 4011159..807051a 100644 --- a/.github/workflows/test_devices_plugin.yml +++ b/.github/workflows/test_devices_plugin.yml @@ -8,12 +8,18 @@ on: branches: ["main"] paths: - plugins/devices/src/turnkeyml_plugin_devices/common/** + - plugins/devices/src/turnkeyml_plugin_devices/onnxrt/** + - plugins/devices/src/turnkeyml_plugin_devices/torchrt/** + - plugins/devices/src/turnkeyml_plugin_devices/tensorrt/** - plugins/devices/setup.py - .github/workflows/test_devices_plugin.yml pull_request: branches: ["main"] paths: - plugins/devices/src/turnkeyml_plugin_devices/common/** + - plugins/devices/src/turnkeyml_plugin_devices/onnxrt/** + - plugins/devices/src/turnkeyml_plugin_devices/torchrt/** + - plugins/devices/src/turnkeyml_plugin_devices/tensorrt/** - plugins/devices/setup.py - .github/workflows/test_devices_plugin.yml From dd27936de2aa5b84795bbb87f9f70a4bacec6f89 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Mon, 26 Aug 2024 16:45:46 -0400 Subject: [PATCH 15/22] try this --- .../turnkeyml_plugin_example_combined/runtime.py | 6 +++--- .../example_rt/turnkeyml_plugin_example_rt/runtime.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py b/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py index 6439520..7e60039 100644 --- a/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py +++ b/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py @@ -3,12 +3,12 @@ from timeit import default_timer as timer import onnxruntime as ort import numpy as np -from turnkeyml.run.basert import BaseRT from turnkeyml.state import load_state import turnkeyml.common.exceptions as exp import turnkeyml.common.filesystem as fs -from turnkeyml.run.onnxrt.within_conda import dummy_inputs -from turnkeyml.run.performance import MeasuredPerformance +from turnkeyml_plugins_devices.common.run.basert import BaseRT +from turnkeyml_plugins_devices.common.run.onnxrt.within_conda import dummy_inputs +from turnkeyml_plugins_devices.common.run.performance import MeasuredPerformance combined_rt_name = "example-combined-rt" diff --git a/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py b/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py index ed01069..1863b57 100644 --- a/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py +++ b/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py @@ -1,7 +1,7 @@ import os import numpy as np -from turnkeyml.run.basert import BaseRT -from turnkeyml.run.performance import MeasuredPerformance +from turnkeyml_plugins_devices.common.run.basert import BaseRT +from turnkeyml_plugins_devices.common.run.performance import MeasuredPerformance import turnkeyml.common.exceptions as exp from turnkeyml.common.filesystem import Stats From 71f23405002d7e608cf547413aacd491ca56dd42 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Mon, 26 Aug 2024 16:59:53 -0400 Subject: [PATCH 16/22] fix lint --- .../turnkeyml_plugin_example_combined/runtime.py | 8 +++++--- .../example_rt/turnkeyml_plugin_example_rt/runtime.py | 2 ++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py b/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py index 7e60039..b5b4bb5 100644 --- a/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py +++ b/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py @@ -3,13 +3,15 @@ from timeit import default_timer as timer import onnxruntime as ort import numpy as np -from turnkeyml.state import load_state -import turnkeyml.common.exceptions as exp -import turnkeyml.common.filesystem as fs + from turnkeyml_plugins_devices.common.run.basert import BaseRT from turnkeyml_plugins_devices.common.run.onnxrt.within_conda import dummy_inputs from turnkeyml_plugins_devices.common.run.performance import MeasuredPerformance +from turnkeyml.state import load_state +import turnkeyml.common.exceptions as exp +import turnkeyml.common.filesystem as fs + combined_rt_name = "example-combined-rt" diff --git a/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py b/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py index 1863b57..a197739 100644 --- a/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py +++ b/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py @@ -1,7 +1,9 @@ import os import numpy as np + from turnkeyml_plugins_devices.common.run.basert import BaseRT from turnkeyml_plugins_devices.common.run.performance import MeasuredPerformance + import turnkeyml.common.exceptions as exp from turnkeyml.common.filesystem import Stats From 5d012c0598335d752006518127dd2a037da055f3 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Mon, 26 Aug 2024 17:05:01 -0400 Subject: [PATCH 17/22] fixes --- .github/workflows/publish-to-test-pypi.yml | 2 +- .github/workflows/test_devices_plugin.yml | 38 +++++++++---------- .../runtime.py | 6 +-- .../turnkeyml_plugin_example_rt/runtime.py | 4 +- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/publish-to-test-pypi.yml b/.github/workflows/publish-to-test-pypi.yml index 5defeac..eff6638 100644 --- a/.github/workflows/publish-to-test-pypi.yml +++ b/.github/workflows/publish-to-test-pypi.yml @@ -32,7 +32,7 @@ jobs: python -m pip install --upgrade pip pip install dist/*.whl models=$(turnkey models-location --quiet) - turnkey -i $models/selftest/linear.py discover export-pytorch benchmark + turnkey -i $models/selftest/linear.py discover export-pytorch - name: Publish distribution package to PyPI if: startsWith(github.ref, 'refs/tags/v') uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/test_devices_plugin.yml b/.github/workflows/test_devices_plugin.yml index 807051a..808d91b 100644 --- a/.github/workflows/test_devices_plugin.yml +++ b/.github/workflows/test_devices_plugin.yml @@ -3,25 +3,25 @@ name: Lint and Test Devices Plugin -on: - push: - branches: ["main"] - paths: - - plugins/devices/src/turnkeyml_plugin_devices/common/** - - plugins/devices/src/turnkeyml_plugin_devices/onnxrt/** - - plugins/devices/src/turnkeyml_plugin_devices/torchrt/** - - plugins/devices/src/turnkeyml_plugin_devices/tensorrt/** - - plugins/devices/setup.py - - .github/workflows/test_devices_plugin.yml - pull_request: - branches: ["main"] - paths: - - plugins/devices/src/turnkeyml_plugin_devices/common/** - - plugins/devices/src/turnkeyml_plugin_devices/onnxrt/** - - plugins/devices/src/turnkeyml_plugin_devices/torchrt/** - - plugins/devices/src/turnkeyml_plugin_devices/tensorrt/** - - plugins/devices/setup.py - - .github/workflows/test_devices_plugin.yml +# on: +# push: +# branches: ["main"] +# paths: +# - plugins/devices/src/turnkeyml_plugin_devices/common/** +# - plugins/devices/src/turnkeyml_plugin_devices/onnxrt/** +# - plugins/devices/src/turnkeyml_plugin_devices/torchrt/** +# - plugins/devices/src/turnkeyml_plugin_devices/tensorrt/** +# - plugins/devices/setup.py +# - .github/workflows/test_devices_plugin.yml +# pull_request: +# branches: ["main"] +# paths: +# - plugins/devices/src/turnkeyml_plugin_devices/common/** +# - plugins/devices/src/turnkeyml_plugin_devices/onnxrt/** +# - plugins/devices/src/turnkeyml_plugin_devices/torchrt/** +# - plugins/devices/src/turnkeyml_plugin_devices/tensorrt/** +# - plugins/devices/setup.py +# - .github/workflows/test_devices_plugin.yml permissions: contents: read diff --git a/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py b/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py index b5b4bb5..28df26b 100644 --- a/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py +++ b/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py @@ -4,9 +4,9 @@ import onnxruntime as ort import numpy as np -from turnkeyml_plugins_devices.common.run.basert import BaseRT -from turnkeyml_plugins_devices.common.run.onnxrt.within_conda import dummy_inputs -from turnkeyml_plugins_devices.common.run.performance import MeasuredPerformance +from turnkeyml_plugin_devices.common.run.basert import BaseRT +from turnkeyml_plugin_devices.common.run.onnxrt.within_conda import dummy_inputs +from turnkeyml_plugin_devices.common.run.performance import MeasuredPerformance from turnkeyml.state import load_state import turnkeyml.common.exceptions as exp diff --git a/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py b/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py index a197739..8dfd534 100644 --- a/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py +++ b/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py @@ -1,8 +1,8 @@ import os import numpy as np -from turnkeyml_plugins_devices.common.run.basert import BaseRT -from turnkeyml_plugins_devices.common.run.performance import MeasuredPerformance +from turnkeyml_plugin_devices.common.run.basert import BaseRT +from turnkeyml_plugin_devices.common.run.performance import MeasuredPerformance import turnkeyml.common.exceptions as exp from turnkeyml.common.filesystem import Stats From ab7c71afb374efc3bb773ed9fd4db1f25407e27b Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Mon, 26 Aug 2024 17:14:24 -0400 Subject: [PATCH 18/22] Get rid of example rt plguins --- .github/workflows/test_turnkey.yml | 17 --- .../cli/plugins/example_combined/setup.py | 8 -- .../__init__.py | 15 --- .../runtime.py | 118 ------------------ .../turnkeyml_plugin_example_combined/tool.py | 29 ----- examples/cli/plugins/example_rt/setup.py | 8 -- .../turnkeyml_plugin_example_rt/__init__.py | 13 -- .../turnkeyml_plugin_example_rt/runtime.py | 89 ------------- test/plugins.py | 59 --------- 9 files changed, 356 deletions(-) delete mode 100644 examples/cli/plugins/example_combined/setup.py delete mode 100644 examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/__init__.py delete mode 100644 examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py delete mode 100644 examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/tool.py delete mode 100644 examples/cli/plugins/example_rt/setup.py delete mode 100644 examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/__init__.py delete mode 100644 examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py delete mode 100644 test/plugins.py diff --git a/.github/workflows/test_turnkey.yml b/.github/workflows/test_turnkey.yml index 16a3175..12ebb06 100644 --- a/.github/workflows/test_turnkey.yml +++ b/.github/workflows/test_turnkey.yml @@ -74,26 +74,9 @@ jobs: - name: Test example plugins shell: bash -el {0} run: | - rm -rf ~/.cache/turnkey - pip install -e examples/cli/plugins/example_rt - turnkey -i examples/cli/scripts/hello_world.py discover export-pytorch benchmark --runtime example-rt - rm -rf ~/.cache/turnkey pip install -e examples/cli/plugins/example_tool turnkey -i examples/cli/scripts/hello_world.py discover export-pytorch example-plugin-tool benchmark - - rm -rf ~/.cache/turnkey - pip install -e examples/cli/plugins/example_combined - - turnkey -i examples/cli/scripts/hello_world.py discover export-pytorch combined-example-tool benchmark --runtime example-combined-rt --rt-args delay_before_benchmarking::5 - turnkey -i examples/cli/scripts/hello_world.py discover export-pytorch combined-example-tool benchmark --device example_family::part1::config2 - turnkey -i examples/cli/scripts/hello_world.py discover export-pytorch combined-example-tool benchmark --device example_family::part1::config1 - turnkey -i examples/cli/scripts/hello_world.py discover export-pytorch combined-example-tool benchmark --device example_family::part1 - turnkey -i examples/cli/scripts/hello_world.py discover export-pytorch combined-example-tool benchmark --device example_family - - # E2E tests - cd test - python plugins.py - name: Install and Start Slurm if: runner.os != 'Windows' shell: bash -el {0} diff --git a/examples/cli/plugins/example_combined/setup.py b/examples/cli/plugins/example_combined/setup.py deleted file mode 100644 index 3528f7c..0000000 --- a/examples/cli/plugins/example_combined/setup.py +++ /dev/null @@ -1,8 +0,0 @@ -from setuptools import setup - -setup( - name="turnkeyml_plugin_example_combined", - version="1.0.0", - packages=["turnkeyml_plugin_example_combined"], - python_requires=">=3.8, <3.12", -) diff --git a/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/__init__.py b/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/__init__.py deleted file mode 100644 index 65d66e1..0000000 --- a/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from .runtime import CombinedExampleRT, combined_rt_name -from .tool import CombinedExampleTool - -implements = { - "runtimes": { - combined_rt_name: { - "RuntimeClass": CombinedExampleRT, - "supported_devices": { - "x86": {}, - "example_family": {"part1": ["config1", "config2"]}, - }, - } - }, - "tools": [CombinedExampleTool], -} diff --git a/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py b/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py deleted file mode 100644 index 28df26b..0000000 --- a/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py +++ /dev/null @@ -1,118 +0,0 @@ -import os -import time -from timeit import default_timer as timer -import onnxruntime as ort -import numpy as np - -from turnkeyml_plugin_devices.common.run.basert import BaseRT -from turnkeyml_plugin_devices.common.run.onnxrt.within_conda import dummy_inputs -from turnkeyml_plugin_devices.common.run.performance import MeasuredPerformance - -from turnkeyml.state import load_state -import turnkeyml.common.exceptions as exp -import turnkeyml.common.filesystem as fs - - -combined_rt_name = "example-combined-rt" - - -class CombinedExampleRT(BaseRT): - def __init__( - self, - cache_dir: str, - build_name: str, - stats: fs.Stats, - iterations: int, - device_type: str, - runtime: str = combined_rt_name, - tensor_type=np.array, - model=None, - inputs=None, - delay_before_benchmarking: str = "0", - ): - # Custom runtime args always arrive as strings, so we need to convert them - # to the appropriate data type here - self.delay_before_benchmarking = int(delay_before_benchmarking) - - super().__init__( - cache_dir=cache_dir, - build_name=build_name, - stats=stats, - tensor_type=tensor_type, - device_type=device_type, - runtime=runtime, - iterations=iterations, - runtimes_supported=[combined_rt_name], - runtime_version="0.0.0", - base_path=os.path.dirname(__file__), - model=model, - inputs=inputs, - ) - - self.throughput_ips = None - self.mean_latency_ms = None - - def _setup(self): - # The BaseRT abstract base class requires us to overload this function, - # however our simple example runtime does not require any additional - # setup steps. - pass - - def benchmark(self): - state = load_state(self.cache_dir, self.build_name) - per_iteration_latency = [] - sess_options = ort.SessionOptions() - sess_options.graph_optimization_level = ( - ort.GraphOptimizationLevel.ORT_ENABLE_ALL - ) - onnx_session = ort.InferenceSession(state.results, sess_options) - sess_input = onnx_session.get_inputs() - input_feed = dummy_inputs(sess_input) - output_name = onnx_session.get_outputs()[0].name - - # Using custom runtime argument - print(f"Sleeping {self.delay_before_benchmarking}s before benchmarking") - time.sleep(self.delay_before_benchmarking) - - for _ in range(self.iterations): - start = timer() - onnx_session.run([output_name], input_feed) - end = timer() - iteration_latency = end - start - per_iteration_latency.append(iteration_latency) - - total_time = sum(per_iteration_latency) - self.throughput_ips = total_time / self.iterations - self.mean_latency_ms = 1 / self.throughput_ips - - return MeasuredPerformance( - mean_latency=self.mean_latency, - throughput=self.throughput, - device=self.device_name(), - device_type=self.device_type, - runtime=self.runtime, - runtime_version=self.runtime_version, - build_name=self.build_name, - ) - - @property - def mean_latency(self) -> float: - if self.mean_latency_ms is not None: - return self.mean_latency_ms - else: - raise exp.BenchmarkException( - "Queried mean latency before self.benchmark() was called" - ) - - @property - def throughput(self) -> float: - if self.throughput_ips is not None: - return self.throughput_ips - else: - raise exp.BenchmarkException( - "Queried throughput before self.benchmark() was called" - ) - - @staticmethod - def device_name() -> str: - return "Example Device" diff --git a/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/tool.py b/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/tool.py deleted file mode 100644 index aaa5cf3..0000000 --- a/examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/tool.py +++ /dev/null @@ -1,29 +0,0 @@ -import argparse -from turnkeyml.tools import Tool -from turnkeyml.state import State - - -class CombinedExampleTool(Tool): - """ - This is an empty Tool that we include in our example that provides both - a sequence and a runtime in a single plugin package. - """ - - unique_name = "combined-example-tool" - - def __init__(self): - super().__init__( - monitor_message="Special step expected by CombinedExampleRT", - ) - - @staticmethod - def parser(add_help: bool = True) -> argparse.ArgumentParser: - parser = __class__.helpful_parser( - short_description="This is an examples tool from the combined example", - add_help=add_help, - ) - - return parser - - def run(self, state: State): - return state diff --git a/examples/cli/plugins/example_rt/setup.py b/examples/cli/plugins/example_rt/setup.py deleted file mode 100644 index e29d690..0000000 --- a/examples/cli/plugins/example_rt/setup.py +++ /dev/null @@ -1,8 +0,0 @@ -from setuptools import setup - -setup( - name="turnkeyml_plugin_example_rt", - version="0.0.0", - packages=["turnkeyml_plugin_example_rt"], - python_requires=">=3.8, <3.12", -) diff --git a/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/__init__.py b/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/__init__.py deleted file mode 100644 index c3f8e18..0000000 --- a/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from .runtime import ExampleRT, example_rt_name - -implements = { - "runtimes": { - example_rt_name: { - "RuntimeClass": ExampleRT, - "supported_devices": {"x86"}, - # magic_perf_points and super_runtime_points are custom stats we will - # have to set in the ExampleRT. - "status_stats": ["magic_perf_points", "super_runtime_points"], - } - } -} diff --git a/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py b/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py deleted file mode 100644 index 8dfd534..0000000 --- a/examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py +++ /dev/null @@ -1,89 +0,0 @@ -import os -import numpy as np - -from turnkeyml_plugin_devices.common.run.basert import BaseRT -from turnkeyml_plugin_devices.common.run.performance import MeasuredPerformance - -import turnkeyml.common.exceptions as exp -from turnkeyml.common.filesystem import Stats - -example_rt_name = "example-rt" - - -class ExampleRT(BaseRT): - def __init__( - self, - cache_dir: str, - build_name: str, - stats: Stats, - iterations: int, - device_type: str, - runtime: str = example_rt_name, - tensor_type=np.array, - model=None, - inputs=None, - ): - self.throughput_ips = None - self.mean_latency_ms = None - - super().__init__( - cache_dir=cache_dir, - build_name=build_name, - stats=stats, - device_type=device_type, - runtime=runtime, - iterations=iterations, - runtimes_supported=[example_rt_name], - runtime_version="0.0.0", - base_path=os.path.dirname(__file__), - tensor_type=tensor_type, - model=model, - inputs=inputs, - ) - - def _setup(self): - # The BaseRT abstract base class requires us to overload this function, - # however our simple example runtime does not require any additional - # setup steps. - pass - - def benchmark(self) -> MeasuredPerformance: - self.throughput_ips = self.iterations - self.mean_latency_ms = 1 / self.iterations - - # Assign values to the stats that will be printed - # out by the CLI when status is reported - self.stats.save_stat("magic_perf_points", 42) - self.stats.save_stat("super_runtime_points", 100) - - return MeasuredPerformance( - mean_latency=self.mean_latency, - throughput=self.throughput, - device=self.device_name(), - device_type=self.device_type, - runtime=self.runtime, - runtime_version=self.runtime_version, - build_name=self.build_name, - ) - - @property - def mean_latency(self) -> float: - if self.mean_latency_ms is not None: - return self.mean_latency_ms - else: - raise exp.BenchmarkException( - "Queried mean latency before self.benchmark() was called" - ) - - @property - def throughput(self) -> float: - if self.throughput_ips is not None: - return self.throughput_ips - else: - raise exp.BenchmarkException( - "Queried throughput before self.benchmark() was called" - ) - - @staticmethod - def device_name() -> str: - return "the x86 cpu of your dreams" diff --git a/test/plugins.py b/test/plugins.py deleted file mode 100644 index ce63a3c..0000000 --- a/test/plugins.py +++ /dev/null @@ -1,59 +0,0 @@ -""" -Tests focused on TurnkeyML plugins -""" - -import os -import unittest -from unittest.mock import patch -import sys -from turnkeyml.cli.cli import main as turnkeycli -import turnkeyml.common.filesystem as filesystem -import turnkeyml.common.build as build -import turnkeyml.common.test_helpers as common - - -class Testing(unittest.TestCase): - def setUp(self) -> None: - filesystem.rmdir(cache_dir) - - return super().setUp() - - def test_001_device_naming(self): - """ - Ensure that the device name is correctly assigned - """ - test_script = "linear.py" - testargs = [ - "turnkey", - "-i", - os.path.join(corpus_dir, test_script), - "--cache-dir", - cache_dir, - "discover", - "export-pytorch", - "optimize-ort", - "benchmark", - "--device", - "example_family", - ] - with patch.object(sys, "argv", testargs): - turnkeycli() - - build_stats, build_state = common.get_stats_and_state(test_script, cache_dir) - - # Check if build was successful - assert build_state.build_status == build.FunctionStatus.SUCCESSFUL - - # Check if default part and config were assigned - expected_device = "example_family::part1::config1" - actual_device = build_stats["device_type"] - assert ( - actual_device == expected_device - ), f"Got {actual_device}, expected {expected_device}" - - -if __name__ == "__main__": - # Create a cache directory a directory with test models - cache_dir, corpus_dir = common.create_test_dir("plugins") - - unittest.main() From 9c4f7dcabb6ef895cd558b5ebd8268613d707396 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers Date: Tue, 27 Aug 2024 10:21:24 -0400 Subject: [PATCH 19/22] Add readme documentation --- plugins/README.md | 13 +++ plugins/devices/README.md | 235 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 248 insertions(+) create mode 100644 plugins/README.md create mode 100644 plugins/devices/README.md diff --git a/plugins/README.md b/plugins/README.md new file mode 100644 index 0000000..dbdc387 --- /dev/null +++ b/plugins/README.md @@ -0,0 +1,13 @@ +# TurnkeyML Plugins + +This directory contains the source code for plugins provided by the TurnkeyML team. + +A __turnkey plugin__ is a pip-installable package that extends the turnkeyml CLI with new __Tools__. + +Plugins provided by this repository: + +| Plugin Name | Link | Install Command | Tools Provided | Description | | +|--------------------------|-------------|-----------------------------------------|----------------|--------------------------------------------------------------------------------------|---| +| turnkeyml_plugin_devices | [devices](https://github.com/onnx/turnkeyml/blob/main/plugins/devices) | `pip install REPO_ROOT/plugins/devices` | `benchmark` | Provides device enablement to run your exported/compiled models on runtime and hardware targets, such as ONNX Runtime and x86, respectively. | | +| | | | | | | +| | | | | | | \ No newline at end of file diff --git a/plugins/devices/README.md b/plugins/devices/README.md new file mode 100644 index 0000000..862dbcd --- /dev/null +++ b/plugins/devices/README.md @@ -0,0 +1,235 @@ +# Devices: The Device Enablement Plugin + +The `devices` plugin installs the `benchmark` tool into the `turnkey` CLI, which in turn provides support for running models against target runtimes and hardware. + +The tools currently support the following combinations of runtimes and devices: + + + +| Device Type | Device arg | Runtime | Runtime arg | Specific Devices | +| ----------- | ---------- | ------------------------------------------------------------------------------------- | -------------------------------- | --------------------------------------------- | +| Nvidia GPU | nvidia | TensorRT† | trt | Any Nvidia GPU supported by TensorRT | +| x86 CPU | x86 | ONNX Runtime‡, Pytorch Eager, Pytoch 2.x Compiled | ort, torch-eager, torch-compiled | Any Intel or AMD CPU supported by the runtime | + + + +† Requires TensorRT >= 8.5.2 +‡ Requires ONNX Runtime >= 1.13.1 +* Requires Pytorch >= 2.0.0 + +## Getting Started + +> Note: these steps assume that your current working directory of your terminal is at the repository root. + +To install, run: + +`pip install -e plugins/devices` + +To learn more about the tool, run: + +`turnkey benchmark -h` + +To use the tool, run: + +`turnkey -i models/transformers/bert.py discover export-pytorch benchmark --runtime ort --device x86` + + +# Definitions + +This package uses the following definitions throughout. + +## Model + +A **model** is a PyTorch (torch.nn.Module) instance that has been instantiated in a Python script, or a `.onnx` file. + +- Examples: BERT-Base, ResNet-50, etc. + +## Device + +A **device** is a piece of hardware capable of running a model. + +- Examples: Nvidia A100 40GB, Intel Xeon Platinum 8380 + +## Runtime + +A **runtime** is a piece of software that executes a model on a device. + +- Different runtimes can produce different performance results on the same device because: + - Runtimes often optimize the model prior to execution. + - The runtime is responsible for orchestrating data movement, device invocation, etc. +- Examples: ONNX Runtime, TensorRT, PyTorch Eager Execution, etc. + +## BaseRT + +BaseRT is an abstract base class (ABC) that defines how our runtimes access and measure hardware. + +## Benchmark + +*Benchmark* is the process by which `BaseRT.benchmark()` collects performance statistics about a [model](#model). `BaseRT` is an abstract base class that defines the common benchmarking infrastructure that TurnkeyML provides across devices and runtimes. + +Specifically, `BaseRT.benchmark()` takes a [build](#build) of a model and executes it on a target device using target runtime software (see [Devices and Runtimes](#devices-and-runtimes)). + +By default, `BaseRT.benchmark()` will run the model 100 times to collect the following statistics: +1. Mean Latency, in milliseconds (ms): the average time it takes the runtime/device combination to execute the model/inputs combination once. This includes the time spent invoking the device and transferring the model's inputs and outputs between host memory and the device (when applicable). +1. Throughput, in inferences per second (IPS): the number of times the model/inputs combination can be executed on the runtime/device combination per second. + > - _Note_: `BaseRT.benchmark()` is not aware of whether `inputs` is a single input or a batch of inputs. If your `inputs` is actually a batch of inputs, you should multiply `BaseRT.benchmark()`'s reported IPS by the batch size. + +# Arguments + +The following arguments are used to configure `turnkey` and the APIs to target a specific device and runtime: + +### Devices + +Specify a device type that will be used for benchmarking. + +Usage: +- `benchmark --device TYPE` + - Benchmark the model(s) in `INPUT_FILES` on a locally installed device of type `TYPE` (eg, a locally installed Nvidia device). + +Valid values of `TYPE` include: +- `x86` (default): Intel and AMD x86 CPUs. +- `nvidia`: Nvidia GPUs. + +> _Note_: The tools are flexible with respect to which specific devices can be used, as long as they meet the requirements in the [Devices and Runtimes table](#devices-runtimes-table). +> - The `turnkey` CLI will simply use whatever device, of the given `TYPE`, is available on the machine. +> - For example, if you specify `--device nvidia` on a machine with an Nvidia A100 40GB installed, then the tools will use that Nvidia A100 40GB device. + +### Runtimes + +Indicates which software runtime should be used for the benchmark (e.g., ONNX Runtime vs. Torch eager execution for a CPU benchmark). + +Usage: +- `benchmark --runtime SW` + +Each device type has its own default runtime, as indicated below. +- Valid runtimes for `x86` device + - `ort`: ONNX Runtime (default). + - `torch-eager`: PyTorch eager execution. + - `torch-compiled`: PyTorch 2.x-style compiled graph execution using TorchInductor. +- Valid runtimes for `nvidia` device + - `trt`: Nvidia TensorRT (default). + +### Iterations + +Iterations takes an integer that specifies the number of times the model inference should be run during benchmarking. This helps in obtaining a more accurate measurement of the model's performance by averaging the results across multiple runs. Default set to 100 iterations per run. + +Usage: +- `turnkey benchmark INPUT_FILES --iterations 1000` + +### Custom Runtime Arguments + +Users can pass arbitrary arguments into a runtime, as long as the target runtime supports those arguments, by using the `--rt-args` argument. + +None of the built-in runtimes support such arguments, however plugin contributors can use this interface to add arguments to their custom runtimes. See [plugins contribution guideline](https://github.com/onnx/turnkeyml/blob/main/docs/contribute.md#contributing-a-plugin) for details. + +# Contributing + +To add a runtime to this plugin: + +1. Pick a unique name, `` for each runtime that will be supported by the plugin. + - This name will be used in the `benchmark --runtime ` [argument](#runtimes) + - For example, a runtime named `example-rt` would be invoked with `turnkey --runtime example-rt` + +1. Populate the [Implements Dictionary](#implements-dictionary) in in [device's `__init__.py`](https://github.com/onnx/turnkeyml/tree/main/plugins/devices/src/turnkeyml_plugin_devices.py) with a new key per-runtime with the following fields: + - `supported_devices: Union[Set,Dict]`: combination of devices supported by the runtime. + - For example, in `example_rt`, `"supported_devices": {"x86"}` indicates that the `x86` device is supported by the `example` runtime. + - A `device` typically refers to an entire family of devices, such as the set of all `x86` CPUs. However, plugins can provide explicit support for specific `device parts` within a device family. Additionally, specific `configurations` within a device model (e.g., a specific device firmware) are also supported. + - Each supported part within a device family must be defined as a dictionary. + - Each supported configuration within a device model must be defined as a list. + - Example: `"supported_devices": {"family1":{"part1":["config1","config2"]}}`. + - See [example_combined](https://github.com/onnx/turnkeyml/tree/main/examples/cli/plugins/example_combined) for a plugin implementation example that leverages this feature. + - Note: If a device is already supported by the tools, this simply adds support for another runtime to that device. If the device is _not_ already supported by the tools, this also adds support for that device and it will start to appear as an option for the `turnkey --device ` argument. + - `"build_required": Bool`: indicates whether the `build_model()` API should be called on the `model` and `inputs`. + - `"docker_required": Bool`: indicates whether benchmarking is implemented through a docker container. + - For example, `"build_required": False` indicates that no build is required, and benchmarking can be performed directly on the `model` and `inputs`. + - An example where `"build_required": True` is the `ort` runtime, which requires the `model` to be [built](#build) (via ONNX exporter) into a `.onnx` file prior to benchmarking. + - `"RuntimeClass": `, where `` is a unique name for a Python class that inherits `BaseRT` and implements the runtime. + - For example, `"RuntimeClass": ExampleRT` implements the `example` runtime. + - The interface for the runtime class is defined in [Runtime Class](#runtime-class) below. + - (Optional) `"status_stats": List[str]`: a list of keys from the build stats that should be printed out at the end of benchmarking in the CLI's `Status` output. These keys, and corresponding values, must be set in the runtime class using `self.stats.save_model_eval_stat(key, value)`. + - (Optional) `"requirement_check": Callable`: a callable that runs before each benchmark. This may be used to check whether the device selected is available and functional before each benchmarking run. Exceptions raised during this callable will halt the benchmark of all selected files. + +1. (Optional) Populate the [Implements Dictionary](#implements-dictionary) in in [device's `__init__.py`](https://github.com/onnx/turnkeyml/tree/main/plugins/devices/src/turnkeyml_plugin_devices.py) with any Tools defined by the plugin. + +1. Populate the package's source code with the following files (see [Plugin Directory Layout](#plugin-directory-layout)): + - A `runtime.py` script that implements the [Runtime Class](#runtime-class). + - (Optional) An `execute` method that follows the [Execute Method](#execute-method) template and implements the benchmarking methodology for the device/runtime combination(s). + - See the `tensorrt` runtime's `execute.py::main()` for a fairly minimal example. + - (Optional) A `within_conda.py` script that executes inside the conda env to collect benchmarking results. + - See the `onnxrt` runtime's `within_conda.py` for an example. + +1. Add a key-value pair to `extras_requires` in [device's setup.py](https://github.com/onnx/turnkeyml/tree/main/plugins/devices/setup.py): + - The `key` is the plugin's name + - The `value` is any dependencies required by the runtime. If the runtime has no special dependencies, you must still put an empty list (`[]` + +### Implements Dictionary + +This dictionary has keys for each type of plugin that will be installed by this package. +- Packages with runtime plugin(s) should have a `runtimes` key in the `implements` dictionary, which in turn includes one dictionary per runtime installed in the plugin. +- Packages with sequence plugin(s) should have a `sequences` key in the `implements` dictionary, which in turn includes one dictionary per runtime installed in the plugin. + +An `implements` dictionary with both sequences and runtimes would have the form: + +```python +implements = { + "runtimes": { + "runtime_1_name" : { + "build_required": Bool, + "RuntimeClass": Class(BaseRT), + "devices": List[str], + "default_sequence": Sequence instance, + "status_stats": ["custom_stat_1", "custom_stat_2"], + }, + "runtime_2_name" : {...}, + ... + }, + "tools": [ToolClass1, ToolClass2, etc.] +} +``` + + +### Runtime Class + +A runtime class inherits the abstract base class [`BaseRT`](https://github.com/onnx/turnkeyml/tree/main/src/turnkeyml/run/basert.py) and implements a one or more [runtimes](#runtime) to provide benchmarking support for one or more [devices](https://github.com/onnx/turnkeyml/blob/main/docs/tools_user_guide.md#devices). + +`BaseRT` has 4 methods that plugin developers must overload: +- `_setup()`: any code that should be called prior to benchmarking as a one-time setup. Called automatically at the end of `BaseRT.__init__()`. +- `mean_latency()`: returns the mean latency, in ms, for the benchmarking run. +- `throughput()`: returns the throughput, in IPS, for the benchmarking run. +- `device_name()`: returns the full device name for the device used in benchmarking. For example, a benchmark on a `x86` device might have a device name like `AMD Ryzen 7 PRO 6850U with Radeon Graphics`. +- [Optional] `_execute()`: method that `BaseRT` can automatically call during `BaseRT.benchmark()`, which implements the specific benchmarking methodology for that device/runtime combination. See [Execute Method](#execute-method) for more details. +- [Optional] `__init__()`: the `__init__` method can be overloaded to take additional keyword arguments, see [Custom Runtime Arguments](#custom-runtime-arguments) for details. + +Developers may also choose to overload the `benchmark()` function. By default, `BaseRT` will automatically invoke the module's [Execute Method](#execute-method) and use `mean_latency()`, `throughput()`, and `device_name()` to populate a `MeasuredPerformance` instance to return. However, some benchmarking methodologies may not lend themselves to a dedicated execute method. For example, `TorchRT` simply implements all of its benchmarking logic within an overloaded `benchmark()` method. + +### Custom Runtime Arguments + +The `turnkey` CLI/APIs allow users to pass arbitrary arguments to the runtime with `--rt-args`. + +Runtime arguments from the user's `--rt-args` will be passed into the runtime class's `__init__()` method as keyword arguments. Runtime plugins must accept any such custom arguments in their overloaded `__init__()` method, at which point the contributor is free to use them any way they like. A common usage would be to store arguments as members of `self` and then access them during `_setup()` or `_execute()`. + +The format for runtime arguments passed through the CLI is: + +``` +--rt-args arg1::value1 arg2::[value2,value3] flag_arg +``` + +Where: +- Arguments are space-delimited. +- Flag arguments (in the style of `argparse`'s `store_true`) are passed by key name only and result in `=True`. +- Arguments with a single value are passed as `key::value`. +- Arguments that are a list of values are passed as `key::[value1, value2, ...]`. + +### Execute Method + +Contributors who are not overloading `BaseRT.benchmark()` must overload `BaseRT._execute()`. By default, `BaseRT` will automatically call `self._execute()` during `BaseRT.benchmark()`, which implements the specific benchmarking methodology for that device/runtime combination. For example, `tensorrt/runtime.py::_execute_()` implements benchmarking on Nvidia GPU devices with the TensorRT runtime. + +Implementation of the execute method is optional, however if you do not implement the execute method you will have to overload `BaseRT.benchmark()` with your own functionality as in `TorchRT`. + +`_execute()` must implement the following arguments (note that it is not required to make use of all of them): +- `output_dir`: path where the benchmarking artifacts (ONNX files, inputs, outputs, performance data, etc.) are located. +- `onnx_file`: path where the ONNX file for the model is located. +- `outputs_file`: path where the benchmarking outputs will be located. +- `iterations`: number of execution iterations of the model to capture the throughput and mean latency. + +Additionally, `self._execute()` can access any custom runtime argument that has been added to `self` by the runtime class. \ No newline at end of file From df5ac131716039ad6effed3185d0ad4274937fb8 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers <80718789+jeremyfowers@users.noreply.github.com> Date: Tue, 27 Aug 2024 17:14:37 -0400 Subject: [PATCH 20/22] Update version.py Signed-off-by: Jeremy Fowers <80718789+jeremyfowers@users.noreply.github.com> --- src/turnkeyml/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/turnkeyml/version.py b/src/turnkeyml/version.py index 35c154a..ce1305b 100644 --- a/src/turnkeyml/version.py +++ b/src/turnkeyml/version.py @@ -1 +1 @@ -__version__ = "3.0.8" +__version__ = "4.0.0" From 18867d09d1ce55d406b7447bed1ba2398dbc3753 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers <80718789+jeremyfowers@users.noreply.github.com> Date: Tue, 27 Aug 2024 17:15:40 -0400 Subject: [PATCH 21/22] Update setup.py Signed-off-by: Jeremy Fowers <80718789+jeremyfowers@users.noreply.github.com> --- plugins/devices/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/devices/setup.py b/plugins/devices/setup.py index 2d977af..e514417 100644 --- a/plugins/devices/setup.py +++ b/plugins/devices/setup.py @@ -33,7 +33,7 @@ def get_specific_version(plugin_name: str, version_key: str) -> str: ], python_requires=">=3.8, <3.12", install_requires=[ - "turnkeyml==3.0.8", + "turnkeyml==4.0.0", "importlib_metadata", "onnx_tool", "numpy<2", From 760f6157cb5ae46d1632528cb1a3e16a69e830b4 Mon Sep 17 00:00:00 2001 From: Jeremy Fowers <80718789+jeremyfowers@users.noreply.github.com> Date: Tue, 27 Aug 2024 17:17:26 -0400 Subject: [PATCH 22/22] Update setup.py Signed-off-by: Jeremy Fowers <80718789+jeremyfowers@users.noreply.github.com> --- plugins/devices/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/devices/setup.py b/plugins/devices/setup.py index e514417..6074e3e 100644 --- a/plugins/devices/setup.py +++ b/plugins/devices/setup.py @@ -21,7 +21,7 @@ def get_specific_version(plugin_name: str, version_key: str) -> str: setup( name="turnkeyml_plugin_devices", - version="0.0.2", + version="1.0.0", package_dir={"": "src"}, packages=[ "turnkeyml_plugin_devices",