From f4cc81226dfbe4cf0c246becc2649ee9b6133495 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 23 May 2024 16:33:21 +0200 Subject: [PATCH 1/8] first commit --- .github/workflows/test.yml | 2 + .../unet_3d/kits_19/run.py | 77 +++++++++---------- tests/test_pytorch_models.py | 35 +++++++++ 3 files changed, 74 insertions(+), 40 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 659dcb43..11f2f7fd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -229,6 +229,8 @@ jobs: S3_URL_COCO_DATASET: ${{ secrets.S3_URL_COCO_DATASET }} S3_URL_COCO_DATASET_ANNOTATIONS: ${{ secrets.S3_URL_COCO_DATASET_ANNOTATIONS }} S3_URL_COVOST2_DATASET: ${{ secrets.S3_URL_COVOST2_DATASET }} + S3_URL_KITS19_REDUCED_DATASET: ${{ secrets.S3_URL_KITS19_REDUCED_DATASET }} + S3_URL_UNET_KITS_PYTORCH_FP32: ${{ secrets.S3_URL_UNET_KITS_PYTORCH_FP32 }} HF_HUB_TOKEN: ${{ secrets.HF_HUB_TOKEN }} steps: - name: Git checkout & pull submodules diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index f548c258..88408d87 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -1,5 +1,7 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2024, Ampere Computing LLC + + try: from utils import misc # noqa except ModuleNotFoundError: @@ -18,31 +20,6 @@ sys.exit(1) -def parse_args(): - import argparse - parser = argparse.ArgumentParser(description="Run 3D Unet KiTS 2019 model.") - parser.add_argument("-m", "--model_path", - type=str, - help="path to the model") - parser.add_argument("-p", "--precision", - type=str, choices=["fp32"], required=True, - help="precision of the model provided") - parser.add_argument("-f", "--framework", - type=str, default="tf", - choices=["tf"], - help="specify the framework in which a model should be run") - parser.add_argument("--timeout", - type=float, default=60.0, - help="timeout in seconds") - parser.add_argument("--num_runs", - type=int, - help="number of passes through network to execute") - parser.add_argument("--kits_path", - type=str, - help="path to directory with KiTS19 dataset") - return parser.parse_args() - - def run_tf_fp(model_path, num_runs, timeout, kits_path): import numpy as np import tensorflow as tf @@ -64,28 +41,48 @@ def run_single_pass(tf_runner, kits): return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) +def run_pytorch_fp(model_path, num_runs, timeout, kits_path): + import torch + import numpy as np + import tensorflow as tf + from utils.pytorch import PyTorchRunnerV2 + from utils.cv.kits import KiTS19 + from utils.benchmark import run_model + + def run_single_pass(pytorch_runner, kits): + output = pytorch_runner.run(1, torch.from_numpy(np.expand_dims(kits.get_input_array(), axis=0))) + kits.submit_predictions(tf.convert_to_tensor(output.numpy())) + + dataset = KiTS19(dataset_dir_path=kits_path) + model = torch.jit.load(model_path, map_location=torch.device('cpu')).eval() + model = torch.jit.freeze(model) + runner = PyTorchRunnerV2(model) + + return run_model(run_single_pass, runner, dataset, 1, num_runs, timeout) + + def run_tf_fp32(model_path, num_runs, timeout, kits_path, **kwargs): return run_tf_fp(model_path, num_runs, timeout, kits_path) +def run_pytorch_fp32(model_path, num_runs, timeout, kits_path, **kwargs): + return run_pytorch_fp(model_path, num_runs, timeout, kits_path) + + def main(): - from utils.misc import print_goodbye_message_and_die - args = parse_args() - if args.framework == "tf": - if args.model_path is None: - print_goodbye_message_and_die( - "a path to model is unspecified!") - - if args.precision == "fp32": - run_tf_fp32(**vars(args)) - else: - print_goodbye_message_and_die( - "this model seems to be unsupported in a specified precision: " + args.precision) + from utils.helpers import DefaultArgParser + parser = DefaultArgParser(["tf", "pytorch"]) + parser.require_model_path() + parser.add_argument("--kits_path", + type=str, + help="path to directory with KiTS19 dataset") + args = parser.parse() + if args.framework == 'tf': + run_tf_fp32(**vars(parser.parse())) else: - print_goodbye_message_and_die( - "this model seems to be unsupported in a specified framework: " + args.framework) + run_pytorch_fp32(**vars(parser.parse())) if __name__ == "__main__": - main() + main() \ No newline at end of file diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 00be4edf..ddbe4f48 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -227,6 +227,41 @@ def wrapper(**kwargs): self.assertTrue(acc["f1"] / f1_ref > 0.95) +class UNET_KITS(unittest.TestCase): + def setUp(self): + self.dataset_path = pathlib.Path(get_downloads_path(), "kits19") + if not self.dataset_path.exists(): + # url = os.environ.get("S3_URL_KITS19_REDUCED_DATASET") + url = "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/kits19_reduced.tar.gz" + assert url is not None + subprocess.run(f"wget -P /tmp {url}".split(), + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.run(f"tar -xf /tmp/kits19_reduced.tar.gz -C {get_downloads_path()}".split(), + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.run("rm /tmp/kits19_reduced.tar.gz".split(), + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + self.model_path = pathlib.Path(get_downloads_path(), "3d_unet_kits_pytorch_fp32.ptc") + if not self.model_path.exists(): + # url = os.environ.get("S3_URL_UNET_KITS_PYTORCH_FP32") + url = "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/3d_unet_kits_pytorch_fp32.ptc" + subprocess.run(f"wget -P {get_downloads_path()} {url}".split(), + check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + def test_unet_kits(self): + from computer_vision.semantic_segmentation.unet_3d.kits_19.run import run_pytorch_fp32 + + def wrapper(**kwargs): + kwargs["q"].put(run_pytorch_fp32(**kwargs)[0]) + + mean_kidney_acc, mean_tumor_acc = 0.927, 0.837 + acc = run_process(wrapper, {"model_path": self.model_path, "kits_path": self.dataset_path, + "batch_size": 1, "num_runs": 500, "timeout": 200, "debug": True}) + + self.assertTrue(acc["mean_kidney_acc"] / mean_kidney_acc > 0.90) + self.assertTrue(acc["mean_tumor_acc"] / mean_tumor_acc > 0.80) + + def download_imagenet_maybe(): dataset_path = pathlib.Path(get_downloads_path(), "ILSVRC2012_onspecta") if not dataset_path.exists(): From 0e149bc139dd65812592c4028cbd6abf0a4ec904 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 23 May 2024 16:39:52 +0200 Subject: [PATCH 2/8] first commit --- tests/test_pytorch_models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index ddbe4f48..ab547ea0 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -231,8 +231,8 @@ class UNET_KITS(unittest.TestCase): def setUp(self): self.dataset_path = pathlib.Path(get_downloads_path(), "kits19") if not self.dataset_path.exists(): - # url = os.environ.get("S3_URL_KITS19_REDUCED_DATASET") - url = "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/kits19_reduced.tar.gz" + url = os.environ.get("S3_URL_KITS19_REDUCED_DATASET") + # url = "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/kits19_reduced.tar.gz" assert url is not None subprocess.run(f"wget -P /tmp {url}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) @@ -243,8 +243,8 @@ def setUp(self): self.model_path = pathlib.Path(get_downloads_path(), "3d_unet_kits_pytorch_fp32.ptc") if not self.model_path.exists(): - # url = os.environ.get("S3_URL_UNET_KITS_PYTORCH_FP32") - url = "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/3d_unet_kits_pytorch_fp32.ptc" + url = os.environ.get("S3_URL_UNET_KITS_PYTORCH_FP32") + # url = "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/3d_unet_kits_pytorch_fp32.ptc" subprocess.run(f"wget -P {get_downloads_path()} {url}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) From f341e9bf0d08672c01e17dbed9378beeb2a93fc4 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 23 May 2024 16:42:37 +0200 Subject: [PATCH 3/8] first commit --- computer_vision/semantic_segmentation/unet_3d/kits_19/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py index 88408d87..807a9379 100644 --- a/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py +++ b/computer_vision/semantic_segmentation/unet_3d/kits_19/run.py @@ -85,4 +85,4 @@ def main(): if __name__ == "__main__": - main() \ No newline at end of file + main() From 3be1fad3352a58e66677407106a0c2fc3d96d23f Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 23 May 2024 16:49:59 +0200 Subject: [PATCH 4/8] wip --- tests/test_pytorch_models.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index ab547ea0..d1ebfd76 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -232,7 +232,6 @@ def setUp(self): self.dataset_path = pathlib.Path(get_downloads_path(), "kits19") if not self.dataset_path.exists(): url = os.environ.get("S3_URL_KITS19_REDUCED_DATASET") - # url = "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/kits19_reduced.tar.gz" assert url is not None subprocess.run(f"wget -P /tmp {url}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) @@ -244,7 +243,6 @@ def setUp(self): self.model_path = pathlib.Path(get_downloads_path(), "3d_unet_kits_pytorch_fp32.ptc") if not self.model_path.exists(): url = os.environ.get("S3_URL_UNET_KITS_PYTORCH_FP32") - # url = "https://ampereaimodelzoo.s3.eu-central-1.amazonaws.com/3d_unet_kits_pytorch_fp32.ptc" subprocess.run(f"wget -P {get_downloads_path()} {url}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) From 46ad21222b626badb5c23b54df1572b46f3680d9 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 23 May 2024 17:38:57 +0200 Subject: [PATCH 5/8] wip --- .github/workflows/test.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 11f2f7fd..93944e15 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -60,6 +60,8 @@ jobs: S3_URL_IMAGENET_DATASET_LABELS: ${{ secrets.S3_URL_IMAGENET_DATASET_LABELS }} S3_URL_COCO_DATASET: ${{ secrets.S3_URL_COCO_DATASET }} S3_URL_COCO_DATASET_ANNOTATIONS: ${{ secrets.S3_URL_COCO_DATASET_ANNOTATIONS }} + S3_URL_KITS19_REDUCED_DATASET: ${{ secrets.S3_URL_KITS19_REDUCED_DATASET }} + S3_URL_UNET_KITS_PYTORCH_FP32: ${{ secrets.S3_URL_UNET_KITS_PYTORCH_FP32 }} HF_HUB_TOKEN: ${{ secrets.HF_HUB_TOKEN }} steps: - name: Install git @@ -123,6 +125,8 @@ jobs: S3_URL_IMAGENET_DATASET_LABELS: ${{ secrets.S3_URL_IMAGENET_DATASET_LABELS }} S3_URL_COCO_DATASET: ${{ secrets.S3_URL_COCO_DATASET }} S3_URL_COCO_DATASET_ANNOTATIONS: ${{ secrets.S3_URL_COCO_DATASET_ANNOTATIONS }} + S3_URL_KITS19_REDUCED_DATASET: ${{ secrets.S3_URL_KITS19_REDUCED_DATASET }} + S3_URL_UNET_KITS_PYTORCH_FP32: ${{ secrets.S3_URL_UNET_KITS_PYTORCH_FP32 }} S3_URL_COVOST2_DATASET: ${{ secrets.S3_URL_COVOST2_DATASET }} HF_HUB_TOKEN: ${{ secrets.HF_HUB_TOKEN }} steps: @@ -185,6 +189,8 @@ jobs: S3_URL_IMAGENET_DATASET_LABELS: ${{ secrets.S3_URL_IMAGENET_DATASET_LABELS }} S3_URL_COCO_DATASET: ${{ secrets.S3_URL_COCO_DATASET }} S3_URL_COCO_DATASET_ANNOTATIONS: ${{ secrets.S3_URL_COCO_DATASET_ANNOTATIONS }} + S3_URL_KITS19_REDUCED_DATASET: ${{ secrets.S3_URL_KITS19_REDUCED_DATASET }} + S3_URL_UNET_KITS_PYTORCH_FP32: ${{ secrets.S3_URL_UNET_KITS_PYTORCH_FP32 }} S3_URL_COVOST2_DATASET: ${{ secrets.S3_URL_COVOST2_DATASET }} HF_HUB_TOKEN: ${{ secrets.HF_HUB_TOKEN }} steps: From 4803d159ea31c59569fdd94b4e86351c0a3435a4 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Fri, 24 May 2024 23:19:18 +0200 Subject: [PATCH 6/8] wip --- .github/workflows/test.yml | 1 + tests/test_pytorch_models.py | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 93944e15..a351ce04 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,6 +52,7 @@ jobs: COCO_IMG_PATH: aio_objdet_dataset COCO_ANNO_PATH: aio_objdet_dataset/annotations.json OMP_NUM_THREADS: 4 + DISABLE_UNET_TEST: 1 S3_URL_CRITEO_DATASET: ${{ secrets.S3_URL_CRITEO_DATASET }} S3_URL_RESNET_50_V15_TF_FP32: ${{ secrets.S3_URL_RESNET_50_V15_TF_FP32 }} S3_URL_SSD_INCEPTION_V2_TF_FP32: ${{ secrets.S3_URL_SSD_INCEPTION_V2_TF_FP32 }} diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index d1ebfd76..2fb5207c 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -13,6 +13,10 @@ TIMEOUT = 3 * 60 * 60 pid = os.getpid() +if os.environ.get("DISABLE_UNET_TEST") == 1: + skip_unet = True +else: + skip_unet = False def run_process(wrapper, kwargs): @@ -228,6 +232,7 @@ def wrapper(**kwargs): class UNET_KITS(unittest.TestCase): + @unittest.skipIf(skip_unet, "if the test runs on x86 skip this model") def setUp(self): self.dataset_path = pathlib.Path(get_downloads_path(), "kits19") if not self.dataset_path.exists(): @@ -246,6 +251,7 @@ def setUp(self): subprocess.run(f"wget -P {get_downloads_path()} {url}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + @unittest.skipIf(skip_unet, "if the test runs on x86 skip this model") def test_unet_kits(self): from computer_vision.semantic_segmentation.unet_3d.kits_19.run import run_pytorch_fp32 From e6e6d0f9ea7fd9a678f96c141917cae8f7497d7d Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 6 Jun 2024 15:56:30 +0200 Subject: [PATCH 7/8] wip --- .github/workflows/test.yml | 1 - tests/test_pytorch_models.py | 8 +++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a351ce04..93944e15 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -52,7 +52,6 @@ jobs: COCO_IMG_PATH: aio_objdet_dataset COCO_ANNO_PATH: aio_objdet_dataset/annotations.json OMP_NUM_THREADS: 4 - DISABLE_UNET_TEST: 1 S3_URL_CRITEO_DATASET: ${{ secrets.S3_URL_CRITEO_DATASET }} S3_URL_RESNET_50_V15_TF_FP32: ${{ secrets.S3_URL_RESNET_50_V15_TF_FP32 }} S3_URL_SSD_INCEPTION_V2_TF_FP32: ${{ secrets.S3_URL_SSD_INCEPTION_V2_TF_FP32 }} diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 2fb5207c..812a1f2f 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -13,10 +13,6 @@ TIMEOUT = 3 * 60 * 60 pid = os.getpid() -if os.environ.get("DISABLE_UNET_TEST") == 1: - skip_unet = True -else: - skip_unet = False def run_process(wrapper, kwargs): @@ -232,7 +228,9 @@ def wrapper(**kwargs): class UNET_KITS(unittest.TestCase): - @unittest.skipIf(skip_unet, "if the test runs on x86 skip this model") + + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") def setUp(self): self.dataset_path = pathlib.Path(get_downloads_path(), "kits19") if not self.dataset_path.exists(): From 973c5cb64dbfbcaa115afbaf8a78aa43c402f248 Mon Sep 17 00:00:00 2001 From: Marcel Wilnicki Date: Thu, 6 Jun 2024 15:58:26 +0200 Subject: [PATCH 8/8] wip --- tests/test_pytorch_models.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/test_pytorch_models.py b/tests/test_pytorch_models.py index 812a1f2f..4b102b05 100644 --- a/tests/test_pytorch_models.py +++ b/tests/test_pytorch_models.py @@ -229,8 +229,6 @@ def wrapper(**kwargs): class UNET_KITS(unittest.TestCase): - @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") - @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") def setUp(self): self.dataset_path = pathlib.Path(get_downloads_path(), "kits19") if not self.dataset_path.exists(): @@ -249,7 +247,8 @@ def setUp(self): subprocess.run(f"wget -P {get_downloads_path()} {url}".split(), check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) - @unittest.skipIf(skip_unet, "if the test runs on x86 skip this model") + @unittest.skipIf(psutil.virtual_memory().available / 1024 ** 3 < 100, "too little memory") + @unittest.skipUnless('_aio_profiler_print' in dir(torch._C), "Ampere optimized PyTorch required") def test_unet_kits(self): from computer_vision.semantic_segmentation.unet_3d.kits_19.run import run_pytorch_fp32