From e8a15ee5b590dd13daf90b84bf120096c058e64c Mon Sep 17 00:00:00 2001 From: zk1998 Date: Thu, 13 Jun 2024 17:11:11 +0800 Subject: [PATCH 01/10] [CI] fix ci timeout --- tests/modifier_models_test.py | 9 +++++++++ tests/modifier_speed_test.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/tests/modifier_models_test.py b/tests/modifier_models_test.py index c0332bc0..4eaf8683 100644 --- a/tests/modifier_models_test.py +++ b/tests/modifier_models_test.py @@ -1,5 +1,7 @@ import unittest +import gc + import torch from tinynn.prune.oneshot_pruner import OneShotChannelPruner @@ -58,6 +60,13 @@ def test_timm(self): model(dummy_input) print(f"test {model_name} over!\n") + def tearDown(self): + gc.collect() + + @classmethod + def tearDownClass(cls): + gc.collect() + if __name__ == '__main__': unittest.main() diff --git a/tests/modifier_speed_test.py b/tests/modifier_speed_test.py index efacb1e5..e40b7c2f 100644 --- a/tests/modifier_speed_test.py +++ b/tests/modifier_speed_test.py @@ -8,6 +8,8 @@ import random import torch.nn.functional +import gc + from tinynn.prune.oneshot_pruner import OneShotChannelPruner from tinynn.util.util import import_from_path, get_logger @@ -119,6 +121,13 @@ def test_densenet121(self): model = torchvision.models.densenet121(pretrained=False) speed_test(model, torch.randn((1, 3, 224, 224))) + def tearDown(self): + gc.collect() + + @classmethod + def tearDownClass(cls): + gc.collect() + if __name__ == '__main__': unittest.main() From 5103c80a410054ae6e97fd98b596b6dec0808a51 Mon Sep 17 00:00:00 2001 From: zk1998 Date: Thu, 13 Jun 2024 17:13:15 +0800 Subject: [PATCH 02/10] trigger full test --- .github/workflows/full-test.yml | 14 ++++++++------ .github/workflows/unit-test.yml | 12 ++++-------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/full-test.yml b/.github/workflows/full-test.yml index e3e6034b..d815cb43 100644 --- a/.github/workflows/full-test.yml +++ b/.github/workflows/full-test.yml @@ -1,8 +1,12 @@ name: Full tests on: - schedule: - - cron: '00 19 * * *' + push: + branches: + - main + pull_request: + branches: + - main jobs: unit-test: @@ -11,11 +15,9 @@ jobs: strategy: fail-fast: false matrix: - os: ["ubuntu-latest", "macos-13", "windows-latest"] - vers: [ {pt_ver: "1.6.0", tv_ver: "0.7.0"}, {pt_ver: "1.7.0", tv_ver: "0.8.1"}, {pt_ver: "1.8.0", tv_ver: "0.9.0"}, {pt_ver: "1.9.0", tv_ver: "0.10.0"}, {pt_ver: "1.10.0", tv_ver: "0.11.1"}, {pt_ver: "1.11.0", tv_ver: "0.12.0"}, {pt_ver: "1.12.0", tv_ver: "0.13.0"} ] include: - os: macos-latest - vers: + vers: pt_ver: latest tv_ver: latest defaults: @@ -54,7 +56,7 @@ jobs: if [[ "$TORCHVISION_VER" == "latest" && "$PYTORCH_VER" == "latest" ]]; then conda install pytorch torchvision cpuonly pillow=6 -c pytorch elif [[ "$TORCHVISION_VER" == "0.9."* || "$TORCHVISION_VER" == "0.10."* ]]; then - conda install pillow=6 -c conda-forge + conda install pillow=6 -c conda-forge conda install pytorch=$PYTORCH_VER torchvision=$TORCHVISION_VER cpuonly -c pytorch else conda install pytorch=$PYTORCH_VER torchvision=$TORCHVISION_VER cpuonly pillow=6 -c pytorch diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 399c2c0d..6a06cbe3 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -1,12 +1,8 @@ name: Unit tests on: - push: - branches: - - main - pull_request: - branches: - - main + schedule: + - cron: '00 19 * * *' jobs: unit-test: @@ -19,7 +15,7 @@ jobs: vers: [ {pt_ver: "1.6.0", tv_ver: "0.7.0"}, {pt_ver: "latest", tv_ver: "latest"} ] include: - os: macos-latest - vers: + vers: pt_ver: latest tv_ver: latest defaults: @@ -48,7 +44,7 @@ jobs: if [[ "$TORCHVISION_VER" == "latest" && "$PYTORCH_VER" == "latest" ]]; then conda install pytorch torchvision cpuonly pillow=6 -c pytorch elif [[ "$TORCHVISION_VER" == "0.9."* || "$TORCHVISION_VER" == "0.10."* ]]; then - conda install pillow=6 -c conda-forge + conda install pillow=6 -c conda-forge conda install pytorch=$PYTORCH_VER torchvision=$TORCHVISION_VER cpuonly -c pytorch else conda install pytorch=$PYTORCH_VER torchvision=$TORCHVISION_VER cpuonly pillow=6 -c pytorch From 60510f463cdf028896b7bd54823c732994a5999a Mon Sep 17 00:00:00 2001 From: zk1998 Date: Fri, 14 Jun 2024 14:56:12 +0800 Subject: [PATCH 03/10] [CI] use subprocess to fix memory leak --- .github/workflows/full-test.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/full-test.yml b/.github/workflows/full-test.yml index d815cb43..7c179cc3 100644 --- a/.github/workflows/full-test.yml +++ b/.github/workflows/full-test.yml @@ -80,9 +80,16 @@ jobs: else pip install black 'ruff<0.0.234' tensorflow pytest scipy interval fi + if [[ "$RUNNER_OS" == "macOS" && "$RUNNER_ARCH" == "ARM64" ]]; then + pip install pytest-forked + fi - name: Lint checks run: python -m ruff . - name: Run tests run: | source activate.sh - pytest tests -v + if [[ "$RUNNER_OS" == "macOS" && "$RUNNER_ARCH" == "ARM64" ]]; then + pytest tests --forked -v + else + pytest tests -v + fi From 52bea8e88811ef68e991992f82d32baa8595c14d Mon Sep 17 00:00:00 2001 From: zk1998 Date: Fri, 14 Jun 2024 16:21:46 +0800 Subject: [PATCH 04/10] [ci] bug fix --- tests/common_utils.py | 12 ++++++++++++ tests/converter_qat_test.py | 3 ++- tests/converter_test.py | 3 ++- tests/modifier_torchvision_test.py | 3 ++- tests/tracer_test.py | 3 ++- 5 files changed, 20 insertions(+), 4 deletions(-) diff --git a/tests/common_utils.py b/tests/common_utils.py index 31e15b2c..87cf9a9f 100644 --- a/tests/common_utils.py +++ b/tests/common_utils.py @@ -9,6 +9,11 @@ except ImportError: models = None +try: + import pytest +except ImportError: + pytest = None + IS_CI = os.getenv('CI', '') == 'true' @@ -69,3 +74,10 @@ def prepare_inputs(model): t = torch.ones(shape) inputs.append(t) return inputs + + +def mark_forked(f): + if pytest is not None: + return pytest.mark.forked(f) + else: + return f diff --git a/tests/converter_qat_test.py b/tests/converter_qat_test.py index 5cd550c8..4a8fd068 100644 --- a/tests/converter_qat_test.py +++ b/tests/converter_qat_test.py @@ -11,7 +11,7 @@ from tinynn.converter import TFLiteConverter from tinynn.graph.tracer import model_tracer from tinynn.graph.quantization.quantizer import QATQuantizer -from common_utils import IS_CI, collect_torchvision_models, prepare_inputs +from common_utils import IS_CI, collect_torchvision_models, mark_forked, prepare_inputs HAS_TF = False @@ -108,6 +108,7 @@ def prepare_q_model(model_name): return qat_model, inputs + @mark_forked def f(self): model_name = model_class.__name__ model_file = model_name diff --git a/tests/converter_test.py b/tests/converter_test.py index 3998cd00..78c93bce 100644 --- a/tests/converter_test.py +++ b/tests/converter_test.py @@ -9,7 +9,7 @@ import numpy as np from tinynn.converter import TFLiteConverter -from common_utils import IS_CI, collect_custom_models, collect_torchvision_models, prepare_inputs +from common_utils import IS_CI, collect_custom_models, collect_torchvision_models, mark_forked, prepare_inputs HAS_TF = False @@ -88,6 +88,7 @@ def __prepare__(mcls, name, bases): @classmethod def build_model_test(cls, model_class): + @mark_forked def f(self): model_name = model_class.__name__ model_file = model_name diff --git a/tests/modifier_torchvision_test.py b/tests/modifier_torchvision_test.py index 70a14bdb..4ee927a2 100644 --- a/tests/modifier_torchvision_test.py +++ b/tests/modifier_torchvision_test.py @@ -12,7 +12,7 @@ from tinynn.prune.oneshot_pruner import OneShotChannelPruner from tinynn.util.util import import_from_path -from common_utils import collect_torchvision_models, prepare_inputs, IS_CI +from common_utils import collect_torchvision_models, mark_forked, prepare_inputs, IS_CI def transform_output(output): @@ -53,6 +53,7 @@ def __prepare__(mcls, name, bases): @classmethod def build_model_test(cls, model_class): + @mark_forked def f(self): model_name = model_class.__name__ diff --git a/tests/tracer_test.py b/tests/tracer_test.py index 84cd3d8a..271ca41b 100644 --- a/tests/tracer_test.py +++ b/tests/tracer_test.py @@ -4,7 +4,7 @@ import gc from tinynn.graph.tracer import trace, model_tracer -from common_utils import collect_custom_models, collect_torchvision_models, prepare_inputs, IS_CI +from common_utils import collect_custom_models, collect_torchvision_models, mark_forked, prepare_inputs, IS_CI BLACKLIST = ( @@ -40,6 +40,7 @@ def __prepare__(mcls, name, bases): @classmethod def build_model_test(cls, model_class, eliminate_dead_graph): + @mark_forked def f(self): model_name = model_class.__name__ model_file = model_name From dc97d87cc316b25ed5265047c024668425d965e0 Mon Sep 17 00:00:00 2001 From: zk1998 Date: Mon, 17 Jun 2024 11:02:07 +0800 Subject: [PATCH 05/10] [CI] use test_run script to do fulltest --- tests/test_run.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 tests/test_run.py diff --git a/tests/test_run.py b/tests/test_run.py new file mode 100644 index 00000000..0f768da5 --- /dev/null +++ b/tests/test_run.py @@ -0,0 +1,37 @@ +import subprocess +import sys + +import glob + +# List of test files to run +test_files = glob.glob('**/*_test.py', recursive=True) + +# Function to run a test file +def run_test(file): + try: + result = subprocess.run(["pytest", "-v", file], check=True, capture_output=True, text=True) + print(result.stdout) + return None # No error + except subprocess.CalledProcessError as e: + error_message = f"Tests failed in {file}\n{e.stdout}\n{e.stderr}" + return error_message # Return the error message + + +# Run all test files +def main(): + errors = [] + for test_file in test_files: + error = run_test(test_file) + if error: + errors.append(error) + + if errors: + raise Exception("\n".join(errors)) + + +if __name__ == "__main__": + try: + main() + except Exception as e: + print(e) + sys.exit(1) From ad32ce596522834ef4b2b333992a756cc1ca20ab Mon Sep 17 00:00:00 2001 From: zk1998 Date: Mon, 17 Jun 2024 11:06:30 +0800 Subject: [PATCH 06/10] [CI] trigger full test --- .github/workflows/full-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/full-test.yml b/.github/workflows/full-test.yml index 7c179cc3..ded74061 100644 --- a/.github/workflows/full-test.yml +++ b/.github/workflows/full-test.yml @@ -89,7 +89,7 @@ jobs: run: | source activate.sh if [[ "$RUNNER_OS" == "macOS" && "$RUNNER_ARCH" == "ARM64" ]]; then - pytest tests --forked -v + python tests/test_run.py else pytest tests -v fi From a7595c93db73b0b37560c4a4b1896c22abeb4341 Mon Sep 17 00:00:00 2001 From: zk1998 Date: Tue, 18 Jun 2024 10:32:50 +0800 Subject: [PATCH 07/10] [CI] del unused code --- tests/common_utils.py | 15 ++++----------- tests/converter_qat_test.py | 3 +-- tests/converter_test.py | 3 +-- tests/modifier_torchvision_test.py | 3 +-- tests/test_run.py | 6 +++++- tests/tracer_test.py | 3 +-- 6 files changed, 13 insertions(+), 20 deletions(-) diff --git a/tests/common_utils.py b/tests/common_utils.py index 87cf9a9f..99099178 100644 --- a/tests/common_utils.py +++ b/tests/common_utils.py @@ -9,10 +9,10 @@ except ImportError: models = None -try: - import pytest -except ImportError: - pytest = None +# try: +# import pytest +# except ImportError: +# pytest = None IS_CI = os.getenv('CI', '') == 'true' @@ -74,10 +74,3 @@ def prepare_inputs(model): t = torch.ones(shape) inputs.append(t) return inputs - - -def mark_forked(f): - if pytest is not None: - return pytest.mark.forked(f) - else: - return f diff --git a/tests/converter_qat_test.py b/tests/converter_qat_test.py index 4a8fd068..5cd550c8 100644 --- a/tests/converter_qat_test.py +++ b/tests/converter_qat_test.py @@ -11,7 +11,7 @@ from tinynn.converter import TFLiteConverter from tinynn.graph.tracer import model_tracer from tinynn.graph.quantization.quantizer import QATQuantizer -from common_utils import IS_CI, collect_torchvision_models, mark_forked, prepare_inputs +from common_utils import IS_CI, collect_torchvision_models, prepare_inputs HAS_TF = False @@ -108,7 +108,6 @@ def prepare_q_model(model_name): return qat_model, inputs - @mark_forked def f(self): model_name = model_class.__name__ model_file = model_name diff --git a/tests/converter_test.py b/tests/converter_test.py index 78c93bce..3998cd00 100644 --- a/tests/converter_test.py +++ b/tests/converter_test.py @@ -9,7 +9,7 @@ import numpy as np from tinynn.converter import TFLiteConverter -from common_utils import IS_CI, collect_custom_models, collect_torchvision_models, mark_forked, prepare_inputs +from common_utils import IS_CI, collect_custom_models, collect_torchvision_models, prepare_inputs HAS_TF = False @@ -88,7 +88,6 @@ def __prepare__(mcls, name, bases): @classmethod def build_model_test(cls, model_class): - @mark_forked def f(self): model_name = model_class.__name__ model_file = model_name diff --git a/tests/modifier_torchvision_test.py b/tests/modifier_torchvision_test.py index 4ee927a2..70a14bdb 100644 --- a/tests/modifier_torchvision_test.py +++ b/tests/modifier_torchvision_test.py @@ -12,7 +12,7 @@ from tinynn.prune.oneshot_pruner import OneShotChannelPruner from tinynn.util.util import import_from_path -from common_utils import collect_torchvision_models, mark_forked, prepare_inputs, IS_CI +from common_utils import collect_torchvision_models, prepare_inputs, IS_CI def transform_output(output): @@ -53,7 +53,6 @@ def __prepare__(mcls, name, bases): @classmethod def build_model_test(cls, model_class): - @mark_forked def f(self): model_name = model_class.__name__ diff --git a/tests/test_run.py b/tests/test_run.py index 0f768da5..d301efcd 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -4,7 +4,11 @@ import glob # List of test files to run -test_files = glob.glob('**/*_test.py', recursive=True) +test_files = glob.glob('tests/*_test.py', recursive=True) + +print("Test files:") +for file in test_files: + print(f" - {file}") # Function to run a test file def run_test(file): diff --git a/tests/tracer_test.py b/tests/tracer_test.py index 271ca41b..84cd3d8a 100644 --- a/tests/tracer_test.py +++ b/tests/tracer_test.py @@ -4,7 +4,7 @@ import gc from tinynn.graph.tracer import trace, model_tracer -from common_utils import collect_custom_models, collect_torchvision_models, mark_forked, prepare_inputs, IS_CI +from common_utils import collect_custom_models, collect_torchvision_models, prepare_inputs, IS_CI BLACKLIST = ( @@ -40,7 +40,6 @@ def __prepare__(mcls, name, bases): @classmethod def build_model_test(cls, model_class, eliminate_dead_graph): - @mark_forked def f(self): model_name = model_class.__name__ model_file = model_name From 715fc20996e0599c315eae668927e1b53dd0a75c Mon Sep 17 00:00:00 2001 From: zk1998 Date: Tue, 18 Jun 2024 11:03:18 +0800 Subject: [PATCH 08/10] [CI] del unused code --- tests/common_utils.py | 4 ---- tests/modifier_models_test.py | 9 --------- tests/modifier_speed_test.py | 9 --------- tests/test_run.py | 28 +++++++++++++--------------- 4 files changed, 13 insertions(+), 37 deletions(-) diff --git a/tests/common_utils.py b/tests/common_utils.py index 99099178..8437e4f7 100644 --- a/tests/common_utils.py +++ b/tests/common_utils.py @@ -9,10 +9,6 @@ except ImportError: models = None -# try: -# import pytest -# except ImportError: -# pytest = None IS_CI = os.getenv('CI', '') == 'true' diff --git a/tests/modifier_models_test.py b/tests/modifier_models_test.py index 4eaf8683..c0332bc0 100644 --- a/tests/modifier_models_test.py +++ b/tests/modifier_models_test.py @@ -1,7 +1,5 @@ import unittest -import gc - import torch from tinynn.prune.oneshot_pruner import OneShotChannelPruner @@ -60,13 +58,6 @@ def test_timm(self): model(dummy_input) print(f"test {model_name} over!\n") - def tearDown(self): - gc.collect() - - @classmethod - def tearDownClass(cls): - gc.collect() - if __name__ == '__main__': unittest.main() diff --git a/tests/modifier_speed_test.py b/tests/modifier_speed_test.py index e40b7c2f..efacb1e5 100644 --- a/tests/modifier_speed_test.py +++ b/tests/modifier_speed_test.py @@ -8,8 +8,6 @@ import random import torch.nn.functional -import gc - from tinynn.prune.oneshot_pruner import OneShotChannelPruner from tinynn.util.util import import_from_path, get_logger @@ -121,13 +119,6 @@ def test_densenet121(self): model = torchvision.models.densenet121(pretrained=False) speed_test(model, torch.randn((1, 3, 224, 224))) - def tearDown(self): - gc.collect() - - @classmethod - def tearDownClass(cls): - gc.collect() - if __name__ == '__main__': unittest.main() diff --git a/tests/test_run.py b/tests/test_run.py index d301efcd..c377f43d 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -13,29 +13,27 @@ # Function to run a test file def run_test(file): try: - result = subprocess.run(["pytest", "-v", file], check=True, capture_output=True, text=True) - print(result.stdout) - return None # No error - except subprocess.CalledProcessError as e: - error_message = f"Tests failed in {file}\n{e.stdout}\n{e.stderr}" - return error_message # Return the error message + subprocess.run(["pytest", "-v", file], check=True) + return True + except subprocess.CalledProcessError: + return False # Run all test files def main(): - errors = [] + error_flag = True for test_file in test_files: error = run_test(test_file) - if error: - errors.append(error) + error_flag = error_flag and error - if errors: - raise Exception("\n".join(errors)) + return error_flag if __name__ == "__main__": - try: - main() - except Exception as e: - print(e) + error_flag = True + for test_file in test_files: + error = run_test(test_file) + error_flag = error_flag and error + + if error_flag: sys.exit(1) From 885f4a42c9fa974070cb56bbf8e4face3714904e Mon Sep 17 00:00:00 2001 From: zk1998 Date: Tue, 18 Jun 2024 12:16:58 +0800 Subject: [PATCH 09/10] [CI] fix typo --- tests/test_run.py | 30 ++++++++++-------------------- 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/tests/test_run.py b/tests/test_run.py index c377f43d..b4969390 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -3,13 +3,6 @@ import glob -# List of test files to run -test_files = glob.glob('tests/*_test.py', recursive=True) - -print("Test files:") -for file in test_files: - print(f" - {file}") - # Function to run a test file def run_test(file): try: @@ -19,21 +12,18 @@ def run_test(file): return False -# Run all test files -def main(): - error_flag = True - for test_file in test_files: - error = run_test(test_file) - error_flag = error_flag and error - - return error_flag +if __name__ == "__main__": + # List of test files to run + test_files = glob.glob('tests/*_test.py', recursive=True) + print("Test files:") + for file in test_files: + print(f" - {file}") -if __name__ == "__main__": - error_flag = True + pass_flag = True for test_file in test_files: - error = run_test(test_file) - error_flag = error_flag and error + passed = run_test(test_file) + pass_flag = pass_flag and passed - if error_flag: + if not pass_flag: sys.exit(1) From 222e2ba6b5ec74a6b43a89c8b411d6e086bd9132 Mon Sep 17 00:00:00 2001 From: zk1998 Date: Tue, 18 Jun 2024 14:14:03 +0800 Subject: [PATCH 10/10] [CI] reset fulltest wrokflow --- .github/workflows/full-test.yml | 19 +++++-------------- .github/workflows/unit-test.yml | 8 ++++++-- tests/common_utils.py | 1 - 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/.github/workflows/full-test.yml b/.github/workflows/full-test.yml index ded74061..c00b1853 100644 --- a/.github/workflows/full-test.yml +++ b/.github/workflows/full-test.yml @@ -1,12 +1,8 @@ name: Full tests on: - push: - branches: - - main - pull_request: - branches: - - main + schedule: + - cron: '00 19 * * *' jobs: unit-test: @@ -15,6 +11,8 @@ jobs: strategy: fail-fast: false matrix: + os: ["ubuntu-latest", "macos-13", "windows-latest"] + vers: [ {pt_ver: "1.6.0", tv_ver: "0.7.0"}, {pt_ver: "1.7.0", tv_ver: "0.8.1"}, {pt_ver: "1.8.0", tv_ver: "0.9.0"}, {pt_ver: "1.9.0", tv_ver: "0.10.0"}, {pt_ver: "1.10.0", tv_ver: "0.11.1"}, {pt_ver: "1.11.0", tv_ver: "0.12.0"}, {pt_ver: "1.12.0", tv_ver: "0.13.0"} ] include: - os: macos-latest vers: @@ -80,16 +78,9 @@ jobs: else pip install black 'ruff<0.0.234' tensorflow pytest scipy interval fi - if [[ "$RUNNER_OS" == "macOS" && "$RUNNER_ARCH" == "ARM64" ]]; then - pip install pytest-forked - fi - name: Lint checks run: python -m ruff . - name: Run tests run: | source activate.sh - if [[ "$RUNNER_OS" == "macOS" && "$RUNNER_ARCH" == "ARM64" ]]; then - python tests/test_run.py - else - pytest tests -v - fi + python tests/test_run.py diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 6a06cbe3..0b52e731 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -1,8 +1,12 @@ name: Unit tests on: - schedule: - - cron: '00 19 * * *' + push: + branches: + - main + pull_request: + branches: + - main jobs: unit-test: diff --git a/tests/common_utils.py b/tests/common_utils.py index 8437e4f7..31e15b2c 100644 --- a/tests/common_utils.py +++ b/tests/common_utils.py @@ -9,7 +9,6 @@ except ImportError: models = None - IS_CI = os.getenv('CI', '') == 'true'