From 51f0f475b926c8629b395f07ffb6f49a5dd9f733 Mon Sep 17 00:00:00 2001 From: asteyo Date: Sat, 6 Nov 2021 18:13:38 +0300 Subject: [PATCH 01/22] profiler is added to runner --- catalyst/runners/runner.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/catalyst/runners/runner.py b/catalyst/runners/runner.py index e0cbd3565b..66819b891b 100644 --- a/catalyst/runners/runner.py +++ b/catalyst/runners/runner.py @@ -12,6 +12,7 @@ from catalyst.callbacks.criterion import CriterionCallback, ICriterionCallback from catalyst.callbacks.misc import CheckRunCallback, TimerCallback, TqdmCallback from catalyst.callbacks.optimizer import IOptimizerCallback, OptimizerCallback +from catalyst.callbacks.profiler import ProfilerCallback from catalyst.callbacks.scheduler import ISchedulerCallback, SchedulerCallback from catalyst.core.callback import Callback from catalyst.core.logger import ILogger @@ -191,6 +192,7 @@ def __init__(self, *args, **kwargs): self._timeit = False self._check = False self._overfit = False + self._profile = False self._load_best_on_end = False @property @@ -303,6 +305,8 @@ def get_callbacks(self, stage: str) -> "OrderedDict[str, Callback]": callbacks["_check"] = CheckRunCallback() if self._overfit and not is_callback_exists(BatchOverfitCallback): callbacks["_overfit"] = BatchOverfitCallback() + if self._profile and not is_callback_exists(ProfilerCallback): + callbacks["_profile"] = ProfilerCallback() if self._logdir is not None and not is_callback_exists(ICheckpointCallback): callbacks["_checkpoint"] = CheckpointCallback( @@ -344,6 +348,7 @@ def train( timeit: bool = False, check: bool = False, overfit: bool = False, + profile: bool = False, load_best_on_end: bool = False, # engine extra params, fp16: bool = False, @@ -510,6 +515,7 @@ def on_loader_end(self, runner): self._timeit = timeit self._check = check self._overfit = overfit + self._profile = profile self._load_best_on_end = load_best_on_end # run self.run() From cb5b4eafccbbf49f0c926ebd6d0eaccea9df00a5 Mon Sep 17 00:00:00 2001 From: asteyo Date: Mon, 8 Nov 2021 11:45:36 +0300 Subject: [PATCH 02/22] profiler minimal example import fix --- catalyst/callbacks/profiler.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/catalyst/callbacks/profiler.py b/catalyst/callbacks/profiler.py index 5690b00153..f4e4c4da3a 100644 --- a/catalyst/callbacks/profiler.py +++ b/catalyst/callbacks/profiler.py @@ -51,7 +51,6 @@ class ProfilerCallback(Callback): from catalyst import dl from catalyst.data import ToTensor from catalyst.contrib.datasets import MNIST - from catalyst.contrib.nn.modules import Flatten loaders = { "train": DataLoader( @@ -64,7 +63,7 @@ class ProfilerCallback(Callback): ), } - model = nn.Sequential(Flatten(), nn.Linear(784, 512), nn.ReLU(), nn.Linear(512, 10)) + model = nn.Sequential(nn.Flatten(), nn.Linear(784, 512), nn.ReLU(), nn.Linear(512, 10)) criterion = nn.CrossEntropyLoss() optimizer = torch.optim.Adam(model.parameters(), lr=1e-2) runner = dl.SupervisedRunner() From de4c5f414a065f3088106f6090d8e722a83a361c Mon Sep 17 00:00:00 2001 From: asteyo Date: Mon, 8 Nov 2021 11:50:03 +0300 Subject: [PATCH 03/22] runner docs updated --- catalyst/runners/runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/catalyst/runners/runner.py b/catalyst/runners/runner.py index 66819b891b..51dc33800a 100644 --- a/catalyst/runners/runner.py +++ b/catalyst/runners/runner.py @@ -390,6 +390,8 @@ def train( overfit: if True, then takes only one batch per loader for model overfitting, for advance usage please check ``BatchOverfitCallback`` + profile: if True, then uses ProfilerCallback, for advance usage please check + ``ProfilerCallback`` load_best_on_end: if True, Runner will load best checkpoint state (model, optimizer, etc) according to validation metrics. Requires specified ``logdir``. From aad700a273994879ee7b95815973e78e17a205d9 Mon Sep 17 00:00:00 2001 From: asteyo Date: Mon, 8 Nov 2021 12:16:24 +0300 Subject: [PATCH 04/22] logdir added to profiler --- catalyst/runners/runner.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/catalyst/runners/runner.py b/catalyst/runners/runner.py index 51dc33800a..bbabffb7f0 100644 --- a/catalyst/runners/runner.py +++ b/catalyst/runners/runner.py @@ -306,7 +306,11 @@ def get_callbacks(self, stage: str) -> "OrderedDict[str, Callback]": if self._overfit and not is_callback_exists(BatchOverfitCallback): callbacks["_overfit"] = BatchOverfitCallback() if self._profile and not is_callback_exists(ProfilerCallback): - callbacks["_profile"] = ProfilerCallback() + callbacks["_profile"] = ProfilerCallback( + on_trace_ready=torch.profiler.tensorboard_trace_handler( + os.path.join(self._logdir, "tb_profile") + ) + ) if self._logdir is not None and not is_callback_exists(ICheckpointCallback): callbacks["_checkpoint"] = CheckpointCallback( From 0fc2564ba6f1672c35ab8b5ea2b6a2e02650c0ba Mon Sep 17 00:00:00 2001 From: asteyo Date: Mon, 8 Nov 2021 12:24:27 +0300 Subject: [PATCH 05/22] tensorboard_path fix --- catalyst/runners/runner.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/catalyst/runners/runner.py b/catalyst/runners/runner.py index bbabffb7f0..1641078961 100644 --- a/catalyst/runners/runner.py +++ b/catalyst/runners/runner.py @@ -307,9 +307,7 @@ def get_callbacks(self, stage: str) -> "OrderedDict[str, Callback]": callbacks["_overfit"] = BatchOverfitCallback() if self._profile and not is_callback_exists(ProfilerCallback): callbacks["_profile"] = ProfilerCallback( - on_trace_ready=torch.profiler.tensorboard_trace_handler( - os.path.join(self._logdir, "tb_profile") - ) + tensorboard_path=os.path.join(self._logdir, "tb_profile") ) if self._logdir is not None and not is_callback_exists(ICheckpointCallback): From 95e5ade32da76e30dcd82e0c16c164b7c4570995 Mon Sep 17 00:00:00 2001 From: asteyo Date: Thu, 11 Nov 2021 09:23:24 +0300 Subject: [PATCH 06/22] runner profiler callback updated --- catalyst/runners/runner.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/catalyst/runners/runner.py b/catalyst/runners/runner.py index 1641078961..66b46520c6 100644 --- a/catalyst/runners/runner.py +++ b/catalyst/runners/runner.py @@ -307,7 +307,19 @@ def get_callbacks(self, stage: str) -> "OrderedDict[str, Callback]": callbacks["_overfit"] = BatchOverfitCallback() if self._profile and not is_callback_exists(ProfilerCallback): callbacks["_profile"] = ProfilerCallback( - tensorboard_path=os.path.join(self._logdir, "tb_profile") + tensorboard_path=os.path.join(self._logdir, "tb_profile"), + loader_key="train", + profiler_kwargs=dict( + activities=[ + torch.profiler.ProfilerActivity.CPU, + torch.profiler.ProfilerActivity.CUDA, + ], + on_trace_ready=torch.profiler.tensorboard_trace_handler( + os.path.join(self._logdir, "tb_profile") + ), + with_stack=True, + with_flops=True, + ), ) if self._logdir is not None and not is_callback_exists(ICheckpointCallback): From 51a899d9ab73aa62ccbb1fb9c845f9b5e2893c2f Mon Sep 17 00:00:00 2001 From: asteyo Date: Thu, 11 Nov 2021 09:59:15 +0300 Subject: [PATCH 07/22] fix codestyle --- catalyst/runners/runner.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/catalyst/runners/runner.py b/catalyst/runners/runner.py index 66b46520c6..4fba4bdfa6 100644 --- a/catalyst/runners/runner.py +++ b/catalyst/runners/runner.py @@ -309,17 +309,17 @@ def get_callbacks(self, stage: str) -> "OrderedDict[str, Callback]": callbacks["_profile"] = ProfilerCallback( tensorboard_path=os.path.join(self._logdir, "tb_profile"), loader_key="train", - profiler_kwargs=dict( - activities=[ + profiler_kwargs={ + activities: [ torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA, ], - on_trace_ready=torch.profiler.tensorboard_trace_handler( + on_trace_ready: torch.profiler.tensorboard_trace_handler( os.path.join(self._logdir, "tb_profile") ), - with_stack=True, - with_flops=True, - ), + with_stack: True, + with_flops: True, + }, ) if self._logdir is not None and not is_callback_exists(ICheckpointCallback): From 5b6ed803b1f2b53d4e4f134b370717b209005c4a Mon Sep 17 00:00:00 2001 From: asteyo Date: Thu, 11 Nov 2021 10:24:58 +0300 Subject: [PATCH 08/22] key_loader fix --- catalyst/runners/runner.py | 1 - 1 file changed, 1 deletion(-) diff --git a/catalyst/runners/runner.py b/catalyst/runners/runner.py index 79a4b019ed..e21c546659 100644 --- a/catalyst/runners/runner.py +++ b/catalyst/runners/runner.py @@ -309,7 +309,6 @@ def get_callbacks(self, stage: str) -> "OrderedDict[str, Callback]": if self._profile and not is_callback_exists(ProfilerCallback): callbacks["_profile"] = ProfilerCallback( tensorboard_path=os.path.join(self._logdir, "tb_profile"), - loader_key="train", profiler_kwargs={ activities: [ torch.profiler.ProfilerActivity.CPU, From d4d3b8164f8bb9c7ffa1dc07a94100e4f5dffdde Mon Sep 17 00:00:00 2001 From: asteyo Date: Thu, 11 Nov 2021 10:37:49 +0300 Subject: [PATCH 09/22] profiler_kwargs fix --- catalyst/runners/runner.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/catalyst/runners/runner.py b/catalyst/runners/runner.py index d99cdab675..8918d22343 100644 --- a/catalyst/runners/runner.py +++ b/catalyst/runners/runner.py @@ -310,15 +310,15 @@ def get_callbacks(self, stage: str) -> "OrderedDict[str, Callback]": callbacks["_profile"] = ProfilerCallback( tensorboard_path=os.path.join(self._logdir, "tb_profile"), profiler_kwargs={ - activities: [ + "activities": [ torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA, ], - on_trace_ready: torch.profiler.tensorboard_trace_handler( + "on_trace_ready": torch.profiler.tensorboard_trace_handler( os.path.join(self._logdir, "tb_profile") ), - with_stack: True, - with_flops: True, + "with_stack": True, + "with_flops": True, }, ) From 64715ef58912ce5d53a89b3eff879eecdd33a3a8 Mon Sep 17 00:00:00 2001 From: asteyo Date: Thu, 11 Nov 2021 12:47:51 +0300 Subject: [PATCH 10/22] profiler updated --- catalyst/callbacks/profiler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/catalyst/callbacks/profiler.py b/catalyst/callbacks/profiler.py index f4e4c4da3a..93466ce9f2 100644 --- a/catalyst/callbacks/profiler.py +++ b/catalyst/callbacks/profiler.py @@ -185,6 +185,9 @@ def on_loader_start(self, runner: IRunner) -> None: Args: runner: current runner """ + if self.loader_key is None: + self.loader_key = runner.loader_key + self._enter_profiler(runner) def on_loader_end(self, runner: IRunner) -> None: From 7db8df529fa8091222e44eefa7ee3be37d84ad68 Mon Sep 17 00:00:00 2001 From: asteyo Date: Thu, 11 Nov 2021 13:26:08 +0300 Subject: [PATCH 11/22] profiler on_experiment_start method refactoring --- catalyst/callbacks/profiler.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/catalyst/callbacks/profiler.py b/catalyst/callbacks/profiler.py index 93466ce9f2..df2569703e 100644 --- a/catalyst/callbacks/profiler.py +++ b/catalyst/callbacks/profiler.py @@ -121,16 +121,6 @@ def __init__( self.profiler = None self.stats = None - def on_experiment_start(self, runner: IRunner) -> None: - """ - On batch end action - - Args: - runner: current runner - """ - if self.loader_key is None: - self.loader_key = runner.loader_key # use first loader for profile - def _should_use_profiler(self, loader_key: str, epoch: int): if self.loader_key == loader_key and self.epoch == epoch: if self.num_batches is not None: @@ -186,7 +176,7 @@ def on_loader_start(self, runner: IRunner) -> None: runner: current runner """ if self.loader_key is None: - self.loader_key = runner.loader_key + self.loader_key = runner.loader_key # use first loader for profile self._enter_profiler(runner) From 5f16fed1f1b8601f27e806ed8c630f545c88c6cd Mon Sep 17 00:00:00 2001 From: asteyo Date: Thu, 11 Nov 2021 13:33:40 +0300 Subject: [PATCH 12/22] logs path fixed --- catalyst/runners/runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/runners/runner.py b/catalyst/runners/runner.py index 8918d22343..b01cbaa593 100644 --- a/catalyst/runners/runner.py +++ b/catalyst/runners/runner.py @@ -308,14 +308,14 @@ def get_callbacks(self, stage: str) -> "OrderedDict[str, Callback]": callbacks["_overfit"] = BatchOverfitCallback() if self._profile and not is_callback_exists(ProfilerCallback): callbacks["_profile"] = ProfilerCallback( - tensorboard_path=os.path.join(self._logdir, "tb_profile"), + tensorboard_path=os.path.join(self._logdir, "/tb_profile"), profiler_kwargs={ "activities": [ torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA, ], "on_trace_ready": torch.profiler.tensorboard_trace_handler( - os.path.join(self._logdir, "tb_profile") + os.path.join(self._logdir, "/tb_profile") ), "with_stack": True, "with_flops": True, From bd03a44cfc272d842f194a2a3b737bccaaaff0e3 Mon Sep 17 00:00:00 2001 From: asteyo Date: Thu, 11 Nov 2021 13:36:32 +0300 Subject: [PATCH 13/22] Revert "logs path fixed" This reverts commit 5f16fed1f1b8601f27e806ed8c630f545c88c6cd. --- catalyst/runners/runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/catalyst/runners/runner.py b/catalyst/runners/runner.py index b01cbaa593..8918d22343 100644 --- a/catalyst/runners/runner.py +++ b/catalyst/runners/runner.py @@ -308,14 +308,14 @@ def get_callbacks(self, stage: str) -> "OrderedDict[str, Callback]": callbacks["_overfit"] = BatchOverfitCallback() if self._profile and not is_callback_exists(ProfilerCallback): callbacks["_profile"] = ProfilerCallback( - tensorboard_path=os.path.join(self._logdir, "/tb_profile"), + tensorboard_path=os.path.join(self._logdir, "tb_profile"), profiler_kwargs={ "activities": [ torch.profiler.ProfilerActivity.CPU, torch.profiler.ProfilerActivity.CUDA, ], "on_trace_ready": torch.profiler.tensorboard_trace_handler( - os.path.join(self._logdir, "/tb_profile") + os.path.join(self._logdir, "tb_profile") ), "with_stack": True, "with_flops": True, From 45fc5b66000f58a857559b2d6ad54b390f3cc7f8 Mon Sep 17 00:00:00 2001 From: asteyo Date: Sun, 21 Nov 2021 23:21:31 +0300 Subject: [PATCH 14/22] profiler test is added --- tests/pipelines/test_runner.py | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 tests/pipelines/test_runner.py diff --git a/tests/pipelines/test_runner.py b/tests/pipelines/test_runner.py new file mode 100644 index 0000000000..ae59e42faa --- /dev/null +++ b/tests/pipelines/test_runner.py @@ -0,0 +1,40 @@ +import os + +import torch +from torch import nn +from torch.utils.data import DataLoader + +from catalyst import dl +from catalyst.contrib.data import ImageToTensor +from catalyst.contrib.datasets import MNIST + + +def train_experiment(): + loaders = { + "train": DataLoader( + MNIST(os.getcwd(), train=False, download=True, transform=ImageToTensor()), + batch_size=32, + ), + "valid": DataLoader( + MNIST(os.getcwd(), train=False, download=True, transform=ImageToTensor()), + batch_size=32, + ), + } + model = nn.Sequential(nn.Flatten(), nn.Linear(784, 512), nn.ReLU(), nn.Linear(512, 10)) + criterion = nn.CrossEntropyLoss() + optimizer = torch.optim.Adam(model.parameters(), lr=1e-2) + runner = dl.SupervisedRunner() + runner.train( + model=model, + loaders=loaders, + criterion=criterion, + optimizer=optimizer, + num_epochs=5, + logdir="./logs", + profile=True, + ) + + +def test_profiler(): + train_experiment() + assert os.path.isdir("./logs/tb_profile") and not len(os.listdir("./logs/tb_profile")) == 0 From 1d69e8c7ce29bcb0c4a00108fd55401b102a373b Mon Sep 17 00:00:00 2001 From: asteyo Date: Sun, 21 Nov 2021 23:32:36 +0300 Subject: [PATCH 15/22] flake8 update --- tests/pipelines/test_runner.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/pipelines/test_runner.py b/tests/pipelines/test_runner.py index ae59e42faa..3cb0c9d9ef 100644 --- a/tests/pipelines/test_runner.py +++ b/tests/pipelines/test_runner.py @@ -1,3 +1,5 @@ +# flake8: noqa + import os import torch From a8c4ea72e97abc662689d95e209bd2f7f2622a4e Mon Sep 17 00:00:00 2001 From: asteyo Date: Mon, 22 Nov 2021 10:31:24 +0300 Subject: [PATCH 16/22] profiler availability check is added --- tests/pipelines/test_runner.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/pipelines/test_runner.py b/tests/pipelines/test_runner.py index 3cb0c9d9ef..feb87c53af 100644 --- a/tests/pipelines/test_runner.py +++ b/tests/pipelines/test_runner.py @@ -2,6 +2,8 @@ import os +from pytest import mark + import torch from torch import nn from torch.utils.data import DataLoader @@ -11,6 +13,15 @@ from catalyst.contrib.datasets import MNIST +def _is_profile_available(): + try: + from torch import profiler # noqa: F401 + + return True + except ImportError: + return False + + def train_experiment(): loaders = { "train": DataLoader( @@ -36,7 +47,7 @@ def train_experiment(): profile=True, ) - +@mark.skipif(not _is_profile_available(), reason="Torch profiler is not available") def test_profiler(): train_experiment() assert os.path.isdir("./logs/tb_profile") and not len(os.listdir("./logs/tb_profile")) == 0 From 64b390e9add35ff4c28618cd6f465679b715fb77 Mon Sep 17 00:00:00 2001 From: asteyo Date: Mon, 22 Nov 2021 10:31:46 +0300 Subject: [PATCH 17/22] profiler reqs is added --- requirements/requirements-profiler.txt | 1 + setup.py | 1 + 2 files changed, 2 insertions(+) create mode 100644 requirements/requirements-profiler.txt diff --git a/requirements/requirements-profiler.txt b/requirements/requirements-profiler.txt new file mode 100644 index 0000000000..e2dd6b1ed4 --- /dev/null +++ b/requirements/requirements-profiler.txt @@ -0,0 +1 @@ +torch_tb_profiler \ No newline at end of file diff --git a/setup.py b/setup.py index 23aa3bfafa..2353237053 100755 --- a/setup.py +++ b/setup.py @@ -53,6 +53,7 @@ def load_version(): "onnx-gpu": load_requirements("requirements/requirements-onnx-gpu.txt"), "onnx": load_requirements("requirements/requirements-onnx.txt"), "optuna": load_requirements("requirements/requirements-optuna.txt"), + "profiler": load_requirements("requirements/requirements-profiler.txt"), "wandb": load_requirements("requirements/requirements-wandb.txt"), # "xla": load_requirements("requirements/requirements-xla.txt"), } From f6c2d67f2329b0b0cda9f84adc72b9df23f98fe9 Mon Sep 17 00:00:00 2001 From: asteyo Date: Mon, 22 Nov 2021 10:34:39 +0300 Subject: [PATCH 18/22] setup docs updated --- docs/faq/settings.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/faq/settings.rst b/docs/faq/settings.rst index b5dbe23d74..f0f177256f 100644 --- a/docs/faq/settings.rst +++ b/docs/faq/settings.rst @@ -24,6 +24,7 @@ Nevertheless, there are much more availabe: pip install catalyst[onnx-gpu] # + onnx, onnxruntime-gpu pip install catalyst[onnx] # + onnx, onnxruntime pip install catalyst[optuna] # + optuna + pip install catalyst[profiler] # + profiler pip install catalyst[wandb] # + wandb pip install catalyst[all] # + catalyst[cv], catalyst[ml], catalyst[optuna] From 023d398b87bd840cb406fdd4cb1b668de808931b Mon Sep 17 00:00:00 2001 From: asteyo Date: Mon, 22 Nov 2021 10:36:00 +0300 Subject: [PATCH 19/22] codestyle fix --- tests/pipelines/test_runner.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pipelines/test_runner.py b/tests/pipelines/test_runner.py index feb87c53af..b977efd754 100644 --- a/tests/pipelines/test_runner.py +++ b/tests/pipelines/test_runner.py @@ -47,6 +47,7 @@ def train_experiment(): profile=True, ) + @mark.skipif(not _is_profile_available(), reason="Torch profiler is not available") def test_profiler(): train_experiment() From dc6371a0582aa278ed6ac516d1420b8502611874 Mon Sep 17 00:00:00 2001 From: asteyo Date: Mon, 22 Nov 2021 19:44:23 +0300 Subject: [PATCH 20/22] dl_cpu.yml refactoring --- .github/workflows/dl_cpu.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/dl_cpu.yml b/.github/workflows/dl_cpu.yml index 59b4ea12ea..323a5cdca4 100644 --- a/.github/workflows/dl_cpu.yml +++ b/.github/workflows/dl_cpu.yml @@ -63,12 +63,7 @@ jobs: - name: set minimal dependencies if: ${{ matrix.requirements == 'minimal' }} - run: | - python -c "req = open('./requirements/requirements.txt').read().replace('>', '=') ; open('./requirements/requirements.txt', 'w').write(req)" - python -c "req = open('./requirements/requirements-cv.txt').read().replace('>', '=') ; open('./requirements/requirements-cv.txt', 'w').write(req)" - python -c "req = open('./requirements/requirements-ml.txt').read().replace('>', '=') ; open('./requirements/requirements-ml.txt', 'w').write(req)" - python -c "req = open('./requirements/requirements-neptune.txt').read().replace('>', '=') ; open('./requirements/requirements-neptune.txt', 'w').write(req)" - python -c "req = open('./requirements/requirements-albu.txt').read().replace('>', '=') ; open('./requirements/requirements-albu.txt', 'w').write(req)" + run: $( printf "python -c \"req = open('"%s"').read().replace('>', '=') ; open('"%s"', 'w').write(req)\"\n" $(ls -d ./requirements/*) ) # https://github.com/actions/cache/blob/master/examples.md # Note: This uses an internal pip API and may not always work @@ -97,7 +92,7 @@ jobs: - name: install dependencies run: | # python -m pip install --upgrade --user pip - pip install -r ./requirements/requirements.txt -r ./requirements/requirements-cv.txt -r ./requirements/requirements-dev.txt -r ./requirements/requirements-hydra.txt -r ./requirements/requirements-ml.txt -r ./requirements/requirements-optuna.txt -r ./requirements/requirements-mlflow.txt -r ./requirements/requirements-nifti.txt -r ./requirements/requirements-neptune.txt -r ./requirements/requirements-albu.txt + pip install$( printf " -r "%s $(ls -d ./requirements/*) ) python --version pip --version pip list From 028c407125f9b5840429646335dfd37424037ea0 Mon Sep 17 00:00:00 2001 From: asteyo Date: Mon, 22 Nov 2021 22:05:01 +0300 Subject: [PATCH 21/22] Revert "dl_cpu.yml refactoring" This reverts commit dc6371a0582aa278ed6ac516d1420b8502611874. --- .github/workflows/dl_cpu.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/dl_cpu.yml b/.github/workflows/dl_cpu.yml index 323a5cdca4..59b4ea12ea 100644 --- a/.github/workflows/dl_cpu.yml +++ b/.github/workflows/dl_cpu.yml @@ -63,7 +63,12 @@ jobs: - name: set minimal dependencies if: ${{ matrix.requirements == 'minimal' }} - run: $( printf "python -c \"req = open('"%s"').read().replace('>', '=') ; open('"%s"', 'w').write(req)\"\n" $(ls -d ./requirements/*) ) + run: | + python -c "req = open('./requirements/requirements.txt').read().replace('>', '=') ; open('./requirements/requirements.txt', 'w').write(req)" + python -c "req = open('./requirements/requirements-cv.txt').read().replace('>', '=') ; open('./requirements/requirements-cv.txt', 'w').write(req)" + python -c "req = open('./requirements/requirements-ml.txt').read().replace('>', '=') ; open('./requirements/requirements-ml.txt', 'w').write(req)" + python -c "req = open('./requirements/requirements-neptune.txt').read().replace('>', '=') ; open('./requirements/requirements-neptune.txt', 'w').write(req)" + python -c "req = open('./requirements/requirements-albu.txt').read().replace('>', '=') ; open('./requirements/requirements-albu.txt', 'w').write(req)" # https://github.com/actions/cache/blob/master/examples.md # Note: This uses an internal pip API and may not always work @@ -92,7 +97,7 @@ jobs: - name: install dependencies run: | # python -m pip install --upgrade --user pip - pip install$( printf " -r "%s $(ls -d ./requirements/*) ) + pip install -r ./requirements/requirements.txt -r ./requirements/requirements-cv.txt -r ./requirements/requirements-dev.txt -r ./requirements/requirements-hydra.txt -r ./requirements/requirements-ml.txt -r ./requirements/requirements-optuna.txt -r ./requirements/requirements-mlflow.txt -r ./requirements/requirements-nifti.txt -r ./requirements/requirements-neptune.txt -r ./requirements/requirements-albu.txt python --version pip --version pip list From 9e0fe35e613a04218251c7e221e723412c9e070d Mon Sep 17 00:00:00 2001 From: asteyo Date: Mon, 22 Nov 2021 22:09:29 +0300 Subject: [PATCH 22/22] profiler added to dl_cpu.yml --- .github/workflows/dl_cpu.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/dl_cpu.yml b/.github/workflows/dl_cpu.yml index 59b4ea12ea..d44e5407ef 100644 --- a/.github/workflows/dl_cpu.yml +++ b/.github/workflows/dl_cpu.yml @@ -69,6 +69,7 @@ jobs: python -c "req = open('./requirements/requirements-ml.txt').read().replace('>', '=') ; open('./requirements/requirements-ml.txt', 'w').write(req)" python -c "req = open('./requirements/requirements-neptune.txt').read().replace('>', '=') ; open('./requirements/requirements-neptune.txt', 'w').write(req)" python -c "req = open('./requirements/requirements-albu.txt').read().replace('>', '=') ; open('./requirements/requirements-albu.txt', 'w').write(req)" + python -c "req = open('./requirements/requirements-profiler.txt').read().replace('>', '=') ; open('./requirements/requirements-profiler.txt', 'w').write(req)" # https://github.com/actions/cache/blob/master/examples.md # Note: This uses an internal pip API and may not always work @@ -97,7 +98,7 @@ jobs: - name: install dependencies run: | # python -m pip install --upgrade --user pip - pip install -r ./requirements/requirements.txt -r ./requirements/requirements-cv.txt -r ./requirements/requirements-dev.txt -r ./requirements/requirements-hydra.txt -r ./requirements/requirements-ml.txt -r ./requirements/requirements-optuna.txt -r ./requirements/requirements-mlflow.txt -r ./requirements/requirements-nifti.txt -r ./requirements/requirements-neptune.txt -r ./requirements/requirements-albu.txt + pip install -r ./requirements/requirements.txt -r ./requirements/requirements-cv.txt -r ./requirements/requirements-dev.txt -r ./requirements/requirements-hydra.txt -r ./requirements/requirements-ml.txt -r ./requirements/requirements-optuna.txt -r ./requirements/requirements-mlflow.txt -r ./requirements/requirements-nifti.txt -r ./requirements/requirements-neptune.txt -r ./requirements/requirements-albu.txt -r ./requirements/requirements-profiler.txt python --version pip --version pip list