From ce4f2bf4d2c8ff6317a0a739e1604116c14c7413 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Sat, 28 Sep 2024 11:28:18 -0700 Subject: [PATCH 1/9] Fix Evaluate to a version --- install/install_requirements.sh | 6 ------ install/requirements.txt | 4 ++++ torchchat/model.py | 1 - torchchat/utils/scripts/install_utils.sh | 6 ------ 4 files changed, 4 insertions(+), 13 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index fa921cc69..216737a6f 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -99,9 +99,3 @@ if [[ -x "$(command -v nvidia-smi)" ]]; then $PYTHON_EXECUTABLE torchchat/utils/scripts/patch_triton.py ) fi - - -( - set -x - $PIP_EXECUTABLE install evaluate=="0.4.3" lm-eval=="0.4.2" psutil=="6.0.0" -) diff --git a/install/requirements.txt b/install/requirements.txt index 3329563b4..a0c66540b 100644 --- a/install/requirements.txt +++ b/install/requirements.txt @@ -18,6 +18,10 @@ blobfile tomli >= 1.1.0 ; python_version < "3.11" openai +# For Evaluation +evaluate==0.4.2 +lm-eval==0.4.2 + # Build tools wheel cmake>=3.24 diff --git a/torchchat/model.py b/torchchat/model.py index 0f60ea318..355df9124 100644 --- a/torchchat/model.py +++ b/torchchat/model.py @@ -32,7 +32,6 @@ from torch.nn import functional as F # TODO: remove this after we figure out where in torchtune an `evaluate` module # is being imported, which is being confused with huggingface's `evaluate``. -import lm_eval # noqa from torchtune.models.clip import clip_vision_encoder from torchtune.models.llama3_1._component_builders import llama3_1 as llama3_1_builder from torchtune.models.llama3_2_vision._component_builders import ( diff --git a/torchchat/utils/scripts/install_utils.sh b/torchchat/utils/scripts/install_utils.sh index ec9677373..48a8d6d54 100644 --- a/torchchat/utils/scripts/install_utils.sh +++ b/torchchat/utils/scripts/install_utils.sh @@ -94,12 +94,6 @@ install_executorch_python_libs() { bash ./install_requirements.sh --pybind xnnpack fi - # TODO: figure out the root cause of 'AttributeError: module 'evaluate' - # has no attribute 'utils'' error from evaluate CI jobs and remove - # `import lm_eval` from torchchat.py since it requires a specific version - # of numpy. - pip install numpy=='1.26.4' - pip3 list popd } From 4f2fbe7c5adc0b1926fb73d1e0871ee244d9c88d Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Sat, 28 Sep 2024 11:30:29 -0700 Subject: [PATCH 2/9] clean comments --- torchchat/model.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/torchchat/model.py b/torchchat/model.py index 355df9124..451dd83ac 100644 --- a/torchchat/model.py +++ b/torchchat/model.py @@ -30,8 +30,6 @@ SequenceParallel, ) from torch.nn import functional as F -# TODO: remove this after we figure out where in torchtune an `evaluate` module -# is being imported, which is being confused with huggingface's `evaluate``. from torchtune.models.clip import clip_vision_encoder from torchtune.models.llama3_1._component_builders import llama3_1 as llama3_1_builder from torchtune.models.llama3_2_vision._component_builders import ( From 143ab0b0cb342ad25a8630d5e6a109879dc28dcf Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Sun, 29 Sep 2024 14:14:55 -0700 Subject: [PATCH 3/9] Adding MPS Support for LLama3.2 11B Multimodal; Bump torchtune 9.28.24 --- install/install_requirements.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index fa921cc69..cd6c302c2 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -52,6 +52,9 @@ PYTORCH_NIGHTLY_VERSION=dev20240901 # Nightly version for torchvision VISION_NIGHTLY_VERSION=dev20240901 +# Nightly version for torchtune +TUNE_NIGHTLY_VERSION=dev20240928 + # Uninstall triton, as nightly will depend on pytorch-triton, which is one and the same ( set -x @@ -72,6 +75,7 @@ fi REQUIREMENTS_TO_INSTALL=( torch=="2.5.0.${PYTORCH_NIGHTLY_VERSION}" torchvision=="0.20.0.${VISION_NIGHTLY_VERSION}" + torchtune=="0.3.0.${TUNE_NIGHTLY_VERSION}" ) # Install the requirements. --extra-index-url tells pip to look for package @@ -87,12 +91,6 @@ REQUIREMENTS_TO_INSTALL=( $PIP_EXECUTABLE install torchao=="0.5.0" ) -# Rely on the latest tochtune for flamingo support -( - set -x - $PIP_EXECUTABLE install -I git+https://github.com/pytorch/torchtune.git@d002d45e3ec700fa770d9dcc61b02c59e2507bf6 -) - if [[ -x "$(command -v nvidia-smi)" ]]; then ( set -x From a0a92baadd74c8f5a49d8e73f693df0bdf707467 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Sun, 29 Sep 2024 15:45:19 -0700 Subject: [PATCH 4/9] Lock onto a pre BC breaking version of datasets --- install/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/install/requirements.txt b/install/requirements.txt index a0c66540b..7243d7733 100644 --- a/install/requirements.txt +++ b/install/requirements.txt @@ -19,6 +19,7 @@ tomli >= 1.1.0 ; python_version < "3.11" openai # For Evaluation +datasets<3.0 evaluate==0.4.2 lm-eval==0.4.2 From cc37640c61de82547635f57470bb23aae02afdfb Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Sun, 29 Sep 2024 16:00:37 -0700 Subject: [PATCH 5/9] Bump lm_eval import before torchtune --- torchchat/usages/eval.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/torchchat/usages/eval.py b/torchchat/usages/eval.py index 5993c3781..346f3e4b3 100644 --- a/torchchat/usages/eval.py +++ b/torchchat/usages/eval.py @@ -3,9 +3,10 @@ # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. -import argparse from typing import Callable, Optional +import lm_eval + import torch import torch._dynamo.config import torch._inductor.config @@ -16,7 +17,6 @@ BuilderArgs, TokenizerArgs, ) -from torchchat.cli.cli import add_arguments_for_verb, arg_init from torchchat.model import Model from torchchat.utils.build_utils import set_precision @@ -28,8 +28,6 @@ torch._inductor.config.triton.cudagraphs = True torch._dynamo.config.cache_size_limit = 100000 -import lm_eval - from lm_eval.evaluator import evaluate from lm_eval.models.huggingface import HFLM as eval_wrapper from lm_eval.tasks import get_task_dict From f68084b8e8462ff60cac7588465fa9282a166d8e Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Sun, 29 Sep 2024 16:50:01 -0700 Subject: [PATCH 6/9] Minimize changes --- install/install_requirements.sh | 6 ++++++ install/requirements.txt | 5 ----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index 573864c12..cd6c302c2 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -97,3 +97,9 @@ if [[ -x "$(command -v nvidia-smi)" ]]; then $PYTHON_EXECUTABLE torchchat/utils/scripts/patch_triton.py ) fi + + +( + set -x + $PIP_EXECUTABLE install evaluate=="0.4.3" lm-eval=="0.4.2" psutil=="6.0.0" +) diff --git a/install/requirements.txt b/install/requirements.txt index 7243d7733..3329563b4 100644 --- a/install/requirements.txt +++ b/install/requirements.txt @@ -18,11 +18,6 @@ blobfile tomli >= 1.1.0 ; python_version < "3.11" openai -# For Evaluation -datasets<3.0 -evaluate==0.4.2 -lm-eval==0.4.2 - # Build tools wheel cmake>=3.24 From b9ecdfa2511edaba62aeca0eafceaa978f99f0ac Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Sun, 29 Sep 2024 16:57:03 -0700 Subject: [PATCH 7/9] Minimize changes --- torchchat/model.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/torchchat/model.py b/torchchat/model.py index 451dd83ac..0175c1d54 100644 --- a/torchchat/model.py +++ b/torchchat/model.py @@ -29,6 +29,9 @@ RowwiseParallel, SequenceParallel, ) +# TODO: remove this after we figure out where in torchtune an `evaluate` module +# is being imported, which is being confused with huggingface's `evaluate``. +import lm_eval # noqa from torch.nn import functional as F from torchtune.models.clip import clip_vision_encoder from torchtune.models.llama3_1._component_builders import llama3_1 as llama3_1_builder From 37a814137aaaacb97dad083b1208ae28f1a7586d Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Sun, 29 Sep 2024 17:19:10 -0700 Subject: [PATCH 8/9] Test local import --- torchchat/model.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/torchchat/model.py b/torchchat/model.py index 0175c1d54..3103bd1fe 100644 --- a/torchchat/model.py +++ b/torchchat/model.py @@ -29,11 +29,7 @@ RowwiseParallel, SequenceParallel, ) -# TODO: remove this after we figure out where in torchtune an `evaluate` module -# is being imported, which is being confused with huggingface's `evaluate``. -import lm_eval # noqa from torch.nn import functional as F -from torchtune.models.clip import clip_vision_encoder from torchtune.models.llama3_1._component_builders import llama3_1 as llama3_1_builder from torchtune.models.llama3_2_vision._component_builders import ( llama3_2_vision_decoder, @@ -227,6 +223,9 @@ def _flamingo(cls): @classmethod def _llava(cls): + # TODO: Import locally until we find where in torchtune an `evaluate` module + # is being imported, which is being confused with huggingface's `evaluate``. + from torchtune.models.clip import clip_vision_encoder return cls( model_type=ModelType.Llava, modules={ From 83090cdfa01c4e06d8f12135e5171fd91f8cad98 Mon Sep 17 00:00:00 2001 From: Jack-Khuu Date: Sun, 29 Sep 2024 17:57:29 -0700 Subject: [PATCH 9/9] Just Try Catch --- install/install_requirements.sh | 10 ++++++---- torchchat/model.py | 12 +++++++++--- torchchat/usages/eval.py | 6 ++++-- torchchat/utils/scripts/install_utils.sh | 6 ++++++ 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/install/install_requirements.sh b/install/install_requirements.sh index cd6c302c2..fa921cc69 100755 --- a/install/install_requirements.sh +++ b/install/install_requirements.sh @@ -52,9 +52,6 @@ PYTORCH_NIGHTLY_VERSION=dev20240901 # Nightly version for torchvision VISION_NIGHTLY_VERSION=dev20240901 -# Nightly version for torchtune -TUNE_NIGHTLY_VERSION=dev20240928 - # Uninstall triton, as nightly will depend on pytorch-triton, which is one and the same ( set -x @@ -75,7 +72,6 @@ fi REQUIREMENTS_TO_INSTALL=( torch=="2.5.0.${PYTORCH_NIGHTLY_VERSION}" torchvision=="0.20.0.${VISION_NIGHTLY_VERSION}" - torchtune=="0.3.0.${TUNE_NIGHTLY_VERSION}" ) # Install the requirements. --extra-index-url tells pip to look for package @@ -91,6 +87,12 @@ REQUIREMENTS_TO_INSTALL=( $PIP_EXECUTABLE install torchao=="0.5.0" ) +# Rely on the latest tochtune for flamingo support +( + set -x + $PIP_EXECUTABLE install -I git+https://github.com/pytorch/torchtune.git@d002d45e3ec700fa770d9dcc61b02c59e2507bf6 +) + if [[ -x "$(command -v nvidia-smi)" ]]; then ( set -x diff --git a/torchchat/model.py b/torchchat/model.py index 3103bd1fe..93a0d7366 100644 --- a/torchchat/model.py +++ b/torchchat/model.py @@ -30,6 +30,15 @@ SequenceParallel, ) from torch.nn import functional as F + +try: + # TODO: remove this after we figure out where in torchtune an `evaluate` module + # is being imported, which is being confused with huggingface's `evaluate``. + import lm_eval # noqa +except Exception: + pass + +from torchtune.models.clip import clip_vision_encoder from torchtune.models.llama3_1._component_builders import llama3_1 as llama3_1_builder from torchtune.models.llama3_2_vision._component_builders import ( llama3_2_vision_decoder, @@ -223,9 +232,6 @@ def _flamingo(cls): @classmethod def _llava(cls): - # TODO: Import locally until we find where in torchtune an `evaluate` module - # is being imported, which is being confused with huggingface's `evaluate``. - from torchtune.models.clip import clip_vision_encoder return cls( model_type=ModelType.Llava, modules={ diff --git a/torchchat/usages/eval.py b/torchchat/usages/eval.py index 346f3e4b3..5993c3781 100644 --- a/torchchat/usages/eval.py +++ b/torchchat/usages/eval.py @@ -3,10 +3,9 @@ # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. +import argparse from typing import Callable, Optional -import lm_eval - import torch import torch._dynamo.config import torch._inductor.config @@ -17,6 +16,7 @@ BuilderArgs, TokenizerArgs, ) +from torchchat.cli.cli import add_arguments_for_verb, arg_init from torchchat.model import Model from torchchat.utils.build_utils import set_precision @@ -28,6 +28,8 @@ torch._inductor.config.triton.cudagraphs = True torch._dynamo.config.cache_size_limit = 100000 +import lm_eval + from lm_eval.evaluator import evaluate from lm_eval.models.huggingface import HFLM as eval_wrapper from lm_eval.tasks import get_task_dict diff --git a/torchchat/utils/scripts/install_utils.sh b/torchchat/utils/scripts/install_utils.sh index 48a8d6d54..ec9677373 100644 --- a/torchchat/utils/scripts/install_utils.sh +++ b/torchchat/utils/scripts/install_utils.sh @@ -94,6 +94,12 @@ install_executorch_python_libs() { bash ./install_requirements.sh --pybind xnnpack fi + # TODO: figure out the root cause of 'AttributeError: module 'evaluate' + # has no attribute 'utils'' error from evaluate CI jobs and remove + # `import lm_eval` from torchchat.py since it requires a specific version + # of numpy. + pip install numpy=='1.26.4' + pip3 list popd }