From d9320d5a181dd934d3167ff64e1a6fe4042054ac Mon Sep 17 00:00:00 2001 From: Daniel King Date: Thu, 16 Feb 2023 14:18:56 -0800 Subject: [PATCH 1/9] add logger parsing to evaluation script --- examples/llm/icl_eval/evaluate_model.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/examples/llm/icl_eval/evaluate_model.py b/examples/llm/icl_eval/evaluate_model.py index f8243e1d4..d93490f8e 100644 --- a/examples/llm/icl_eval/evaluate_model.py +++ b/examples/llm/icl_eval/evaluate_model.py @@ -10,7 +10,7 @@ from omegaconf import DictConfig from omegaconf import OmegaConf as om -from examples.common.builders import build_icl_evaluators +from examples.common.builders import build_icl_evaluators, build_logger from examples.llm.src.model_registry import COMPOSER_MODEL_REGISTRY from examples.llm.src.tokenizer import TOKENIZER_REGISTRY @@ -28,6 +28,11 @@ composer_model.add_eval_metrics(evaluator) in_memory_logger = InMemoryLogger() # track metrics in the in_memory_logger + loggers = [ + build_logger(name, logger_cfg) + for name, logger_cfg in (cfg.get('loggers') or {}).items() + ] + loggers.append(in_memory_logger) fsdp_config = cfg.get('fsdp_config', None) fsdp_config = om.to_container( @@ -37,7 +42,7 @@ trainer = Trainer( model=composer_model, - loggers=in_memory_logger, + loggers=loggers, fsdp_config=fsdp_config, # type: ignore load_path=load_path, load_weights_only=True, From afc28234f608df773b81baf794235a2c4942f941 Mon Sep 17 00:00:00 2001 From: Daniel King Date: Thu, 16 Feb 2023 14:24:19 -0800 Subject: [PATCH 2/9] add seed to icl eval script --- examples/llm/icl_eval/evaluate_model.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/examples/llm/icl_eval/evaluate_model.py b/examples/llm/icl_eval/evaluate_model.py index d93490f8e..a51ce09a9 100644 --- a/examples/llm/icl_eval/evaluate_model.py +++ b/examples/llm/icl_eval/evaluate_model.py @@ -7,6 +7,7 @@ import torch from composer.loggers import InMemoryLogger from composer.trainer import Trainer +from composer.utils import reproducibility from omegaconf import DictConfig from omegaconf import OmegaConf as om @@ -21,6 +22,8 @@ cli_cfg = om.from_cli(args_list) cfg = DictConfig(om.merge(yaml_cfg, cli_cfg)) + reproducibility.seed_all(cfg.get('seed', 1234)) + composer_model = COMPOSER_MODEL_REGISTRY[cfg.model.name](cfg.model) tokenizer = TOKENIZER_REGISTRY[cfg.tokenizer.type](**cfg.tokenizer.args) evaluators, logger_keys = build_icl_evaluators(cfg, tokenizer) From d3e75776b0dcc7bd484d683bb0f2c45793951970 Mon Sep 17 00:00:00 2001 From: Daniel King Date: Thu, 16 Feb 2023 14:39:09 -0800 Subject: [PATCH 3/9] add space for continuation delimiter --- examples/llm/icl_eval/yamls/gpt_neo_eval.yaml | 2 +- examples/llm/icl_eval/yamls/mosaic_gpt_eval.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/llm/icl_eval/yamls/gpt_neo_eval.yaml b/examples/llm/icl_eval/yamls/gpt_neo_eval.yaml index 36e2f8762..5e4ecbecf 100644 --- a/examples/llm/icl_eval/yamls/gpt_neo_eval.yaml +++ b/examples/llm/icl_eval/yamls/gpt_neo_eval.yaml @@ -41,4 +41,4 @@ icl_tasks: - InContextLearningLMAccuracy prompt_string: '' # this goes at the beginning of each input example_delimiter: '\n' # this goes between fewshot examples - continuation_delimiter: '' # this separates contexts from continuations + continuation_delimiter: ' ' # this separates contexts from continuations diff --git a/examples/llm/icl_eval/yamls/mosaic_gpt_eval.yaml b/examples/llm/icl_eval/yamls/mosaic_gpt_eval.yaml index 14eea2fbe..2da8af112 100644 --- a/examples/llm/icl_eval/yamls/mosaic_gpt_eval.yaml +++ b/examples/llm/icl_eval/yamls/mosaic_gpt_eval.yaml @@ -50,4 +50,4 @@ icl_tasks: - InContextLearningLMAccuracy prompt_string: '' # this goes at the beginning of each input example_delimiter: '\n' # this goes between fewshot examples - continuation_delimiter: '' # this separates contexts from continuations + continuation_delimiter: ' ' # this separates contexts from continuations From 35d12064770dc5b8bd09bb9fdc5e080bc9eb637c Mon Sep 17 00:00:00 2001 From: Daniel King Date: Thu, 16 Feb 2023 14:42:52 -0800 Subject: [PATCH 4/9] pyright --- examples/llm/icl_eval/evaluate_model.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/llm/icl_eval/evaluate_model.py b/examples/llm/icl_eval/evaluate_model.py index a51ce09a9..7e305b011 100644 --- a/examples/llm/icl_eval/evaluate_model.py +++ b/examples/llm/icl_eval/evaluate_model.py @@ -3,9 +3,10 @@ import sys import time +from typing import List import torch -from composer.loggers import InMemoryLogger +from composer.loggers import InMemoryLogger, LoggerDestination from composer.trainer import Trainer from composer.utils import reproducibility from omegaconf import DictConfig @@ -31,7 +32,7 @@ composer_model.add_eval_metrics(evaluator) in_memory_logger = InMemoryLogger() # track metrics in the in_memory_logger - loggers = [ + loggers: List[LoggerDestination] = [ build_logger(name, logger_cfg) for name, logger_cfg in (cfg.get('loggers') or {}).items() ] From cebae6b3146092736fdf1a4a748ba8e028552d2d Mon Sep 17 00:00:00 2001 From: Daniel King Date: Thu, 16 Feb 2023 17:55:54 -0800 Subject: [PATCH 5/9] pass destination path through --- examples/common/builders.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/examples/common/builders.py b/examples/common/builders.py index 312d2e94e..ae12a84dc 100644 --- a/examples/common/builders.py +++ b/examples/common/builders.py @@ -124,7 +124,8 @@ def _validate_cfg(icl_cfg): num_fewshot=num_fewshot, prompt_string=icl_cfg.prompt_string, example_delimiter=icl_cfg.example_delimiter, - continuation_delimiter=icl_cfg.continuation_delimiter) + continuation_delimiter=icl_cfg.continuation_delimiter, + destination_path=cfg.get('destination_path', 'icl_task.jsonl')) logger_keys.extend([f'metrics/{label}/{m}' for m in metric_names]) evaluators.append( Evaluator(label=label, From b66bb8e92674c2268e6b52966d90b0126e1e4af9 Mon Sep 17 00:00:00 2001 From: Daniel King Date: Thu, 16 Feb 2023 17:59:35 -0800 Subject: [PATCH 6/9] undo --- examples/common/builders.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/common/builders.py b/examples/common/builders.py index ae12a84dc..312d2e94e 100644 --- a/examples/common/builders.py +++ b/examples/common/builders.py @@ -124,8 +124,7 @@ def _validate_cfg(icl_cfg): num_fewshot=num_fewshot, prompt_string=icl_cfg.prompt_string, example_delimiter=icl_cfg.example_delimiter, - continuation_delimiter=icl_cfg.continuation_delimiter, - destination_path=cfg.get('destination_path', 'icl_task.jsonl')) + continuation_delimiter=icl_cfg.continuation_delimiter) logger_keys.extend([f'metrics/{label}/{m}' for m in metric_names]) evaluators.append( Evaluator(label=label, From 8cc87fdd300dee0a916586ab0cdaa2ec6b6abb64 Mon Sep 17 00:00:00 2001 From: Daniel King Date: Thu, 16 Feb 2023 18:07:18 -0800 Subject: [PATCH 7/9] initalize dist --- examples/llm/icl_eval/evaluate_model.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/examples/llm/icl_eval/evaluate_model.py b/examples/llm/icl_eval/evaluate_model.py index 7e305b011..273414d0a 100644 --- a/examples/llm/icl_eval/evaluate_model.py +++ b/examples/llm/icl_eval/evaluate_model.py @@ -6,9 +6,10 @@ from typing import List import torch +from composer.device import get_device from composer.loggers import InMemoryLogger, LoggerDestination from composer.trainer import Trainer -from composer.utils import reproducibility +from composer.utils import dist, reproducibility from omegaconf import DictConfig from omegaconf import OmegaConf as om @@ -24,6 +25,7 @@ cfg = DictConfig(om.merge(yaml_cfg, cli_cfg)) reproducibility.seed_all(cfg.get('seed', 1234)) + dist.initialize_dist(get_device(None), 1800) composer_model = COMPOSER_MODEL_REGISTRY[cfg.model.name](cfg.model) tokenizer = TOKENIZER_REGISTRY[cfg.tokenizer.type](**cfg.tokenizer.args) From 2334ac977c0039097e356cc5e1ea84b84c80c125 Mon Sep 17 00:00:00 2001 From: Daniel King Date: Thu, 16 Feb 2023 18:12:51 -0800 Subject: [PATCH 8/9] remove initalize dist --- examples/llm/icl_eval/evaluate_model.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/examples/llm/icl_eval/evaluate_model.py b/examples/llm/icl_eval/evaluate_model.py index 273414d0a..7e305b011 100644 --- a/examples/llm/icl_eval/evaluate_model.py +++ b/examples/llm/icl_eval/evaluate_model.py @@ -6,10 +6,9 @@ from typing import List import torch -from composer.device import get_device from composer.loggers import InMemoryLogger, LoggerDestination from composer.trainer import Trainer -from composer.utils import dist, reproducibility +from composer.utils import reproducibility from omegaconf import DictConfig from omegaconf import OmegaConf as om @@ -25,7 +24,6 @@ cfg = DictConfig(om.merge(yaml_cfg, cli_cfg)) reproducibility.seed_all(cfg.get('seed', 1234)) - dist.initialize_dist(get_device(None), 1800) composer_model = COMPOSER_MODEL_REGISTRY[cfg.model.name](cfg.model) tokenizer = TOKENIZER_REGISTRY[cfg.tokenizer.type](**cfg.tokenizer.args) From 67ae41a9d9c47e26d666a2730e17e17be19c7956 Mon Sep 17 00:00:00 2001 From: Daniel King Date: Thu, 16 Feb 2023 20:11:20 -0800 Subject: [PATCH 9/9] use destination path --- examples/common/builders.py | 5 ++++- examples/llm/icl_eval/yamls/gpt_neo_eval.yaml | 2 ++ examples/llm/icl_eval/yamls/mosaic_gpt_eval.yaml | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/common/builders.py b/examples/common/builders.py index 312d2e94e..7081e56b5 100644 --- a/examples/common/builders.py +++ b/examples/common/builders.py @@ -103,6 +103,7 @@ def _validate_cfg(icl_cfg): assert 'example_delimiter' in icl_cfg assert 'continuation_delimiter' in icl_cfg assert 'label' in icl_cfg + assert 'destination_path' in icl_cfg for icl_cfg in cfg.icl_tasks: _validate_cfg(icl_cfg) @@ -124,7 +125,9 @@ def _validate_cfg(icl_cfg): num_fewshot=num_fewshot, prompt_string=icl_cfg.prompt_string, example_delimiter=icl_cfg.example_delimiter, - continuation_delimiter=icl_cfg.continuation_delimiter) + continuation_delimiter=icl_cfg.continuation_delimiter, + destination_path=icl_cfg.destination_path, + ) logger_keys.extend([f'metrics/{label}/{m}' for m in metric_names]) evaluators.append( Evaluator(label=label, diff --git a/examples/llm/icl_eval/yamls/gpt_neo_eval.yaml b/examples/llm/icl_eval/yamls/gpt_neo_eval.yaml index 5e4ecbecf..c9c91ab9e 100644 --- a/examples/llm/icl_eval/yamls/gpt_neo_eval.yaml +++ b/examples/llm/icl_eval/yamls/gpt_neo_eval.yaml @@ -30,6 +30,7 @@ icl_tasks: prompt_string: '' # this goes at the beginning of each input example_delimiter: '\n' # this goes between fewshot examples continuation_delimiter: ' ' # this separates questions from answers + destination_path: piqa_local.jsonl - label: lambada dataset_uri: # ADD YOUR OWN DATASET URI @@ -42,3 +43,4 @@ icl_tasks: prompt_string: '' # this goes at the beginning of each input example_delimiter: '\n' # this goes between fewshot examples continuation_delimiter: ' ' # this separates contexts from continuations + destination_path: lambada_local.jsonl diff --git a/examples/llm/icl_eval/yamls/mosaic_gpt_eval.yaml b/examples/llm/icl_eval/yamls/mosaic_gpt_eval.yaml index 2da8af112..16733e087 100644 --- a/examples/llm/icl_eval/yamls/mosaic_gpt_eval.yaml +++ b/examples/llm/icl_eval/yamls/mosaic_gpt_eval.yaml @@ -39,6 +39,7 @@ icl_tasks: prompt_string: '' # this goes at the beginning of each input example_delimiter: '\n' # this goes between fewshot examples continuation_delimiter: ' ' # this separates questions from answers + destination_path: piqa_local.jsonl - label: lambada dataset_uri: # ADD YOUR OWN DATASET URI @@ -51,3 +52,4 @@ icl_tasks: prompt_string: '' # this goes at the beginning of each input example_delimiter: '\n' # this goes between fewshot examples continuation_delimiter: ' ' # this separates contexts from continuations + destination_path: lambada_local.jsonl