Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support MetaMath #2748

Merged
merged 7 commits into from
Dec 9, 2023
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions docs/model_support.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,7 @@
- [OpenLemur/lemur-70b-chat-v1](https://huggingface.co/OpenLemur/lemur-70b-chat-v1)
- [allenai/tulu-2-dpo-7b](https://huggingface.co/allenai/tulu-2-dpo-7b)
- [Microsoft/Orca-2-7b](https://huggingface.co/microsoft/Orca-2-7b)
- [meta-math/MetaMath-7B-V1.0](https://huggingface.co/meta-math/MetaMath-7B-V1.0)
- Any [EleutherAI](https://huggingface.co/EleutherAI) pythia model such as [pythia-6.9b](https://huggingface.co/EleutherAI/pythia-6.9b)
- Any [Peft](https://github.com/huggingface/peft) adapter trained on top of a
model above. To activate, must have `peft` in the model path. Note: If
Expand Down
27 changes: 26 additions & 1 deletion fastchat/conversation.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ class SeparatorStyle(IntEnum):
ROBIN = auto()
FALCON_CHAT = auto()
CHATGLM3 = auto()
METAMATH = auto()


@dataclasses.dataclass
Expand Down Expand Up @@ -222,7 +223,17 @@ def get_prompt(self) -> str:
ret += role + ": " + message + self.sep
else:
ret += role + ":"

return ret
elif self.sep_style == SeparatorStyle.METAMATH:
ret = "" if system_prompt == "" else system_prompt + self.sep
for i, (role, message) in enumerate(self.messages):
# For MetaMath, sep2 is used to prefix the message.
starting_sep = ":\n" if i % 2 == 0 else ": " + self.sep2
ending_sep = self.sep if i % 2 == 0 else ""
if message:
ret += role + starting_sep + message + ending_sep
else:
ret += role + starting_sep
return ret
else:
raise ValueError(f"Invalid style: {self.sep_style}")
Expand Down Expand Up @@ -668,6 +679,20 @@ def get_conv_template(name: str) -> Conversation:
)
)

# MetaMath default template
# reference: https://github.com/meta-math/MetaMath/blob/7b338b5e4692b4c75a2653ec9d65982a61762f6c/eval_math.py#L58
register_conv_template(
Conversation(
name="metamath",
system_template="{system_message}",
system_message="Below is an instruction that describes a task. Write a response that appropriately completes the request.",
roles=("### Instruction", "### Response"),
sep_style=SeparatorStyle.METAMATH,
sep="\n\n",
sep2="Let's think step by step.",
)
)

# MPT default template
register_conv_template(
Conversation(
Expand Down
11 changes: 11 additions & 0 deletions fastchat/model/model_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -1934,6 +1934,16 @@ def get_default_conv_template(self, model_path: str) -> Conversation:
return get_conv_template("Yi-34b-chat")


class MetaMathAdapter(BaseModelAdapter):
"""The model adapter for MetaMath models"""

def match(self, model_path: str):
return "metamath" in model_path.lower()

def get_default_conv_template(self, model_path: str) -> Conversation:
return get_conv_template("metamath")


# Note: the registration order matters.
# The one registered earlier has a higher matching priority.
register_model_adapter(PeftModelAdapter)
Expand Down Expand Up @@ -2005,6 +2015,7 @@ def get_default_conv_template(self, model_path: str) -> Conversation:
register_model_adapter(PygmalionAdapter)
register_model_adapter(MicrosoftOrcaAdapter)
register_model_adapter(YiAdapter)
register_model_adapter(MetaMathAdapter)

# After all adapters, try the default base adapter.
register_model_adapter(BaseModelAdapter)
7 changes: 7 additions & 0 deletions fastchat/model/model_registry.py
Original file line number Diff line number Diff line change
Expand Up @@ -410,3 +410,10 @@ def get_model_info(name: str) -> ModelInfo:
"https://huggingface.co/01-ai",
"A large language model by 01.AI.",
)

register_model_info(
["MetaMath-70B-V1.0", "MetaMath-7B-V1.0"],
"MetaMath",
"https://huggingface.co/meta-math",
"MetaMath is a finetune of Llama2 on [MetaMathQA](https://huggingface.co/datasets/meta-math/MetaMathQA) that specializes in mathematical reasoning.",
)