ChatOllama Tool Use not working with Mistral despite labeled with Tool support

[pascal456]

Checked other resources

• I added a very descriptive title to this issue.
• I searched the LangChain documentation with the integrated search.
• I used the GitHub search to find a similar question and didn't find it.
• I am sure that this is a bug in LangChain rather than my code.
• The bug is not resolved by updating to the latest stable version of LangChain (or the specific integration package).

Example Code

from langchain_ollama import ChatOllama
from langchain_core.pydantic_v1 import BaseModel
from langchain_core.pydantic_v1 import Field
from langchain_core.prompts import ChatPromptTemplate


class Classification(BaseModel):
    sentiment: str = Field(description="The sentiment of the text")
    aggressiveness: int = Field(
        description="How aggressive the text is on a scale from 1 to 10"
    )
    language: str = Field(description="The language the text is written in")


llm = ChatOllama(
    # model="llama3.1:8b-instruct-q4_K_M",
    model="mistral:7b-instruct-q5_K_M",
    temperature=0,
).with_structured_output(schema=Classification)

tagging_prompt = ChatPromptTemplate.from_template(
    """
Extract the desired information from the following passage.

Only extract the properties mentioned in the 'Classification' function.

Passage:
{input}
"""
)

tagging_chain = tagging_prompt | llm

inp = "Estoy increiblemente contento de haberte conocido! Creo que seremos muy buenos amigos!"
tagging_chain.invoke({"input": inp})

Error Message and Stack Trace (if applicable)

---------------------------------------------------------------------------
ResponseError                             Traceback (most recent call last)
Cell In[1], line 36
     33 tagging_chain = tagging_prompt | llm
     35 inp = \"Estoy increiblemente contento de haberte conocido! Creo que seremos muy buenos amigos!\"
---> 36 tagging_chain.invoke({\"input\": inp})

File /workspaces/llm-university-examples/.venv/lib/python3.12/site-packages/langchain_core/runnables/base.py:2878, in RunnableSequence.invoke(self, input, config, **kwargs)
   2876             input = context.run(step.invoke, input, config, **kwargs)
   2877         else:
-> 2878             input = context.run(step.invoke, input, config)
   2879 # finish the root run
   2880 except BaseException as e:

File /workspaces/llm-university-examples/.venv/lib/python3.12/site-packages/langchain_core/runnables/base.py:5092, in RunnableBindingBase.invoke(self, input, config, **kwargs)
   5086 def invoke(
   5087     self,
   5088     input: Input,
   5089     config: Optional[RunnableConfig] = None,
   5090     **kwargs: Optional[Any],
   5091 ) -> Output:
-> 5092     return self.bound.invoke(
   5093         input,
   5094         self._merge_configs(config),
   5095         **{**self.kwargs, **kwargs},
   5096     )

File /workspaces/llm-university-examples/.venv/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:277, in BaseChatModel.invoke(self, input, config, stop, **kwargs)
    266 def invoke(
    267     self,
    268     input: LanguageModelInput,
   (...)
    272     **kwargs: Any,
    273 ) -> BaseMessage:
    274     config = ensure_config(config)
    275     return cast(
    276         ChatGeneration,
--> 277         self.generate_prompt(
    278             [self._convert_input(input)],
    279             stop=stop,
    280             callbacks=config.get(\"callbacks\"),
    281             tags=config.get(\"tags\"),
    282             metadata=config.get(\"metadata\"),
    283             run_name=config.get(\"run_name\"),
    284             run_id=config.pop(\"run_id\", None),
    285             **kwargs,
    286         ).generations[0][0],
    287     ).message

File /workspaces/llm-university-examples/.venv/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:777, in BaseChatModel.generate_prompt(self, prompts, stop, callbacks, **kwargs)
    769 def generate_prompt(
    770     self,
    771     prompts: List[PromptValue],
   (...)
    774     **kwargs: Any,
    775 ) -> LLMResult:
    776     prompt_messages = [p.to_messages() for p in prompts]
--> 777     return self.generate(prompt_messages, stop=stop, callbacks=callbacks, **kwargs)

File /workspaces/llm-university-examples/.venv/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:634, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, run_id, **kwargs)
    632         if run_managers:
    633             run_managers[i].on_llm_error(e, response=LLMResult(generations=[]))
--> 634         raise e
    635 flattened_outputs = [
    636     LLMResult(generations=[res.generations], llm_output=res.llm_output)  # type: ignore[list-item]
    637     for res in results
    638 ]
    639 llm_output = self._combine_llm_outputs([res.llm_output for res in results])

File /workspaces/llm-university-examples/.venv/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:624, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, run_id, **kwargs)
    621 for i, m in enumerate(messages):
    622     try:
    623         results.append(
--> 624             self._generate_with_cache(
    625                 m,
    626                 stop=stop,
    627                 run_manager=run_managers[i] if run_managers else None,
    628                 **kwargs,
    629             )
    630         )
    631     except BaseException as e:
    632         if run_managers:

File /workspaces/llm-university-examples/.venv/lib/python3.12/site-packages/langchain_core/language_models/chat_models.py:846, in BaseChatModel._generate_with_cache(self, messages, stop, run_manager, **kwargs)
    844 else:
    845     if inspect.signature(self._generate).parameters.get(\"run_manager\"):
--> 846         result = self._generate(
    847             messages, stop=stop, run_manager=run_manager, **kwargs
    848         )
    849     else:
    850         result = self._generate(messages, stop=stop, **kwargs)

File /workspaces/llm-university-examples/.venv/lib/python3.12/site-packages/langchain_ollama/chat_models.py:642, in ChatOllama._generate(self, messages, stop, run_manager, **kwargs)
    635 def _generate(
    636     self,
    637     messages: List[BaseMessage],
   (...)
    640     **kwargs: Any,
    641 ) -> ChatResult:
--> 642     final_chunk = self._chat_stream_with_aggregation(
    643         messages, stop, run_manager, verbose=self.verbose, **kwargs
    644     )
    645     generation_info = final_chunk.generation_info
    646     chat_generation = ChatGeneration(
    647         message=AIMessage(
    648             content=final_chunk.text,
   (...)
    652         generation_info=generation_info,
    653     )

File /workspaces/llm-university-examples/.venv/lib/python3.12/site-packages/langchain_ollama/chat_models.py:543, in ChatOllama._chat_stream_with_aggregation(self, messages, stop, run_manager, verbose, **kwargs)
    534 def _chat_stream_with_aggregation(
    535     self,
    536     messages: List[BaseMessage],
   (...)
    540     **kwargs: Any,
    541 ) -> ChatGenerationChunk:
    542     final_chunk = None
--> 543     for stream_resp in self._create_chat_stream(messages, stop, **kwargs):
    544         if not isinstance(stream_resp, str):
    545             chunk = ChatGenerationChunk(
    546                 message=AIMessageChunk(
    547                     content=(
   (...)
    560                 ),
    561             )

File /workspaces/llm-university-examples/.venv/lib/python3.12/site-packages/langchain_ollama/chat_models.py:515, in ChatOllama._create_chat_stream(self, messages, stop, **kwargs)
    513 params[\"options\"][\"stop\"] = stop
    514 if \"tools\" in kwargs:
--> 515     yield self._client.chat(
    516         model=params[\"model\"],
    517         messages=ollama_messages,
    518         stream=False,
    519         options=Options(**params[\"options\"]),
    520         keep_alive=params[\"keep_alive\"],
    521         format=params[\"format\"],
    522         tools=kwargs[\"tools\"],
    523     )
    524 else:
    525     yield from self._client.chat(
    526         model=params[\"model\"],
    527         messages=ollama_messages,
   (...)
    531         format=params[\"format\"],
    532     )

File /workspaces/llm-university-examples/.venv/lib/python3.12/site-packages/ollama/_client.py:236, in Client.chat(self, model, messages, tools, stream, format, options, keep_alive)
    233   if images := message.get('images'):
    234     message['images'] = [_encode_image(image) for image in images]
--> 236 return self._request_stream(
    237   'POST',
    238   '/api/chat',
    239   json={
    240     'model': model,
    241     'messages': messages,
    242     'tools': tools or [],
    243     'stream': stream,
    244     'format': format,
    245     'options': options or {},
    246     'keep_alive': keep_alive,
    247   },
    248   stream=stream,
    249 )

File /workspaces/llm-university-examples/.venv/lib/python3.12/site-packages/ollama/_client.py:99, in Client._request_stream(self, stream, *args, **kwargs)
     93 def _request_stream(
     94   self,
     95   *args,
     96   stream: bool = False,
     97   **kwargs,
     98 ) -> Union[Mapping[str, Any], Iterator[Mapping[str, Any]]]:
---> 99   return self._stream(*args, **kwargs) if stream else self._request(*args, **kwargs).json()

File /workspaces/llm-university-examples/.venv/lib/python3.12/site-packages/ollama/_client.py:75, in Client._request(self, method, url, **kwargs)
     73   response.raise_for_status()
     74 except httpx.HTTPStatusError as e:
---> 75   raise ResponseError(e.response.text, e.response.status_code) from None
     77 return response

ResponseError: mistral:7b-instruct-q5_K_M does not support tools"

Description

The ollama model library reports mistral to support Tools. Actually when using the above example provided in the langchain docs on Tagging / Classification the output reports mistral to not support Tools.

used mistral:7b-instruct-q5_K_M

with llama3.1:8b-instruct-q4_K_M it is working as expected

System Info

System Information

OS: Linux
OS Version: #1 SMP Fri Mar 29 23:14:13 UTC 2024
Python Version: 3.12.5 (main, Aug 13 2024, 02:19:05) [GCC 12.2.0]

Package Information

langchain_core: 0.2.38
langsmith: 0.1.116
langchain_ollama: 0.1.3

Optional packages not installed

langgraph
langserve

Other Dependencies

httpx: 0.27.2
jsonpatch: 1.33
ollama: 0.3.2
orjson: 3.10.7
packaging: 24.1
pydantic: 2.9.1
PyYAML: 6.0.2
requests: 2.32.3
tenacity: 8.5.0
typing-extensions: 4.12.2

[robbiemu]

mistal-nemo does. you may find better success with that