diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 82427b9f33..943be7af12 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -69,14 +69,14 @@ jobs: - name: Install packages and dependencies run: | python -m pip install --upgrade pip wheel - pip install -e .[cosmosdb] + pip install -e .[test,cosmosdb,interop] python -c "import autogen" pip install pytest-cov>=5 mock - name: Install optional dependencies for code executors # code executors and udfs auto skip without deps, so only run for python 3.11 if: matrix.python-version == '3.11' run: | - pip install -e ".[jupyter-executor,test]" + pip install -e ".[jupyter-executor]" python -m ipykernel install --user --name python3 - name: Set AUTOGEN_USE_DOCKER based on OS shell: bash @@ -97,11 +97,10 @@ jobs: - name: Coverage with Redis if: matrix.python-version == '3.10' run: | - pip install -e .[test,redis,websockets] + pip install -e .[redis,websockets] pytest test --ignore=test/agentchat/contrib --skip-openai --durations=10 --durations-min=1.0 - name: Test with Cosmos DB run: | - pip install -e .[test,cosmosdb] pytest test/cache/test_cosmos_db_cache.py --skip-openai --durations=10 --durations-min=1.0 - name: Upload coverage to Codecov if: matrix.python-version == '3.10' diff --git a/.github/workflows/openai.yml b/.github/workflows/openai.yml index 7dcf3175ba..d7dbbba492 100644 --- a/.github/workflows/openai.yml +++ b/.github/workflows/openai.yml @@ -55,7 +55,7 @@ jobs: if: matrix.python-version == '3.9' run: | pip install docker - pip install -e .[redis] + pip install -e .[redis,interop] - name: Coverage if: matrix.python-version == '3.9' env: diff --git a/=8 b/=8 new file mode 100644 index 0000000000..d47c38ce90 --- /dev/null +++ b/=8 @@ -0,0 +1,6 @@ +Requirement already satisfied: pytest in ./.venv-3.9/lib/python3.9/site-packages (7.4.4) +Requirement already satisfied: iniconfig in ./.venv-3.9/lib/python3.9/site-packages (from pytest) (2.0.0) +Requirement already satisfied: packaging in ./.venv-3.9/lib/python3.9/site-packages (from pytest) (24.2) +Requirement already satisfied: pluggy<2.0,>=0.12 in ./.venv-3.9/lib/python3.9/site-packages (from pytest) (1.5.0) +Requirement already satisfied: exceptiongroup>=1.0.0rc8 in ./.venv-3.9/lib/python3.9/site-packages (from pytest) (1.2.2) +Requirement already satisfied: tomli>=1.0.0 in ./.venv-3.9/lib/python3.9/site-packages (from pytest) (2.2.1) diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index ffd6923721..b2f22ce9c5 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -2600,7 +2600,7 @@ def update_function_signature(self, func_sig: Union[str, Dict], is_remove: None) self.client = OpenAIWrapper(**self.llm_config) - def update_tool_signature(self, tool_sig: Union[str, Dict], is_remove: None): + def update_tool_signature(self, tool_sig: Union[str, Dict], is_remove: bool): """update a tool_signature in the LLM configuration for tool_call. Args: diff --git a/autogen/coding/jupyter/embedded_ipython_code_executor.py b/autogen/coding/jupyter/embedded_ipython_code_executor.py index 09e4a06043..231dca0ffd 100644 --- a/autogen/coding/jupyter/embedded_ipython_code_executor.py +++ b/autogen/coding/jupyter/embedded_ipython_code_executor.py @@ -11,8 +11,10 @@ import uuid from pathlib import Path from queue import Empty -from typing import Any, ClassVar, List +from typing import Any, List +# this is needed for CI to work. The import of this file should fail if jupyter-kernel-gateway is not installed +import jupyter_kernel_gateway from jupyter_client import KernelManager # type: ignore[attr-defined] from jupyter_client.kernelspec import KernelSpecManager from pydantic import BaseModel, Field, field_validator diff --git a/autogen/interop/__init__.py b/autogen/interop/__init__.py new file mode 100644 index 0000000000..8f070c8f24 --- /dev/null +++ b/autogen/interop/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +from .crewai import CrewAIInteroperability +from .interoperability import Interoperability +from .interoperable import Interoperable +from .langchain import LangChainInteroperability +from .pydantic_ai import PydanticAIInteroperability +from .registry import register_interoperable_class + +__all__ = ["Interoperability", "Interoperable", "register_interoperable_class"] diff --git a/autogen/interop/crewai/__init__.py b/autogen/interop/crewai/__init__.py new file mode 100644 index 0000000000..50abbf3913 --- /dev/null +++ b/autogen/interop/crewai/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +from .crewai import CrewAIInteroperability + +__all__ = ["CrewAIInteroperability"] diff --git a/autogen/interop/crewai/crewai.py b/autogen/interop/crewai/crewai.py new file mode 100644 index 0000000000..cfaa9f5ada --- /dev/null +++ b/autogen/interop/crewai/crewai.py @@ -0,0 +1,83 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +import re +import sys +from typing import Any, Optional + +from ...tools import Tool +from ..registry import register_interoperable_class + +__all__ = ["CrewAIInteroperability"] + + +def _sanitize_name(s: str) -> str: + return re.sub(r"\W|^(?=\d)", "_", s) + + +@register_interoperable_class("crewai") +class CrewAIInteroperability: + """ + A class implementing the `Interoperable` protocol for converting CrewAI tools + to a general `Tool` format. + + This class takes a `CrewAITool` and converts it into a standard `Tool` object. + """ + + @classmethod + def convert_tool(cls, tool: Any, **kwargs: Any) -> Tool: + """ + Converts a given CrewAI tool into a general `Tool` format. + + This method ensures that the provided tool is a valid `CrewAITool`, sanitizes + the tool's name, processes its description, and prepares a function to interact + with the tool's arguments. It then returns a standardized `Tool` object. + + Args: + tool (Any): The tool to convert, expected to be an instance of `CrewAITool`. + **kwargs (Any): Additional arguments, which are not supported by this method. + + Returns: + Tool: A standardized `Tool` object converted from the CrewAI tool. + + Raises: + ValueError: If the provided tool is not an instance of `CrewAITool`, or if + any additional arguments are passed. + """ + from crewai.tools import BaseTool as CrewAITool + + if not isinstance(tool, CrewAITool): + raise ValueError(f"Expected an instance of `crewai.tools.BaseTool`, got {type(tool)}") + if kwargs: + raise ValueError(f"The CrewAIInteroperability does not support any additional arguments, got {kwargs}") + + # needed for type checking + crewai_tool: CrewAITool = tool # type: ignore[no-any-unimported] + + name = _sanitize_name(crewai_tool.name) + description = ( + crewai_tool.description.split("Tool Description: ")[-1] + + " (IMPORTANT: When using arguments, put them all in an `args` dictionary)" + ) + + def func(args: crewai_tool.args_schema) -> Any: # type: ignore[no-any-unimported] + return crewai_tool.run(**args.model_dump()) + + return Tool( + name=name, + description=description, + func=func, + ) + + @classmethod + def get_unsupported_reason(cls) -> Optional[str]: + if sys.version_info < (3, 10) or sys.version_info >= (3, 13): + return "This submodule is only supported for Python versions 3.10, 3.11, and 3.12" + + try: + import crewai.tools + except ImportError: + return "Please install `interop-crewai` extra to use this module:\n\n\tpip install ag2[interop-crewai]" + + return None diff --git a/autogen/interop/interoperability.py b/autogen/interop/interoperability.py new file mode 100644 index 0000000000..b86285d6a6 --- /dev/null +++ b/autogen/interop/interoperability.py @@ -0,0 +1,73 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 +from typing import Any, Dict, List, Type + +from ..tools import Tool +from .interoperable import Interoperable +from .registry import InteroperableRegistry + +__all__ = ["Interoperable"] + + +class Interoperability: + """ + A class to handle interoperability between different tool types. + + This class allows the conversion of tools to various interoperability classes and provides functionality + for retrieving and registering interoperability classes. + """ + + registry = InteroperableRegistry.get_instance() + + @classmethod + def convert_tool(cls, *, tool: Any, type: str, **kwargs: Any) -> Tool: + """ + Converts a given tool to an instance of a specified interoperability type. + + Args: + tool (Any): The tool object to be converted. + type (str): The type of interoperability to convert the tool to. + **kwargs (Any): Additional arguments to be passed during conversion. + + Returns: + Tool: The converted tool. + + Raises: + ValueError: If the interoperability class for the provided type is not found. + """ + interop = cls.get_interoperability_class(type) + return interop.convert_tool(tool, **kwargs) + + @classmethod + def get_interoperability_class(cls, type: str) -> Type[Interoperable]: + """ + Retrieves the interoperability class corresponding to the specified type. + + Args: + type (str): The type of the interoperability class to retrieve. + + Returns: + Type[Interoperable]: The interoperability class type. + + Raises: + ValueError: If no interoperability class is found for the provided type. + """ + supported_types = cls.registry.get_supported_types() + if type not in supported_types: + supported_types_formated = ", ".join(["'t'" for t in supported_types]) + raise ValueError( + f"Interoperability class {type} is not supported, supported types: {supported_types_formated}" + ) + + return cls.registry.get_class(type) + + @classmethod + def get_supported_types(cls) -> List[str]: + """ + Returns a sorted list of all supported interoperability types. + + Returns: + List[str]: A sorted list of strings representing the supported interoperability types. + """ + return sorted(cls.registry.get_supported_types()) diff --git a/autogen/interop/interoperable.py b/autogen/interop/interoperable.py new file mode 100644 index 0000000000..185e36089d --- /dev/null +++ b/autogen/interop/interoperable.py @@ -0,0 +1,46 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Any, Optional, Protocol, runtime_checkable + +from ..tools import Tool + +__all__ = ["Interoperable"] + + +@runtime_checkable +class Interoperable(Protocol): + """ + A Protocol defining the interoperability interface for tool conversion. + + This protocol ensures that any class implementing it provides the method + `convert_tool` to convert a given tool into a desired format or type. + """ + + @classmethod + def convert_tool(cls, tool: Any, **kwargs: Any) -> Tool: + """ + Converts a given tool to a desired format or type. + + This method should be implemented by any class adhering to the `Interoperable` protocol. + + Args: + tool (Any): The tool object to be converted. + **kwargs (Any): Additional parameters to pass during the conversion process. + + Returns: + Tool: The converted tool in the desired format or type. + """ + ... + + @classmethod + def get_unsupported_reason(cls) -> Optional[str]: + """Returns the reason for the tool being unsupported. + + This method should be implemented by any class adhering to the `Interoperable` protocol. + + Returns: + str: The reason for the interoperability class being unsupported. + """ + ... diff --git a/autogen/interop/langchain/__init__.py b/autogen/interop/langchain/__init__.py new file mode 100644 index 0000000000..1aa1f7892c --- /dev/null +++ b/autogen/interop/langchain/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +from .langchain import LangChainInteroperability + +__all__ = ["LangChainInteroperability"] diff --git a/autogen/interop/langchain/langchain.py b/autogen/interop/langchain/langchain.py new file mode 100644 index 0000000000..a6689d4dad --- /dev/null +++ b/autogen/interop/langchain/langchain.py @@ -0,0 +1,76 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +import sys +from typing import Any, Optional + +from ...tools import Tool +from ..registry import register_interoperable_class + +__all__ = ["LangChainInteroperability"] + + +@register_interoperable_class("langchain") +class LangChainInteroperability: + """ + A class implementing the `Interoperable` protocol for converting Langchain tools + into a general `Tool` format. + + This class takes a `LangchainTool` and converts it into a standard `Tool` object, + ensuring compatibility between Langchain tools and other systems that expect + the `Tool` format. + """ + + @classmethod + def convert_tool(cls, tool: Any, **kwargs: Any) -> Tool: + """ + Converts a given Langchain tool into a general `Tool` format. + + This method verifies that the provided tool is a valid `LangchainTool`, + processes the tool's input and description, and returns a standardized + `Tool` object. + + Args: + tool (Any): The tool to convert, expected to be an instance of `LangchainTool`. + **kwargs (Any): Additional arguments, which are not supported by this method. + + Returns: + Tool: A standardized `Tool` object converted from the Langchain tool. + + Raises: + ValueError: If the provided tool is not an instance of `LangchainTool`, or if + any additional arguments are passed. + """ + from langchain_core.tools import BaseTool as LangchainTool + + if not isinstance(tool, LangchainTool): + raise ValueError(f"Expected an instance of `langchain_core.tools.BaseTool`, got {type(tool)}") + if kwargs: + raise ValueError(f"The LangchainInteroperability does not support any additional arguments, got {kwargs}") + + # needed for type checking + langchain_tool: LangchainTool = tool # type: ignore + + def func(tool_input: langchain_tool.args_schema) -> Any: # type: ignore + return langchain_tool.run(tool_input.model_dump()) + + return Tool( + name=langchain_tool.name, + description=langchain_tool.description, + func=func, + ) + + @classmethod + def get_unsupported_reason(cls) -> Optional[str]: + if sys.version_info < (3, 9): + return "This submodule is only supported for Python versions 3.9 and above" + + try: + import langchain_core.tools + except ImportError: + return ( + "Please install `interop-langchain` extra to use this module:\n\n\tpip install ag2[interop-langchain]" + ) + + return None diff --git a/autogen/interop/pydantic_ai/__init__.py b/autogen/interop/pydantic_ai/__init__.py new file mode 100644 index 0000000000..c022ebc414 --- /dev/null +++ b/autogen/interop/pydantic_ai/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +from .pydantic_ai import PydanticAIInteroperability + +__all__ = ["PydanticAIInteroperability"] diff --git a/autogen/interop/pydantic_ai/pydantic_ai.py b/autogen/interop/pydantic_ai/pydantic_ai.py new file mode 100644 index 0000000000..78c48dc0cd --- /dev/null +++ b/autogen/interop/pydantic_ai/pydantic_ai.py @@ -0,0 +1,162 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + + +import sys +import warnings +from functools import wraps +from inspect import signature +from typing import Any, Callable, Optional + +from ..registry import register_interoperable_class +from .pydantic_ai_tool import PydanticAITool as AG2PydanticAITool + +__all__ = ["PydanticAIInteroperability"] + + +@register_interoperable_class("pydanticai") +class PydanticAIInteroperability: + """ + A class implementing the `Interoperable` protocol for converting Pydantic AI tools + into a general `Tool` format. + + This class takes a `PydanticAITool` and converts it into a standard `Tool` object, + ensuring compatibility between Pydantic AI tools and other systems that expect + the `Tool` format. It also provides a mechanism for injecting context parameters + into the tool's function. + """ + + @staticmethod + def inject_params( + ctx: Any, + tool: Any, + ) -> Callable[..., Any]: + """ + Wraps the tool's function to inject context parameters and handle retries. + + This method ensures that context parameters are properly passed to the tool + when invoked and that retries are managed according to the tool's settings. + + Args: + ctx (Optional[RunContext[Any]]): The run context, which may include dependencies and retry information. + tool (PydanticAITool): The Pydantic AI tool whose function is to be wrapped. + + Returns: + Callable[..., Any]: A wrapped function that includes context injection and retry handling. + + Raises: + ValueError: If the tool fails after the maximum number of retries. + """ + from pydantic_ai import RunContext + from pydantic_ai.tools import Tool as PydanticAITool + + ctx_typed: Optional[RunContext[Any]] = ctx # type: ignore + tool_typed: PydanticAITool[Any] = tool # type: ignore + + max_retries = tool_typed.max_retries if tool_typed.max_retries is not None else 1 + f = tool_typed.function + + @wraps(f) + def wrapper(*args: Any, **kwargs: Any) -> Any: + if tool_typed.current_retry >= max_retries: + raise ValueError(f"{tool_typed.name} failed after {max_retries} retries") + + try: + if ctx_typed is not None: + kwargs.pop("ctx", None) + ctx_typed.retry = tool_typed.current_retry + result = f(**kwargs, ctx=ctx_typed) # type: ignore[call-arg] + else: + result = f(**kwargs) # type: ignore[call-arg] + tool_typed.current_retry = 0 + except Exception as e: + tool_typed.current_retry += 1 + raise e + + return result + + sig = signature(f) + if ctx_typed is not None: + new_params = [param for name, param in sig.parameters.items() if name != "ctx"] + else: + new_params = list(sig.parameters.values()) + + wrapper.__signature__ = sig.replace(parameters=new_params) # type: ignore[attr-defined] + + return wrapper + + @classmethod + def convert_tool(cls, tool: Any, deps: Any = None, **kwargs: Any) -> AG2PydanticAITool: + """ + Converts a given Pydantic AI tool into a general `Tool` format. + + This method verifies that the provided tool is a valid `PydanticAITool`, + handles context dependencies if necessary, and returns a standardized `Tool` object. + + Args: + tool (Any): The tool to convert, expected to be an instance of `PydanticAITool`. + deps (Any, optional): The dependencies to inject into the context, required if + the tool takes a context. Defaults to None. + **kwargs (Any): Additional arguments that are not used in this method. + + Returns: + AG2PydanticAITool: A standardized `Tool` object converted from the Pydantic AI tool. + + Raises: + ValueError: If the provided tool is not an instance of `PydanticAITool`, or if + dependencies are missing for tools that require a context. + UserWarning: If the `deps` argument is provided for a tool that does not take a context. + """ + from pydantic_ai import RunContext + from pydantic_ai.tools import Tool as PydanticAITool + + if not isinstance(tool, PydanticAITool): + raise ValueError(f"Expected an instance of `pydantic_ai.tools.Tool`, got {type(tool)}") + + # needed for type checking + pydantic_ai_tool: PydanticAITool[Any] = tool # type: ignore[no-any-unimported] + + if tool.takes_ctx and deps is None: + raise ValueError("If the tool takes a context, the `deps` argument must be provided") + if not tool.takes_ctx and deps is not None: + warnings.warn( + "The `deps` argument is provided but will be ignored because the tool does not take a context.", + UserWarning, + ) + + if tool.takes_ctx: + ctx = RunContext( + deps=deps, + retry=0, + # All messages send to or returned by a model. + # This is mostly used on pydantic_ai Agent level. + messages=[], # TODO: check in the future if this is needed on Tool level + tool_name=pydantic_ai_tool.name, + ) + else: + ctx = None + + func = PydanticAIInteroperability.inject_params( + ctx=ctx, + tool=pydantic_ai_tool, + ) + + return AG2PydanticAITool( + name=pydantic_ai_tool.name, + description=pydantic_ai_tool.description, + func=func, + parameters_json_schema=pydantic_ai_tool._parameters_json_schema, + ) + + @classmethod + def get_unsupported_reason(cls) -> Optional[str]: + if sys.version_info < (3, 9): + return "This submodule is only supported for Python versions 3.9 and above" + + try: + import pydantic_ai.tools + except ImportError: + return "Please install `interop-pydantic-ai` extra to use this module:\n\n\tpip install ag2[interop-pydantic-ai]" + + return None diff --git a/autogen/interop/pydantic_ai/pydantic_ai_tool.py b/autogen/interop/pydantic_ai/pydantic_ai_tool.py new file mode 100644 index 0000000000..629f65e7ad --- /dev/null +++ b/autogen/interop/pydantic_ai/pydantic_ai_tool.py @@ -0,0 +1,61 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Any, Callable, Dict + +from ...agentchat.conversable_agent import ConversableAgent +from ...tools import Tool + +__all__ = ["PydanticAITool"] + + +class PydanticAITool(Tool): + """ + A class representing a Pydantic AI Tool that extends the general Tool functionality + with additional functionality specific to Pydantic AI tools. + + This class inherits from the Tool class and adds functionality for registering + tools with a ConversableAgent, along with providing additional schema information + specific to Pydantic AI tools, such as parameters and function signatures. + + Attributes: + parameters_json_schema (Dict[str, Any]): A schema describing the parameters + that the tool's function expects. + """ + + def __init__( + self, name: str, description: str, func: Callable[..., Any], parameters_json_schema: Dict[str, Any] + ) -> None: + """ + Initializes a PydanticAITool object with the provided name, description, + function, and parameter schema. + + Args: + name (str): The name of the tool. + description (str): A description of what the tool does. + func (Callable[..., Any]): The function that is executed when the tool is called. + parameters_json_schema (Dict[str, Any]): A schema describing the parameters + that the function accepts. + """ + super().__init__(name, description, func) + self._func_schema = { + "type": "function", + "function": { + "name": name, + "description": description, + "parameters": parameters_json_schema, + }, + } + + def register_for_llm(self, agent: ConversableAgent) -> None: + """ + Registers the tool with the ConversableAgent for use with a language model (LLM). + + This method updates the agent's tool signature to include the function schema, + allowing the agent to invoke the tool correctly during interactions with the LLM. + + Args: + agent (ConversableAgent): The agent with which the tool will be registered. + """ + agent.update_tool_signature(self._func_schema, is_remove=False) diff --git a/autogen/interop/registry.py b/autogen/interop/registry.py new file mode 100644 index 0000000000..443dcb5beb --- /dev/null +++ b/autogen/interop/registry.py @@ -0,0 +1,67 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Callable, Dict, Generic, List, Type, TypeVar + +from .interoperable import Interoperable + +__all__ = ["register_interoperable_class", "InteroperableRegistry"] + +InteroperableClass = TypeVar("InteroperableClass", bound=Type[Interoperable]) + + +class InteroperableRegistry: + def __init__(self) -> None: + self._registry: Dict[str, Type[Interoperable]] = {} + + def register(self, short_name: str, cls: InteroperableClass) -> InteroperableClass: + if short_name in self._registry: + raise ValueError(f"Duplicate registration for {short_name}") + + self._registry[short_name] = cls + + return cls + + def get_short_names(self) -> List[str]: + return sorted(self._registry.keys()) + + def get_supported_types(self) -> List[str]: + short_names = self.get_short_names() + supported_types = [name for name in short_names if self._registry[name].get_unsupported_reason() is None] + return supported_types + + def get_class(self, short_name: str) -> Type[Interoperable]: + return self._registry[short_name] + + @classmethod + def get_instance(cls) -> "InteroperableRegistry": + return _register + + +# global registry +_register = InteroperableRegistry() + + +# register decorator +def register_interoperable_class(short_name: str) -> Callable[[InteroperableClass], InteroperableClass]: + """Register an Interoperable class in the global registry. + + Returns: + Callable[[InteroperableClass], InteroperableClass]: Decorator function + + Example: + ```python + @register_interoperable_class("myinterop") + class MyInteroperability(Interoperable): + def convert_tool(self, tool: Any) -> Tool: + # implementation + ... + ``` + """ + + def inner(cls: InteroperableClass) -> InteroperableClass: + global _register + return _register.register(short_name, cls) + + return inner diff --git a/autogen/tools/__init__.py b/autogen/tools/__init__.py new file mode 100644 index 0000000000..5902681ce0 --- /dev/null +++ b/autogen/tools/__init__.py @@ -0,0 +1,7 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +from .tool import Tool + +__all__ = ["Tool"] diff --git a/autogen/tools/tool.py b/autogen/tools/tool.py new file mode 100644 index 0000000000..43914aa59c --- /dev/null +++ b/autogen/tools/tool.py @@ -0,0 +1,71 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +from typing import Any, Callable + +from ..agentchat.conversable_agent import ConversableAgent + +__all__ = ["Tool"] + + +class Tool: + """ + A class representing a Tool that can be used by an agent for various tasks. + + This class encapsulates a tool with a name, description, and an executable function. + The tool can be registered with a ConversableAgent for use either with an LLM or for direct execution. + + Attributes: + name (str): The name of the tool. + description (str): A brief description of the tool's purpose or function. + func (Callable[..., Any]): The function to be executed when the tool is called. + """ + + def __init__(self, name: str, description: str, func: Callable[..., Any]) -> None: + """Create a new Tool object. + + Args: + name (str): The name of the tool. + description (str): The description of the tool. + func (Callable[..., Any]): The function that will be executed when the tool is called. + """ + self._name = name + self._description = description + self._func = func + + @property + def name(self) -> str: + return self._name + + @property + def description(self) -> str: + return self._description + + @property + def func(self) -> Callable[..., Any]: + return self._func + + def register_for_llm(self, agent: ConversableAgent) -> None: + """ + Registers the tool for use with a ConversableAgent's language model (LLM). + + This method registers the tool so that it can be invoked by the agent during + interactions with the language model. + + Args: + agent (ConversableAgent): The agent to which the tool will be registered. + """ + agent.register_for_llm(name=self._name, description=self._description)(self._func) + + def register_for_execution(self, agent: ConversableAgent) -> None: + """ + Registers the tool for direct execution by a ConversableAgent. + + This method registers the tool so that it can be executed by the agent, + typically outside of the context of an LLM interaction. + + Args: + agent (ConversableAgent): The agent to which the tool will be registered. + """ + agent.register_for_execution(name=self._name)(self._func) diff --git a/notebook/autobuild_agent_library.ipynb b/notebook/autobuild_agent_library.ipynb index 02fcbffc2c..2c2010c3f4 100644 --- a/notebook/autobuild_agent_library.ipynb +++ b/notebook/autobuild_agent_library.ipynb @@ -1,932 +1,933 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "6264276d39875995", - "metadata": { - "collapsed": false - }, - "source": [ - "# Automatically Build Multi-agent System from Agent Library\n", - "\n", - "By: [Linxin Song](https://linxins97.github.io/), [Jieyu Zhang](https://jieyuz2.github.io/)\n", - "\n", - "In this notebook, we introduce a new feature for AutoBuild, `build_from_library`, which help users build an automatic task-solving process powered by a multi-agent system from a pre-defined agent library. \n", - "Specifically, in `build_from_library`, we prompt an LLM to explore useful agents from a pre-defined agent library, generating configurations for those agents for a group chat to solve the user's task." - ] - }, - { - "cell_type": "markdown", - "id": "ec78dda8e3826d8a", - "metadata": { - "collapsed": false - }, - "source": [ - "## Requirement\n", - "\n", - "AutoBuild require `autogen[autobuild]`, which can be installed by the following command:" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "e8e9ae50658be975", - "metadata": { - "collapsed": false - }, - "outputs": [], - "source": [ - "%pip install autogen[autobuild]" - ] - }, - { - "cell_type": "markdown", - "id": "176c200804af63f3", - "metadata": { - "collapsed": false - }, - "source": [ - "## Preparation and useful tools\n", - "We need to specify a `config_path`, `default_llm_config` that include backbone LLM configurations." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "2505f029423b21ab", - "metadata": { - "ExecuteTime": { - "end_time": "2023-12-26T16:58:02.762702600Z", - "start_time": "2023-12-26T16:58:02.472073Z" - }, - "collapsed": false - }, - "outputs": [], - "source": [ - "import json\n", - "\n", - "import autogen\n", - "from autogen.agentchat.contrib.agent_builder import AgentBuilder\n", - "\n", - "config_file_or_env = \"OAI_CONFIG_LIST\" # modify path\n", - "llm_config = {\"temperature\": 0}\n", - "config_list = autogen.config_list_from_json(config_file_or_env, filter_dict={\"model\": [\"gpt-4-1106-preview\", \"gpt-4\"]})\n", - "\n", - "def start_task(execution_task: str, agent_list: list):\n", - " group_chat = autogen.GroupChat(agents=agent_list, messages=[], max_round=12)\n", - " manager = autogen.GroupChatManager(groupchat=group_chat, llm_config={\"config_list\": config_list, **llm_config})\n", - " agent_list[0].initiate_chat(manager, message=execution_task)" - ] - }, - { - "cell_type": "markdown", - "id": "5fb3db8885dd6ee6", - "metadata": { - "collapsed": false - }, - "source": [ - "## Example for generating an agent library\n", - "Here, we show an example of generating an agent library from a pre-defined list of agents' names by prompting a `gpt-4`. You can also prepare a handcrafted library yourself.\n", - "\n", - "A Library contains each agent's name, description and system_message. The description is a brief introduction about agent's characteristics. As we will feed all agents' names and description to gpt-4 and let it choose the best agents for us, each agent's description should be simple but informative. \n", - "\n", - "First, we define a prompt template for description and system_message generation and a list of agents' name:" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "68315f6ec912c58a", - "metadata": { - "ExecuteTime": { - "end_time": "2023-12-23T07:39:03.317527600Z", - "start_time": "2023-12-23T07:39:03.276859600Z" - }, - "collapsed": false - }, - "outputs": [], - "source": [ - "AGENT_SYS_MSG_PROMPT = \"\"\"Acccording to the following postion name, write a high quality instruction for the position following a given example. You should only return the instruction.\n", - "\n", - "# Position Name\n", - "{position}\n", - "\n", - "# Example instruction for Data Analyst\n", - "\n", - "As Data Analyst, you are tasked with leveraging your extensive knowledge in data analysis to recognize and extract meaningful features from vast datasets. Your expertise in machine learning, specifically with the Random Forest Classifier, allows you to construct robust predictive models adept at handling both classification and regression tasks. You excel in model evaluation and interpretation, ensuring that the performance of your algorithms is not just assessed with precision, but also understood in the context of the data and the problem at hand. With a command over Python and proficiency in using the pandas library, you manipulate and preprocess data with ease.\n", - "\"\"\"\n", - "\n", - "AGENT_DESC_PROMPT = \"\"\"According to position name and the instruction, summarize the position into a high quality one sentence description.\n", - "\n", - "# Position Name\n", - "{position}\n", - "\n", - "# Instruction\n", - "{instruction}\n", - "\"\"\"\n", - "\n", - "position_list = [\n", - " \"Environmental_Scientist\",\n", - " \"Astronomer\",\n", - " \"Software_Developer\",\n", - " \"Data_Analyst\",\n", - " \"Journalist\",\n", - " \"Teacher\",\n", - " \"Lawyer\",\n", - " \"Programmer\",\n", - " \"Accountant\",\n", - " \"Mathematician\",\n", - " \"Physicist\",\n", - " \"Biologist\",\n", - " \"Chemist\",\n", - " \"Statistician\",\n", - " \"IT_Specialist\",\n", - " \"Cybersecurity_Expert\",\n", - " \"Artificial_Intelligence_Engineer\",\n", - " \"Financial_Analyst\",\n", - "]" - ] - }, - { - "cell_type": "markdown", - "id": "72b8e7d9d334a5c2", - "metadata": { - "collapsed": false - }, - "source": [ - "Then we can prompt a `gpt-4` model to generate each agent's system message as well as the description:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "8fbfef9268fc5191", - "metadata": { - "ExecuteTime": { - "end_time": "2023-12-23T07:40:01.703372Z", - "start_time": "2023-12-23T07:39:04.472589200Z" - }, - "collapsed": false - }, - "outputs": [], - "source": [ - "build_manager = autogen.OpenAIWrapper(config_list=config_list)\n", - "sys_msg_list = []\n", - "\n", - "for pos in position_list:\n", - " resp_agent_sys_msg = (\n", - " build_manager.create(\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": AGENT_SYS_MSG_PROMPT.format(\n", - " position=pos,\n", - " ),\n", - " }\n", - " ]\n", - " )\n", - " .choices[0]\n", - " .message.content\n", - " )\n", - " resp_desc_msg = (\n", - " build_manager.create(\n", - " messages=[\n", - " {\n", - " \"role\": \"user\",\n", - " \"content\": AGENT_DESC_PROMPT.format(\n", - " position=pos,\n", - " instruction=resp_agent_sys_msg,\n", - " ),\n", - " }\n", - " ]\n", - " )\n", - " .choices[0]\n", - " .message.content\n", - " )\n", - " sys_msg_list.append({\"name\": pos, \"system_message\": resp_agent_sys_msg, \"description\": resp_desc_msg})" - ] - }, - { - "cell_type": "markdown", - "id": "9e26c6db4befacc5", - "metadata": { - "collapsed": false - }, - "source": [ - "The generated profile will have the following format:" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "8ede1d7088eb183d", - "metadata": { - "ExecuteTime": { - "end_time": "2023-12-23T07:40:01.712399300Z", - "start_time": "2023-12-23T07:40:01.707400200Z" - }, - "collapsed": false - }, - "outputs": [ - { - "data": { - "text/plain": [ - "[{'name': 'Environmental_Scientist',\n", - " 'system_message': 'As an Environmental Scientist, you are responsible for applying your profound knowledge of environmental science to analyze ecological data and assess the impact of human activities on natural resources and ecosystems. Your proficiency in environmental assessment techniques enables you to design and conduct field studies, collect samples, and monitor environmental parameters effectively. Utilizing Geographic Information Systems (GIS), you spatially analyze and visualize environmental data to better understand patterns and changes in the landscape. You are adept at interpreting the results and communicating your findings clearly to stakeholders, policymakers, and the public, thereby contributing to informed decision-making on environmental issues. Your role is essential in developing sustainable practices and recommending mitigation measures to minimize environmental degradation and promote conservation.',\n", - " 'description': 'As an Environmental Scientist, you are tasked with analyzing and assessing the impact of human activities on ecosystems by conducting field studies, using GIS for spatial analysis, and communicating your findings to inform sustainable practices and conservation efforts.'},\n", - " {'name': 'Astronomer',\n", - " 'system_message': 'As an Astronomer, your duty involves diligent observation and analysis of celestial phenomena across the universe. Utilize cutting-edge telescopes and instruments to gather astronomical data, looking for patterns and irregularities that can lead to groundbreaking discoveries. Your profound knowledge in astrophysics is pivotal in interpreting these findings, which may include identifying new celestial objects, scrutinizing the properties and behaviors of stars, planets, and galaxies, and understanding cosmic events. Mastery of complex astronomical software and advanced mathematics is crucial for modeling astronomical phenomena and processing the vast amounts of data. Your role is essential in advancing our understanding of the cosmos, contributing to the broader scientific community by publishing your findings in reputable journals and engaging in peer collaboration to further space exploration and research.',\n", - " 'description': 'An Astronomer is a professional who meticulously observes, analyzes, and interprets celestial phenomena using advanced telescopes and instruments, requiring a deep knowledge of astrophysics, proficiency in mathematical modeling, and collaboration in scientific communities to enhance our comprehension of the universe.'},\n", - " {'name': 'Software_Developer',\n", - " 'system_message': 'As a Software Developer, your objective is to craft, test, and maintain the software that will meet the needs of our users and clients. Your proficiency in programming languages such as Java, C#, or JavaScript is essential, enabling you to write clean, efficient, and maintainable code. You will design algorithms and flowcharts to create systems that are logical and user-friendly. Collaboration with cross-functional teams, including product managers and designers, is crucial in order to understand software requirements and deliver innovative solutions. With your understanding of the software development life cycle, you will work through the processes of coding, debugging, testing, and deployment. You will employ industry best practices such as version control with Git and conduct code reviews to maintain high standards of software quality. Your role places you at the heart of our development efforts, where your technical prowess advances the functionality, scalability, and reliability of our software products.',\n", - " 'description': 'A Software Developer is responsible for designing, coding, testing, and maintaining software that meets client needs using languages like Java, C#, or JavaScript, collaborating with teams, adhering to best practices like Git for version control, and ensuring quality and innovation throughout the development life cycle.'},\n", - " {'name': 'Data_Analyst',\n", - " 'system_message': 'As a Data Analyst, your role is pivotal in interpreting complex data and providing insights that inform strategic decision-making. Utilize your analytical skills to cleanse and organize large sets of structured and unstructured data, ensuring its accuracy and readiness for in-depth analysis. Apply statistical analysis and predictive modeling to uncover trends, patterns, and correlations that drive operational improvements and innovative solutions. Use your proficiency in SQL for database interactions, and harness visualization tools such as Tableau or Power BI to craft compelling stories from data, aiding stakeholders in visualizing the implications of your findings. Stay abreast with the latest analytics techniques and continuously refine your models for enhanced performance, contributing significantly to the data-driven culture of our organization.',\n", - " 'description': 'The Data Analyst interprets complex datasets to provide strategic insights, cleanses and organizes data, performs statistical analysis and predictive modeling to identify trends and inform improvements, utilizes SQL for database management, and employs visualization tools like Tableau or Power BI to effectively communicate findings to stakeholders.'},\n", - " {'name': 'Journalist',\n", - " 'system_message': 'As a Journalist, you are responsible for identifying and pursuing newsworthy stories with the utmost ethical standards and a commitment to factual reporting. Your innate curiosity and excellent communication skills enable you to conduct thorough research and interviews, uncovering the details that make each story compelling and informative. Skilled in both written and verbal storytelling, you craft articles, reports, and features that engage and inform the public, adhering to strict deadlines without compromising on the integrity and accuracy of your work. Proficient in multimedia journalism, you adeptly use digital tools and social media to reach a wider audience, ensuring that your stories have the maximum impact.',\n", - " 'description': 'A Journalist is tasked with ethically sourcing and meticulously reporting newsworthy events, utilizing strong research and storytelling abilities across multiple platforms to accurately inform and engage a diverse audience.'},\n", - " {'name': 'Teacher',\n", - " 'system_message': 'As a Teacher, you are entrusted with the essential responsibility of fostering knowledge and encouraging academic and personal growth in your students. Your deep understanding of pedagogy, coupled with your expertise in the subject matter, enables you to create and deliver curricula that are both engaging and educational. Your adeptness at differentiated instruction allows you to tailor your teaching methods to suit the varied learning styles and needs within your classroom. By skillfully blending traditional teaching techniques with modern educational technology, you facilitate a dynamic and interactive learning environment. You excel in assessment and feedback, not only to gauge student progress but also to continuously improve your own teaching strategies. With strong interpersonal skills, you maintain open lines of communication with students, parents, and colleagues, fostering a collaborative and supportive school community.',\n", - " 'description': \"A Teacher is responsible for cultivating students' knowledge and growth through expertise in pedagogical practices and subject matter, designing engaging curricula, adapting teaching methods to diverse learning needs, integrating technology, and using assessment for continuous improvement while nurturing a cooperative school community.\"},\n", - " {'name': 'Lawyer',\n", - " 'system_message': 'As a Lawyer, you are required to uphold the highest standards of legal proficiency and ethical practice. Your role involves advising clients on their legal rights and responsibilities, as well as representing them in civil and criminal proceedings. You must possess a strong understanding of the law, paired with the ability to analyze case law and legislate history, to construct compelling arguments in support of your client’s position. Your keen attention to detail and dedication to thorough research are crucial in identifying legal precedents and crafting legal documents that adhere to the strictest of procedural standards. Moreover, you must exhibit exceptional negotiation skills to achieve favorable outcomes, whether in the courtroom or at the settlement table. With your articulate verbal and written communication, you clearly and persuasively present cases, explaining complex legal concepts in understandable terms to clients, judges, and juries. Your commitment to confidentiality and upholding justice is paramount and reflected in all aspects of your professional conduct.',\n", - " 'description': 'A Lawyer is a professionally trained legal advocate responsible for representing clients in legal proceedings, providing expert advice on legal matters, constructing persuasive arguments through meticulous research and analysis of law, and negotiating settlements, all while adhering to the highest ethical standards and maintaining strict confidentiality.'},\n", - " {'name': 'Programmer',\n", - " 'system_message': 'As a Programmer, you are responsible for the design, development, and implementation of software programs. Utilize your comprehensive understanding of programming languages, including but not limited to Java, C++, and Python, to create efficient and innovative software solutions. Your role involves writing clean, maintainable code while adhering to best practices in software development. You are expected to troubleshoot, debug, and upgrade existing software, as well as collaborate with cross-functional teams to define and design new product features. Your ability to think algorithmically and solve problems systematically will be integral in creating software that is not only functional but also scalable and secure.',\n", - " 'description': 'A Programmer designs, develops, and implements innovative and efficient software solutions using languages like Java, C++, and Python, ensuring code maintainability, collaborating on new features, and enhancing existing applications with a strong focus on scalability and security.'},\n", - " {'name': 'Accountant',\n", - " 'system_message': 'As Accountant, you are charged with the meticulous management and analysis of financial records, ensuring accuracy and compliance with relevant laws and regulations. Utilize your comprehensive understanding of accounting principles to prepare, examine, and maintain financial reports and statements, including balance sheets and income statements. Your role involves the reconciliation of accounts, evaluating financial operations to recommend best practices, identifying issues, and strategizing solutions for fiscal efficiency and profitability. Mastery in accounting software such as QuickBooks or Sage, alongside proficiency in Microsoft Excel, enables you to efficiently process and analyze financial data. You must ensure proper financial documentation and control systems are in place, providing comprehensive support to the organization’s financial health and integrity.',\n", - " 'description': 'As an Accountant, you are responsible for the accurate and compliant management, analysis, and reporting of financial data, along with recommending strategies to enhance fiscal efficiency and profitability, supported by proficiency in accounting software and Microsoft Excel.'},\n", - " {'name': 'Mathematician',\n", - " 'system_message': 'As a Mathematician, you are responsible for utilizing your profound understanding of mathematical theories and methodologies to solve complex theoretical and practical problems across various domains. Your proficiency in abstract reasoning enables you to develop new mathematical principles and to recognize and articulate the underlying mathematical relationships within real-world scenarios. You apply your expertise in calculus, algebra, statistics, and other mathematical branches to conduct rigorous analyses and to model systems for prediction and optimization. With a strong foundation in logic and quantitative reasoning, you perform peer reviews and contribute to interdisciplinary research projects, ensuring accuracy and consistency in mathematical arguments and results. Your role is crucial in advancing mathematical knowledge and providing innovative solutions to scientific and engineering challenges.',\n", - " 'description': 'As a Mathematician, you apply advanced mathematical theories and analytical skills to solve theoretical and practical problems in various industries, develop new principles, and provide innovative solutions to complex scientific and engineering challenges.'},\n", - " {'name': 'Physicist',\n", - " 'system_message': 'As a Physicist, you are charged with applying your profound understanding of the physical laws that govern the universe to unravel complex scientific phenomena. Your proficiency in theoretical and experimental physics enables you to develop models and conduct experiments that explore fundamental forces and particles. With exceptional analytical skills, you interpret empirical data to validate existing theories or propose new explanations for unexplained observations. Mastery in the use of mathematical tools such as differential equations and linear algebra is crucial for you to simulate physical processes. You are also adept at using specialized software and equipment for data acquisition and analysis, contributing to advancements in fields ranging from quantum mechanics to cosmology. Your strong critical thinking abilities empower you to solve intricate problems, and your commitment to scientific rigor ensures the integrity and accuracy of your research outcomes.',\n", - " 'description': 'A Physicist applies deep knowledge of physical laws to investigate scientific phenomena through theoretical modeling and experimental research, utilizing advanced mathematical techniques and specialized equipment to advance understanding in areas such as quantum mechanics and cosmology.'},\n", - " {'name': 'Biologist',\n", - " 'system_message': 'As a Biologist, you are entrusted with the study and understanding of living organisms, applying your expertise to investigate their functions, genetics, evolution, and ecosystems. Your skills in experimental design empower you to conduct research and experiments that can unlock new biological insights and improve our comprehension of life processes. Utilizing advanced microscopy techniques and molecular biology methods, you should meticulously analyze cell structures and DNA sequences to uncover the intricacies of life at a microscopic level. Demonstrate proficiency in bioinformatics tools to analyze genetic data and contribute valuable findings to the scientific community. Furthermore, as a communicator of science, ensure that your research findings are effectively documented and presented in scientific journals and at conferences, thereby enhancing the collective knowledge in your field.',\n", - " 'description': 'A Biologist meticulously studies and understands living organisms, conducting advanced research to decode genetics and ecosystems and sharing findings through scientific publications and presentations.'},\n", - " {'name': 'Chemist',\n", - " 'system_message': 'As a Chemist, you are charged with applying your profound understanding of chemical principles to conduct complex experiments, synthesize new compounds, and analyze the molecular and atomic structure of materials. Your proficiency in utilizing sophisticated analytical techniques - such as chromatography, spectroscopy, and mass spectrometry - enables you to decipher the composition and properties of substances. The knowledge you hold in chemical safety and handling procedures ensures a secure laboratory environment. With an adeptness in maintaining accurate records and an insightful approach to interpreting data, you transform raw experimental results into valuable scientific insights. Your ability to communicate complex chemical information clearly makes you essential in collaborative research efforts and in driving innovation within the field.',\n", - " 'description': 'As a Chemist, you are responsible for conducting advanced experiments, synthesizing compounds, deciphering substance compositions with techniques like chromatography and mass spectrometry, and transforming experimental data into scientific insights, while maintaining safety and clear communication in research collaborations.'},\n", - " {'name': 'Statistician',\n", - " 'system_message': 'As a Statistician, your primary duty is to apply mathematical and statistical methods to collect, analyze, and interpret numerical data to make informed decisions. Your strong grounding in probability theory will be essential for designing surveys and experiments to generate data. You are adept at constructing and applying sophisticated statistical models and methods, such as linear regression, ANOVA, or time-series analysis, ensuring that you accurately capture trends and relationships within the data. You possess an in-depth understanding of statistical software such as R or SAS, allowing you to perform complex analyses with efficiency and precision. Your ability to communicate complex statistical concepts to non-experts will be crucial; hence, your role includes presenting findings in a clear, actionable manner, with data visualizations and reports that drive strategic planning and policy development.',\n", - " 'description': 'A Statistician employs and interprets advanced statistical techniques to design data-collection processes, analyze data, and present findings in a comprehensible manner, supporting evidence-based decision-making and policy formation.'},\n", - " {'name': 'IT_Specialist',\n", - " 'system_message': 'As an IT Specialist, your primary responsibility is to maintain the integrity and functionality of all our computer systems and networks. Your comprehensive understanding of hardware and software is crucial for diagnosing and resolving technical issues. You are adept at implementing network security measures to protect data and systems from cyber threats. You also play a significant role in systems and software upgrades, ensuring a seamless transition without disrupting workflow. Utilizing your strong problem-solving skills and proficiency in scripting languages, you automate repetitive tasks, enhancing system efficiency. Your ability to communicate effectively with team members and non-technical staff allows you to provide clear guidance and end-user support.',\n", - " 'description': 'An IT Specialist is responsible for upholding and optimizing our computer systems and networks through maintenance, security, upgrades, issue resolution, automation, and providing support and clear communication to both technical and non-technical personnel.'},\n", - " {'name': 'Cybersecurity_Expert',\n", - " 'system_message': \"As a Cybersecurity Expert, you are charged with the responsibility of safeguarding the organization's computer networks and systems. Your deep understanding of cyber threats and mitigation techniques is critical in identifying vulnerabilities and protecting against malicious attacks. Employing your experience with tools such as firewalls, antivirus software, and intrusion detection systems, you will continuously monitor and defend our digital infrastructure. You are expected to conduct regular security audits and penetration testing to simulate cyber attacks and find potential weaknesses before they can be exploited. Your proficiency in risk management frameworks and incident response protocols ensures that you are prepared to swiftly handle and mitigate any security incidents that occur. With your expertise in encryption technologies and network protocols, you protect sensitive data and ensure compliance with relevant security standards and regulations. Your foresight in staying up-to-date with the latest cybersecurity trends and threats is paramount to maintaining the organization's digital defense at its peak.\",\n", - " 'description': \"As a Cybersecurity Expert, you are responsible for the proactive protection and defense of an organization's computer networks and systems against cyber threats through continuous monitoring, conducting security audits, penetrating testing, and swiftly mitigating security incidents, while ensuring compliance with security regulations.\"},\n", - " {'name': 'Artificial_Intelligence_Engineer',\n", - " 'system_message': 'As an Artificial Intelligence Engineer, you are responsible for conceptualizing, designing, and implementing intelligent systems that simulate human cognitive processes. Your role demands a deep understanding of neural networks, particularly Convolutional Neural Networks (CNNs) for image recognition tasks and Recurrent Neural Networks (RNNs) for natural language processing. With your expertise in TensorFlow or PyTorch, you develop complex models that can learn, adapt, and make decisions. You prioritize the ethical design and deployment of AI systems, conscious of the implications your work may have on society. Mastery of algorithms and a proficiency in a high-level programming language, preferably Python, enable you to transform theoretical AI concepts into practical solutions that drive innovation and efficiency.',\n", - " 'description': 'An Artificial Intelligence Engineer specializes in creating and implementing advanced intelligent systems, with a mastery of neural networks, machine learning frameworks, and ethical AI principles, to develop innovative solutions that emulate human cognition.'},\n", - " {'name': 'Financial_Analyst',\n", - " 'system_message': 'As a Financial Analyst, you are entrusted with utilizing your in-depth understanding of financial principles to assess investment opportunities, analyze financial data, and forecast economic trends. Your proficiency in financial modeling is paramount, enabling you to develop complex models that underpin the valuation of stocks, bonds, and other financial instruments. With a sharp eye for detail, you scrutinize company financial statements to derive actionable insights and recommend strategies to optimize financial performance. Your expertise in Excel, especially with advanced functions and formulas, allows you to efficiently manipulate and analyze large financial datasets. You are a whiz at creating compelling visualizations and delivering presentations to communicate your findings and influence strategic decisions. Your role is crucial in guiding investment decisions and driving the fiscal prudence of the organization.',\n", - " 'description': \"A Financial Analyst performs in-depth financial analysis and modeling to evaluate investments, forecast economic trends, and deliver strategic recommendations, leveraging advanced Excel skills to inform and guide the organization's financial decisions.\"}]" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "sys_msg_list" - ] - }, - { - "cell_type": "markdown", - "id": "256dd32b03a7a172", - "metadata": { - "collapsed": false - }, - "source": [ - "We can save the generated agents' information into a json file." - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "53111125938845cf", - "metadata": { - "ExecuteTime": { - "end_time": "2023-12-23T07:40:01.750855900Z", - "start_time": "2023-12-23T07:40:01.710399600Z" - }, - "collapsed": false - }, - "outputs": [], - "source": [ - "json.dump(sys_msg_list, open(\"./agent_library_example.json\", \"w\"), indent=4)" - ] - }, - { - "cell_type": "markdown", - "id": "cfd883b79a3bd932", - "metadata": { - "collapsed": false - }, - "source": [ - "## Build agents from library (by LLM)\n", - "Here, we introduce how to build agents from the generated library. As in the previous `build`, we also need to specify a `building_task` that lets the build manager know which agents should be selected from the library according to the task. \n", - "\n", - "We also need to specify a `library_path_or_json`, which can be a path of library or a JSON string with agents' configs. Here, we use the previously saved path as the library path." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "8963a8709c8e92e2", - "metadata": { - "ExecuteTime": { - "end_time": "2023-12-23T07:40:01.752918500Z", - "start_time": "2023-12-23T07:40:01.735461Z" - }, - "collapsed": false - }, - "outputs": [], - "source": [ - "library_path_or_json = \"./agent_library_example.json\"\n", - "building_task = \"Find a paper on arxiv by programming, and analyze its application in some domain. For example, find a recent paper about gpt-4 on arxiv and find its potential applications in software.\"" - ] - }, - { - "cell_type": "markdown", - "id": "72656a8d0c1a9b12", - "metadata": { - "collapsed": false - }, - "source": [ - "Then, we can call the `build_from_library` from the AgentBuilder to generate a list of agents from the library and let them complete the user's `execution_task` in a group chat." - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "5c669b76b2c9b750", - "metadata": { - "ExecuteTime": { - "end_time": "2023-12-23T07:46:02.075542200Z", - "start_time": "2023-12-23T07:43:55.489042900Z" - }, - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[32m==> Looking for suitable agents in the library...\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['Programmer', 'Mathematician'] are selected.\n", - "\u001b[32m==> Creating agents...\u001b[0m\n", - "Creating agent Programmer...\n", - "Creating agent Mathematician...\n", - "Adding user console proxy...\n", - "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", - "\n", - "Find a recent paper about explainable AI on arxiv and find its potential applications in medical.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Computer_terminal\n", - "\u001b[0m\n", - "\u001b[33mComputer_terminal\u001b[0m (to chat_manager):\n", - "\n", - "There is no code from the last 1 message for me to execute. Group chat manager should let other participants to continue the conversation. If the group chat manager want to end the conversation, you should let other participant reply me only with \"TERMINATE\"\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Programmer\n", - "\u001b[0m\n", - "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", - "\n", - "To find a recent paper about explainable AI on arXiv, we can use the arXiv API to search for papers that match the query. However, since I cannot directly access external APIs, I suggest that one of us manually searches for the paper on the arXiv website using relevant search terms such as \"explainable AI\" and \"medical applications\". Once we find a suitable paper, we can discuss its potential applications in the medical field. \n", - "\n", - "Mathematician, would you like to perform the search, or shall I provide a Python script that could be used to perform the search programmatically?\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Mathematician\n", - "\u001b[0m\n", - "\u001b[33mMathematician\u001b[0m (to chat_manager):\n", - "\n", - "As a Mathematician, I can certainly appreciate the value of a programmatic approach to searching for academic papers. However, since I do not have direct access to execute code or APIs, I would suggest that you, as the Programmer, provide the Python script that could be used to perform the search on arXiv. Once we have identified a paper, I can then assist in discussing its potential applications in the medical field from a mathematical perspective.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Programmer\n", - "\u001b[0m\n", - "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", - "\n", - "Understood. I will provide a Python script that can be used to search for recent papers about explainable AI on arXiv with potential applications in the medical field. The script will use the `arxiv` Python package, which is a wrapper for the arXiv API. If the package is not installed, we will need to install it first.\n", - "\n", - "Let's start by checking if the `arxiv` package is installed and if not, we will install it. Computer_terminal, please execute the following command to check for the `arxiv` package and install it if necessary.\n", - "\n", - "```sh\n", - "pip show arxiv || pip install arxiv\n", - "```\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Computer_terminal\n", - "\u001b[0m\n", - "\u001b[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is sh)...\u001b[0m\n", - "\u001b[33mComputer_terminal\u001b[0m (to chat_manager):\n", - "\n", - "exitcode: 0 (execution succeeded)\n", - "Code output: \n", - "Name: arxiv\n", - "Version: 2.1.0\n", - "Summary: Python wrapper for the arXiv API: https://arxiv.org/help/api/\n", - "Home-page: https://github.com/lukasschwab/arxiv.py\n", - "Author: Lukas Schwab\n", - "Author-email: lukas.schwab@gmail.com\n", - "License: MIT\n", - "Location: /home/vscode/.local/lib/python3.10/site-packages\n", - "Requires: feedparser, requests\n", - "Required-by: \n", - "\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Programmer\n", - "\u001b[0m\n", - "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", - "\n", - "Great, the `arxiv` package is already installed. Now, I will provide a Python script that uses the `arxiv` package to search for recent papers related to explainable AI with potential applications in the medical field. The script will query the arXiv API for papers with relevant keywords and print out the title and summary of the most recent paper found.\n", - "\n", - "Computer_terminal, please execute the following Python script.\n", - "\n", - "```python\n", - "import arxiv\n", - "\n", - "# Define the search query\n", - "search_query = 'all:explainable AI AND all:medical'\n", - "\n", - "# Search for papers on arXiv\n", - "search = arxiv.Search(\n", - " query = search_query,\n", - " max_results = 1,\n", - " sort_by = arxiv.SortCriterion.SubmittedDate\n", - ")\n", - "\n", - "# Fetch the most recent paper\n", - "for paper in search.results():\n", - " print(\"Title:\", paper.title)\n", - " print(\"Summary:\", paper.summary)\n", - " # Only print the most recent paper\n", - " break\n", - "```\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Computer_terminal\n", - "\u001b[0m\n", - "\u001b[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n", - "\u001b[33mComputer_terminal\u001b[0m (to chat_manager):\n", - "\n", - "exitcode: 0 (execution succeeded)\n", - "Code output: \n", - "Title: Automated Information Extraction from Thyroid Operation Narrative: A Comparative Study of GPT-4 and Fine-tuned KoELECTRA\n", - "Summary: In the rapidly evolving field of healthcare, the integration of artificial\n", - "intelligence (AI) has become a pivotal component in the automation of clinical\n", - "workflows, ushering in a new era of efficiency and accuracy. This study focuses\n", - "on the transformative capabilities of the fine-tuned KoELECTRA model in\n", - "comparison to the GPT-4 model, aiming to facilitate automated information\n", - "extraction from thyroid operation narratives. The current research landscape is\n", - "dominated by traditional methods heavily reliant on regular expressions, which\n", - "often face challenges in processing free-style text formats containing critical\n", - "details of operation records, including frozen biopsy reports. Addressing this,\n", - "the study leverages advanced natural language processing (NLP) techniques to\n", - "foster a paradigm shift towards more sophisticated data processing systems.\n", - "Through this comparative study, we aspire to unveil a more streamlined,\n", - "precise, and efficient approach to document processing in the healthcare\n", - "domain, potentially revolutionizing the way medical data is handled and\n", - "analyzed.\n", - "\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Mathematician\n", - "\u001b[0m\n", - "\u001b[33mMathematician\u001b[0m (to chat_manager):\n", - "\n", - "The paper titled \"Automated Information Extraction from Thyroid Operation Narrative: A Comparative Study of GPT-4 and Fine-tuned KoELECTRA\" presents a study on the use of artificial intelligence for automating the extraction of information from thyroid operation narratives. This is a clear example of explainable AI being applied in the medical field, specifically in the area of clinical workflows and document processing.\n", - "\n", - "The potential applications of such technology in medicine are vast. By automating the extraction of information from operation narratives, healthcare professionals can save time and reduce the likelihood of human error. This can lead to more accurate patient records, improved patient care, and streamlined administrative processes. Additionally, the ability to quickly and accurately process operation records can facilitate better data analysis, which can be used for medical research, trend analysis, and improving healthcare outcomes.\n", - "\n", - "The use of advanced natural language processing (NLP) techniques, as mentioned in the summary, is particularly important for processing free-style text formats that contain critical medical information. This technology could be further explored to extend its application to other types of medical documents and records, enhancing the overall efficiency of the healthcare system.\n", - "\n", - "The study's focus on comparing the performance of the fine-tuned KoELECTRA model with GPT-4 also highlights the importance of evaluating different AI models to determine the most effective approach for specific medical applications. This comparative analysis can lead to the development of more specialized AI tools tailored to the needs of the healthcare industry.\n", - "\n", - "In conclusion, the research presented in this paper has significant implications for the future of medical document processing and the broader integration of AI in healthcare.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Programmer\n", - "\u001b[0m\n", - "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", - "\n", - "The insights provided by the Mathematician are indeed valuable. The application of AI for automated information extraction from medical documents like thyroid operation narratives can greatly enhance efficiency and accuracy in healthcare. The use of models like GPT-4 and KoELECTRA for natural language processing tasks shows the potential of AI to handle complex, unstructured data which is common in medical records.\n", - "\n", - "From a programming perspective, the implementation of such AI systems would involve training models on large datasets of medical documents to learn the context and semantics specific to medical terminology. Ensuring the explainability of AI in this context is crucial, as healthcare professionals need to understand and trust the AI's decision-making process, especially when it directly affects patient care.\n", - "\n", - "Moreover, the integration of explainable AI into healthcare systems must adhere to strict privacy and security regulations to protect sensitive patient data. This requires careful design and implementation of data handling procedures within the AI system.\n", - "\n", - "The potential applications extend beyond just document processing to diagnostic assistance, personalized treatment plans, and predictive analytics for patient outcomes. As AI technology continues to evolve, its role in supporting and enhancing the capabilities of healthcare professionals will undoubtedly expand.\n", - "\n", - "Given the importance of the topic and the potential impact on healthcare, it would be beneficial to keep an eye on further developments in this field. If there are no further questions or points to discuss, we can conclude our conversation on this topic.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Mathematician\n", - "\u001b[0m\n", - "\u001b[33mMathematician\u001b[0m (to chat_manager):\n", - "\n", - "I agree with the Programmer's assessment. The implementation of AI in healthcare does indeed require careful consideration of the models used, the training data, and the explainability of the AI's decisions. The ethical implications, particularly concerning patient privacy and data security, are paramount and must be addressed with the utmost rigor.\n", - "\n", - "The potential for AI to assist in diagnostics, treatment planning, and predictive analytics is a promising development for the future of medicine. It is essential that these systems are developed in collaboration with healthcare professionals to ensure they meet the real-world needs of the field.\n", - "\n", - "The interdisciplinary nature of this work, combining expertise in mathematics, computer science, and medicine, is a testament to the collaborative efforts needed to advance healthcare technology. It has been a pleasure discussing the potential applications of explainable AI in medicine with you.\n", - "\n", - "If there are no further points to add, I believe we have reached a natural conclusion to our conversation.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Computer_terminal\n", - "\u001b[0m\n", - "\u001b[33mComputer_terminal\u001b[0m (to chat_manager):\n", - "\n", - "There is no code from the last 1 message for me to execute. Group chat manager should let other participants to continue the conversation. If the group chat manager want to end the conversation, you should let other participant reply me only with \"TERMINATE\"\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[33mAll agents have been cleared.\u001b[0m\n" - ] - } - ], - "source": [ - "new_builder = AgentBuilder(\n", - " config_file_or_env=config_file_or_env, builder_model=\"gpt-4-1106-preview\", agent_model=\"gpt-4-1106-preview\"\n", - ")\n", - "agent_list, _ = new_builder.build_from_library(building_task, library_path_or_json, llm_config)\n", - "start_task(\n", - " execution_task=\"Find a recent paper about explainable AI on arxiv and find its potential applications in medical.\",\n", - " agent_list=agent_list,\n", - ")\n", - "new_builder.clear_all_agents()" - ] - }, - { - "cell_type": "markdown", - "id": "c7a10e6fa00a5a0d", - "metadata": { - "collapsed": false - }, - "source": [ - "## Build agents from library (by description-task similarity)\n", - "We also support using embedding similarity to select agents. You can use a [Sentence-Transformers model](https://www.sbert.net/docs/pretrained_models.html) as an embedding extractor, and AgentBuilder will select agents with profiles that are the most similar to the building task from the library by comparing their embedding similarity. This will reduce the use of LLMs but may have less accuracy." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "521dc5f961efde59", - "metadata": { - "ExecuteTime": { - "end_time": "2023-12-26T17:01:29.333975100Z", - "start_time": "2023-12-26T16:58:11.070813500Z" - }, - "collapsed": false - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\u001b[32m==> Looking for suitable agents in the library...\u001b[0m\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "['Programmer', 'Mathematician'] are selected.\n", - "\u001b[32m==> Creating agents...\u001b[0m\n", - "Creating agent Programmer...\n", - "Creating agent Mathematician...\n", - "Adding user console proxy...\n", - "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", - "\n", - "Find a recent paper about gpt-4 on arxiv and find its potential applications in software.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Computer_terminal\n", - "\u001b[0m\n", - "\u001b[33mComputer_terminal\u001b[0m (to chat_manager):\n", - "\n", - "There is no code from the last 1 message for me to execute. Group chat manager should let other participants to continue the conversation. If the group chat manager want to end the conversation, you should let other participant reply me only with \"TERMINATE\"\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Programmer\n", - "\u001b[0m\n", - "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", - "\n", - "To find a recent paper about GPT-4 on arXiv, we can use the arXiv API to search for papers. However, since I can't directly access external APIs, I can write a Python script that you can run on your local machine to perform this search. Would you like me to provide you with such a script?\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Computer_terminal\n", - "\u001b[0m\n", - "\u001b[33mComputer_terminal\u001b[0m (to chat_manager):\n", - "\n", - "There is no code from the last 1 message for me to execute. Group chat manager should let other participants to continue the conversation. If the group chat manager want to end the conversation, you should let other participant reply me only with \"TERMINATE\"\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Programmer\n", - "\u001b[0m\n", - "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", - "\n", - "I apologize for the confusion. I will provide a Python script that can be executed by the Computer_terminal to search for recent papers about GPT-4 on arXiv. Let's proceed with that.\n", - "\n", - "```python\n", - "import requests\n", - "from xml.etree import ElementTree\n", - "\n", - "# Define the search parameters and URL for the arXiv API\n", - "search_query = 'all:gpt-4'\n", - "start = 0\n", - "max_results = 5\n", - "sort_by = 'submittedDate'\n", - "sort_order = 'descending'\n", - "url = f'http://export.arxiv.org/api/query?search_query={search_query}&start={start}&max_results={max_results}&sortBy={sort_by}&sortOrder={sort_order}'\n", - "\n", - "# Send a GET request to the arXiv API\n", - "response = requests.get(url)\n", - "\n", - "# Parse the response if it was successful\n", - "if response.status_code == 200:\n", - " root = ElementTree.fromstring(response.content)\n", - " # Find and print the entries (papers)\n", - " for entry in root.findall('{http://www.w3.org/2005/Atom}entry'):\n", - " title = entry.find('{http://www.w3.org/2005/Atom}title').text\n", - " summary = entry.find('{http://www.w3.org/2005/Atom}summary').text\n", - " published = entry.find('{http://www.w3.org/2005/Atom}published').text\n", - " print(f\"Title: {title}\\nSummary: {summary}\\nPublished Date: {published}\\n\")\n", - "else:\n", - " print(f\"Failed to fetch data from arXiv. Status code: {response.status_code}\")\n", - "```\n", - "\n", - "This script will fetch the most recent papers related to GPT-4 from the arXiv API and print out their titles, summaries, and publication dates. Please execute this script to find the information we need.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Computer_terminal\n", - "\u001b[0m\n", - "\u001b[31m\n", - ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n", - "\u001b[33mComputer_terminal\u001b[0m (to chat_manager):\n", - "\n", - "exitcode: 0 (execution succeeded)\n", - "Code output: \n", - "Title: What If We Recaption Billions of Web Images with LLaMA-3?\n", - "Summary: Web-crawled image-text pairs are inherently noisy. Prior studies demonstrate\n", - "that semantically aligning and enriching textual descriptions of these pairs\n", - "can significantly enhance model training across various vision-language tasks,\n", - "particularly text-to-image generation. However, large-scale investigations in\n", - "this area remain predominantly closed-source. Our paper aims to bridge this\n", - "community effort, leveraging the powerful and \\textit{open-sourced} LLaMA-3, a\n", - "GPT-4 level LLM. Our recaptioning pipeline is simple: first, we fine-tune a\n", - "LLaMA-3-8B powered LLaVA-1.5 and then employ it to recaption 1.3 billion images\n", - "from the DataComp-1B dataset. Our empirical results confirm that this enhanced\n", - "dataset, Recap-DataComp-1B, offers substantial benefits in training advanced\n", - "vision-language models. For discriminative models like CLIP, we observe\n", - "enhanced zero-shot performance in cross-modal retrieval tasks. For generative\n", - "models like text-to-image Diffusion Transformers, the generated images exhibit\n", - "a significant improvement in alignment with users' text instructions,\n", - "especially in following complex queries. Our project page is\n", - "https://www.haqtu.me/Recap-Datacomp-1B/\n", - "\n", - "Published Date: 2024-06-12T17:59:07Z\n", - "\n", - "Title: DafnyBench: A Benchmark for Formal Software Verification\n", - "Summary: We introduce DafnyBench, the largest benchmark of its kind for training and\n", - "evaluating machine learning systems for formal software verification. We test\n", - "the ability of LLMs such as GPT-4 and Claude 3 to auto-generate enough hints\n", - "for the Dafny formal verification engine to successfully verify over 750\n", - "programs with about 53,000 lines of code. The best model and prompting scheme\n", - "achieved 68% success rate, and we quantify how this rate improves when retrying\n", - "with error message feedback and how it deteriorates with the amount of required\n", - "code and hints. We hope that DafnyBench will enable rapid improvements from\n", - "this baseline as LLMs and verification techniques grow in quality.\n", - "\n", - "Published Date: 2024-06-12T17:53:31Z\n", - "\n", - "Title: A Sociotechnical Lens for Evaluating Computer Vision Models: A Case\n", - " Study on Detecting and Reasoning about Gender and Emotion\n", - "Summary: In the evolving landscape of computer vision (CV) technologies, the automatic\n", - "detection and interpretation of gender and emotion in images is a critical area\n", - "of study. This paper investigates social biases in CV models, emphasizing the\n", - "limitations of traditional evaluation metrics such as precision, recall, and\n", - "accuracy. These metrics often fall short in capturing the complexities of\n", - "gender and emotion, which are fluid and culturally nuanced constructs. Our\n", - "study proposes a sociotechnical framework for evaluating CV models,\n", - "incorporating both technical performance measures and considerations of social\n", - "fairness. Using a dataset of 5,570 images related to vaccination and climate\n", - "change, we empirically compared the performance of various CV models, including\n", - "traditional models like DeepFace and FER, and generative models like GPT-4\n", - "Vision. Our analysis involved manually validating the gender and emotional\n", - "expressions in a subset of images to serve as benchmarks. Our findings reveal\n", - "that while GPT-4 Vision outperforms other models in technical accuracy for\n", - "gender classification, it exhibits discriminatory biases, particularly in\n", - "response to transgender and non-binary personas. Furthermore, the model's\n", - "emotion detection skew heavily towards positive emotions, with a notable bias\n", - "towards associating female images with happiness, especially when prompted by\n", - "male personas. These findings underscore the necessity of developing more\n", - "comprehensive evaluation criteria that address both validity and discriminatory\n", - "biases in CV models. Our proposed framework provides guidelines for researchers\n", - "to critically assess CV tools, ensuring their application in communication\n", - "research is both ethical and effective. The significant contribution of this\n", - "study lies in its emphasis on a sociotechnical approach, advocating for CV\n", - "technologies that support social good and mitigate biases rather than\n", - "perpetuate them.\n", - "\n", - "Published Date: 2024-06-12T13:52:30Z\n", - "\n", - "Title: Supportiveness-based Knowledge Rewriting for Retrieval-augmented\n", - " Language Modeling\n", - "Summary: Retrieval-augmented language models (RALMs) have recently shown great\n", - "potential in mitigating the limitations of implicit knowledge in LLMs, such as\n", - "untimely updating of the latest expertise and unreliable retention of long-tail\n", - "knowledge. However, since the external knowledge base, as well as the\n", - "retriever, can not guarantee reliability, potentially leading to the knowledge\n", - "retrieved not being helpful or even misleading for LLM generation. In this\n", - "paper, we introduce Supportiveness-based Knowledge Rewriting (SKR), a robust\n", - "and pluggable knowledge rewriter inherently optimized for LLM generation.\n", - "Specifically, we introduce the novel concept of \"supportiveness\"--which\n", - "represents how effectively a knowledge piece facilitates downstream tasks--by\n", - "considering the perplexity impact of augmented knowledge on the response text\n", - "of a white-box LLM. Based on knowledge supportiveness, we first design a\n", - "training data curation strategy for our rewriter model, effectively identifying\n", - "and filtering out poor or irrelevant rewrites (e.g., with low supportiveness\n", - "scores) to improve data efficacy. We then introduce the direct preference\n", - "optimization (DPO) algorithm to align the generated rewrites to optimal\n", - "supportiveness, guiding the rewriter model to summarize augmented content that\n", - "better improves the final response. Comprehensive evaluations across six\n", - "popular knowledge-intensive tasks and four LLMs have demonstrated the\n", - "effectiveness and superiority of SKR. With only 7B parameters, SKR has shown\n", - "better knowledge rewriting capability over GPT-4, the current state-of-the-art\n", - "general-purpose LLM.\n", - "\n", - "Published Date: 2024-06-12T11:52:35Z\n", - "\n", - "Title: Automated Information Extraction from Thyroid Operation Narrative: A\n", - " Comparative Study of GPT-4 and Fine-tuned KoELECTRA\n", - "Summary: In the rapidly evolving field of healthcare, the integration of artificial\n", - "intelligence (AI) has become a pivotal component in the automation of clinical\n", - "workflows, ushering in a new era of efficiency and accuracy. This study focuses\n", - "on the transformative capabilities of the fine-tuned KoELECTRA model in\n", - "comparison to the GPT-4 model, aiming to facilitate automated information\n", - "extraction from thyroid operation narratives. The current research landscape is\n", - "dominated by traditional methods heavily reliant on regular expressions, which\n", - "often face challenges in processing free-style text formats containing critical\n", - "details of operation records, including frozen biopsy reports. Addressing this,\n", - "the study leverages advanced natural language processing (NLP) techniques to\n", - "foster a paradigm shift towards more sophisticated data processing systems.\n", - "Through this comparative study, we aspire to unveil a more streamlined,\n", - "precise, and efficient approach to document processing in the healthcare\n", - "domain, potentially revolutionizing the way medical data is handled and\n", - "analyzed.\n", - "\n", - "Published Date: 2024-06-12T06:44:05Z\n", - "\n", - "\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Programmer\n", - "\u001b[0m\n", - "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", - "\n", - "Based on the search results from the arXiv API, we have found several papers that discuss potential applications of GPT-4 in software:\n", - "\n", - "1. **Recaptioning Web Images with LLaMA-3 and GPT-4**: This paper discusses the use of GPT-4 level LLMs for recaptioning web images, which can enhance model training across various vision-language tasks. This has implications for improving the quality of datasets used in machine learning and could be particularly beneficial for text-to-image generation and cross-modal retrieval tasks.\n", - "\n", - "2. **DafnyBench: A Benchmark for Formal Software Verification**: This paper introduces a benchmark for training and evaluating machine learning systems for formal software verification. It tests the ability of LLMs such as GPT-4 to auto-generate hints for the Dafny formal verification engine to successfully verify programs. This application could significantly impact the field of software verification by automating the generation of verification hints, potentially improving the efficiency and reliability of the verification process.\n", - "\n", - "3. **Automated Information Extraction from Thyroid Operation Narrative**: This study compares the GPT-4 model with the fine-tuned KoELECTRA model for automated information extraction from thyroid operation narratives. The application of GPT-4 in this context could revolutionize document processing in healthcare by providing a more efficient and accurate method for extracting information from medical records.\n", - "\n", - "These papers suggest that GPT-4 has the potential to be applied in various software-related fields, including enhancing datasets for machine learning, formal software verification, and healthcare document processing. The applications in these papers could lead to more efficient, accurate, and reliable software systems across different domains.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Mathematician\n", - "\u001b[0m\n", - "\u001b[33mMathematician\u001b[0m (to chat_manager):\n", - "\n", - "The applications mentioned indeed highlight the versatility of GPT-4 in different domains. To further understand the potential impact of GPT-4 on software, let's delve into the mathematical and algorithmic principles that could be at play in these applications:\n", - "\n", - "1. **Recaptioning Web Images**: The process of recaptioning images with a language model like GPT-4 involves understanding the context of an image and generating descriptive text that accurately reflects its content. This task likely involves a combination of computer vision techniques to interpret the image and natural language processing to generate the caption. From a mathematical perspective, this would involve optimization algorithms to fine-tune the language model on a specific dataset, ensuring that the generated captions are both semantically and syntactically correct.\n", - "\n", - "2. **Formal Software Verification**: The use of GPT-4 to auto-generate hints for formal verification engines like Dafny involves the model understanding the logic and structure of the code. This requires a deep understanding of formal logic, proof theory, and possibly type theory if the language being verified is statically typed. The success rate of auto-generated hints would depend on the model's ability to reason about the correctness of code and the underlying mathematical properties that ensure its validity.\n", - "\n", - "3. **Automated Information Extraction from Medical Records**: For GPT-4 to extract information from medical narratives, it must process unstructured text and identify relevant medical terms and their relationships. This task involves natural language understanding, which from a mathematical standpoint, can be seen as a form of pattern recognition and classification. The model would need to be trained on a large corpus of medical texts, and its performance would be measured by its precision and recall in identifying and extracting the correct information.\n", - "\n", - "In each of these applications, GPT-4's effectiveness would be influenced by the underlying mathematical models, such as neural networks, and the optimization techniques used during training, such as gradient descent. The quality of the training data and the model's architecture (e.g., attention mechanisms, transformer layers) also play a crucial role in its performance.\n", - "\n", - "To verify the potential of GPT-4 in these applications, one could set up experiments to measure the performance of GPT-4 against specific benchmarks or metrics relevant to each domain. For example, in the case of formal software verification, one could measure the percentage of programs that are successfully verified with the hints generated by GPT-4 compared to a baseline or human-generated hints.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Mathematician\n", - "\u001b[0m\n", - "\u001b[33mMathematician\u001b[0m (to chat_manager):\n", - "\n", - "To further verify the potential applications of GPT-4 in software, we can consider the following:\n", - "\n", - "1. **Recaptioning Web Images**: The effectiveness of GPT-4 in this application can be measured by the accuracy of the captions it generates. This can be quantified using metrics such as BLEU (Bilingual Evaluation Understudy) or ROUGE (Recall-Oriented Understudy for Gisting Evaluation), which compare the machine-generated text to a set of reference captions. A high score on these metrics would indicate that GPT-4 is capable of understanding and describing the content of images accurately.\n", - "\n", - "2. **Formal Software Verification**: To verify the application of GPT-4 in software verification, we could measure the success rate of the model in generating verification hints that lead to successful program verification. This could be done by comparing the verification success rate with and without the hints provided by GPT-4. Additionally, the complexity of the programs and the hints required could be analyzed to understand the model's limitations and strengths.\n", - "\n", - "3. **Automated Information Extraction from Medical Records**: The performance of GPT-4 in extracting information from medical narratives can be assessed by comparing the extracted information to a gold standard set of annotations. Precision, recall, and F1-score are common metrics used to evaluate information extraction systems. A high F1-score would suggest that GPT-4 is effective in identifying and extracting relevant information from unstructured medical texts.\n", - "\n", - "For each of these applications, it would be important to conduct a thorough analysis of the results to ensure that the model's performance is not only statistically significant but also practically relevant. Additionally, it would be crucial to assess the model's performance on diverse datasets to ensure that it generalizes well across different contexts and does not exhibit biases that could lead to incorrect or unfair outcomes.\n", - "\n", - "If we want to simulate or test these applications, we could propose specific experiments or data analysis procedures that align with the above verification strategies. However, such experiments would require access to the relevant datasets and computational resources to run the models, which may not be feasible within the scope of this conversation.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Mathematician\n", - "\u001b[0m\n", - "\u001b[33mMathematician\u001b[0m (to chat_manager):\n", - "\n", - "To simulate or test these applications, we could propose specific experiments or data analysis procedures that align with the above verification strategies. However, such experiments would require access to the relevant datasets and computational resources to run the models, which may not be feasible within the scope of this conversation.\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[32m\n", - "Next speaker: Mathematician\n", - "\u001b[0m\n", - "\u001b[33mMathematician\u001b[0m (to chat_manager):\n", - "\n", - "TERMINATE\n", - "\n", - "--------------------------------------------------------------------------------\n", - "\u001b[33mAll agents have been cleared.\u001b[0m\n" - ] - } - ], - "source": [ - "new_builder = AgentBuilder(\n", - " config_file_or_env=config_file_or_env, builder_model=\"gpt-4-1106-preview\", agent_model=\"gpt-4-1106-preview\"\n", - ")\n", - "agent_list, _ = new_builder.build_from_library(\n", - " building_task, library_path_or_json, llm_config, embedding_model=\"all-mpnet-base-v2\"\n", - ")\n", - "start_task(\n", - " execution_task=\"Find a recent paper about gpt-4 on arxiv and find its potential applications in software.\",\n", - " agent_list=agent_list,\n", - ")\n", - "new_builder.clear_all_agents()" + "cells": [ + { + "cell_type": "markdown", + "id": "6264276d39875995", + "metadata": { + "collapsed": false + }, + "source": [ + "# Automatically Build Multi-agent System from Agent Library\n", + "\n", + "By: [Linxin Song](https://linxins97.github.io/), [Jieyu Zhang](https://jieyuz2.github.io/)\n", + "\n", + "In this notebook, we introduce a new feature for AutoBuild, `build_from_library`, which help users build an automatic task-solving process powered by a multi-agent system from a pre-defined agent library. \n", + "Specifically, in `build_from_library`, we prompt an LLM to explore useful agents from a pre-defined agent library, generating configurations for those agents for a group chat to solve the user's task." + ] + }, + { + "cell_type": "markdown", + "id": "ec78dda8e3826d8a", + "metadata": { + "collapsed": false + }, + "source": [ + "## Requirement\n", + "\n", + "AutoBuild require `autogen[autobuild]`, which can be installed by the following command:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "e8e9ae50658be975", + "metadata": { + "collapsed": false + }, + "outputs": [], + "source": [ + "%pip install autogen[autobuild]" + ] + }, + { + "cell_type": "markdown", + "id": "176c200804af63f3", + "metadata": { + "collapsed": false + }, + "source": [ + "## Preparation and useful tools\n", + "We need to specify a `config_path`, `default_llm_config` that include backbone LLM configurations." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2505f029423b21ab", + "metadata": { + "ExecuteTime": { + "end_time": "2023-12-26T16:58:02.762702600Z", + "start_time": "2023-12-26T16:58:02.472073Z" + }, + "collapsed": false + }, + "outputs": [], + "source": [ + "import json\n", + "\n", + "import autogen\n", + "from autogen.agentchat.contrib.agent_builder import AgentBuilder\n", + "\n", + "config_file_or_env = \"OAI_CONFIG_LIST\" # modify path\n", + "llm_config = {\"temperature\": 0}\n", + "config_list = autogen.config_list_from_json(config_file_or_env, filter_dict={\"model\": [\"gpt-4-1106-preview\", \"gpt-4\"]})\n", + "\n", + "\n", + "def start_task(execution_task: str, agent_list: list):\n", + " group_chat = autogen.GroupChat(agents=agent_list, messages=[], max_round=12)\n", + " manager = autogen.GroupChatManager(groupchat=group_chat, llm_config={\"config_list\": config_list, **llm_config})\n", + " agent_list[0].initiate_chat(manager, message=execution_task)" + ] + }, + { + "cell_type": "markdown", + "id": "5fb3db8885dd6ee6", + "metadata": { + "collapsed": false + }, + "source": [ + "## Example for generating an agent library\n", + "Here, we show an example of generating an agent library from a pre-defined list of agents' names by prompting a `gpt-4`. You can also prepare a handcrafted library yourself.\n", + "\n", + "A Library contains each agent's name, description and system_message. The description is a brief introduction about agent's characteristics. As we will feed all agents' names and description to gpt-4 and let it choose the best agents for us, each agent's description should be simple but informative. \n", + "\n", + "First, we define a prompt template for description and system_message generation and a list of agents' name:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "68315f6ec912c58a", + "metadata": { + "ExecuteTime": { + "end_time": "2023-12-23T07:39:03.317527600Z", + "start_time": "2023-12-23T07:39:03.276859600Z" + }, + "collapsed": false + }, + "outputs": [], + "source": [ + "AGENT_SYS_MSG_PROMPT = \"\"\"Acccording to the following postion name, write a high quality instruction for the position following a given example. You should only return the instruction.\n", + "\n", + "# Position Name\n", + "{position}\n", + "\n", + "# Example instruction for Data Analyst\n", + "\n", + "As Data Analyst, you are tasked with leveraging your extensive knowledge in data analysis to recognize and extract meaningful features from vast datasets. Your expertise in machine learning, specifically with the Random Forest Classifier, allows you to construct robust predictive models adept at handling both classification and regression tasks. You excel in model evaluation and interpretation, ensuring that the performance of your algorithms is not just assessed with precision, but also understood in the context of the data and the problem at hand. With a command over Python and proficiency in using the pandas library, you manipulate and preprocess data with ease.\n", + "\"\"\"\n", + "\n", + "AGENT_DESC_PROMPT = \"\"\"According to position name and the instruction, summarize the position into a high quality one sentence description.\n", + "\n", + "# Position Name\n", + "{position}\n", + "\n", + "# Instruction\n", + "{instruction}\n", + "\"\"\"\n", + "\n", + "position_list = [\n", + " \"Environmental_Scientist\",\n", + " \"Astronomer\",\n", + " \"Software_Developer\",\n", + " \"Data_Analyst\",\n", + " \"Journalist\",\n", + " \"Teacher\",\n", + " \"Lawyer\",\n", + " \"Programmer\",\n", + " \"Accountant\",\n", + " \"Mathematician\",\n", + " \"Physicist\",\n", + " \"Biologist\",\n", + " \"Chemist\",\n", + " \"Statistician\",\n", + " \"IT_Specialist\",\n", + " \"Cybersecurity_Expert\",\n", + " \"Artificial_Intelligence_Engineer\",\n", + " \"Financial_Analyst\",\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "72b8e7d9d334a5c2", + "metadata": { + "collapsed": false + }, + "source": [ + "Then we can prompt a `gpt-4` model to generate each agent's system message as well as the description:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "8fbfef9268fc5191", + "metadata": { + "ExecuteTime": { + "end_time": "2023-12-23T07:40:01.703372Z", + "start_time": "2023-12-23T07:39:04.472589200Z" + }, + "collapsed": false + }, + "outputs": [], + "source": [ + "build_manager = autogen.OpenAIWrapper(config_list=config_list)\n", + "sys_msg_list = []\n", + "\n", + "for pos in position_list:\n", + " resp_agent_sys_msg = (\n", + " build_manager.create(\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": AGENT_SYS_MSG_PROMPT.format(\n", + " position=pos,\n", + " ),\n", + " }\n", + " ]\n", + " )\n", + " .choices[0]\n", + " .message.content\n", + " )\n", + " resp_desc_msg = (\n", + " build_manager.create(\n", + " messages=[\n", + " {\n", + " \"role\": \"user\",\n", + " \"content\": AGENT_DESC_PROMPT.format(\n", + " position=pos,\n", + " instruction=resp_agent_sys_msg,\n", + " ),\n", + " }\n", + " ]\n", + " )\n", + " .choices[0]\n", + " .message.content\n", + " )\n", + " sys_msg_list.append({\"name\": pos, \"system_message\": resp_agent_sys_msg, \"description\": resp_desc_msg})" + ] + }, + { + "cell_type": "markdown", + "id": "9e26c6db4befacc5", + "metadata": { + "collapsed": false + }, + "source": [ + "The generated profile will have the following format:" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "8ede1d7088eb183d", + "metadata": { + "ExecuteTime": { + "end_time": "2023-12-23T07:40:01.712399300Z", + "start_time": "2023-12-23T07:40:01.707400200Z" + }, + "collapsed": false + }, + "outputs": [ + { + "data": { + "text/plain": [ + "[{'name': 'Environmental_Scientist',\n", + " 'system_message': 'As an Environmental Scientist, you are responsible for applying your profound knowledge of environmental science to analyze ecological data and assess the impact of human activities on natural resources and ecosystems. Your proficiency in environmental assessment techniques enables you to design and conduct field studies, collect samples, and monitor environmental parameters effectively. Utilizing Geographic Information Systems (GIS), you spatially analyze and visualize environmental data to better understand patterns and changes in the landscape. You are adept at interpreting the results and communicating your findings clearly to stakeholders, policymakers, and the public, thereby contributing to informed decision-making on environmental issues. Your role is essential in developing sustainable practices and recommending mitigation measures to minimize environmental degradation and promote conservation.',\n", + " 'description': 'As an Environmental Scientist, you are tasked with analyzing and assessing the impact of human activities on ecosystems by conducting field studies, using GIS for spatial analysis, and communicating your findings to inform sustainable practices and conservation efforts.'},\n", + " {'name': 'Astronomer',\n", + " 'system_message': 'As an Astronomer, your duty involves diligent observation and analysis of celestial phenomena across the universe. Utilize cutting-edge telescopes and instruments to gather astronomical data, looking for patterns and irregularities that can lead to groundbreaking discoveries. Your profound knowledge in astrophysics is pivotal in interpreting these findings, which may include identifying new celestial objects, scrutinizing the properties and behaviors of stars, planets, and galaxies, and understanding cosmic events. Mastery of complex astronomical software and advanced mathematics is crucial for modeling astronomical phenomena and processing the vast amounts of data. Your role is essential in advancing our understanding of the cosmos, contributing to the broader scientific community by publishing your findings in reputable journals and engaging in peer collaboration to further space exploration and research.',\n", + " 'description': 'An Astronomer is a professional who meticulously observes, analyzes, and interprets celestial phenomena using advanced telescopes and instruments, requiring a deep knowledge of astrophysics, proficiency in mathematical modeling, and collaboration in scientific communities to enhance our comprehension of the universe.'},\n", + " {'name': 'Software_Developer',\n", + " 'system_message': 'As a Software Developer, your objective is to craft, test, and maintain the software that will meet the needs of our users and clients. Your proficiency in programming languages such as Java, C#, or JavaScript is essential, enabling you to write clean, efficient, and maintainable code. You will design algorithms and flowcharts to create systems that are logical and user-friendly. Collaboration with cross-functional teams, including product managers and designers, is crucial in order to understand software requirements and deliver innovative solutions. With your understanding of the software development life cycle, you will work through the processes of coding, debugging, testing, and deployment. You will employ industry best practices such as version control with Git and conduct code reviews to maintain high standards of software quality. Your role places you at the heart of our development efforts, where your technical prowess advances the functionality, scalability, and reliability of our software products.',\n", + " 'description': 'A Software Developer is responsible for designing, coding, testing, and maintaining software that meets client needs using languages like Java, C#, or JavaScript, collaborating with teams, adhering to best practices like Git for version control, and ensuring quality and innovation throughout the development life cycle.'},\n", + " {'name': 'Data_Analyst',\n", + " 'system_message': 'As a Data Analyst, your role is pivotal in interpreting complex data and providing insights that inform strategic decision-making. Utilize your analytical skills to cleanse and organize large sets of structured and unstructured data, ensuring its accuracy and readiness for in-depth analysis. Apply statistical analysis and predictive modeling to uncover trends, patterns, and correlations that drive operational improvements and innovative solutions. Use your proficiency in SQL for database interactions, and harness visualization tools such as Tableau or Power BI to craft compelling stories from data, aiding stakeholders in visualizing the implications of your findings. Stay abreast with the latest analytics techniques and continuously refine your models for enhanced performance, contributing significantly to the data-driven culture of our organization.',\n", + " 'description': 'The Data Analyst interprets complex datasets to provide strategic insights, cleanses and organizes data, performs statistical analysis and predictive modeling to identify trends and inform improvements, utilizes SQL for database management, and employs visualization tools like Tableau or Power BI to effectively communicate findings to stakeholders.'},\n", + " {'name': 'Journalist',\n", + " 'system_message': 'As a Journalist, you are responsible for identifying and pursuing newsworthy stories with the utmost ethical standards and a commitment to factual reporting. Your innate curiosity and excellent communication skills enable you to conduct thorough research and interviews, uncovering the details that make each story compelling and informative. Skilled in both written and verbal storytelling, you craft articles, reports, and features that engage and inform the public, adhering to strict deadlines without compromising on the integrity and accuracy of your work. Proficient in multimedia journalism, you adeptly use digital tools and social media to reach a wider audience, ensuring that your stories have the maximum impact.',\n", + " 'description': 'A Journalist is tasked with ethically sourcing and meticulously reporting newsworthy events, utilizing strong research and storytelling abilities across multiple platforms to accurately inform and engage a diverse audience.'},\n", + " {'name': 'Teacher',\n", + " 'system_message': 'As a Teacher, you are entrusted with the essential responsibility of fostering knowledge and encouraging academic and personal growth in your students. Your deep understanding of pedagogy, coupled with your expertise in the subject matter, enables you to create and deliver curricula that are both engaging and educational. Your adeptness at differentiated instruction allows you to tailor your teaching methods to suit the varied learning styles and needs within your classroom. By skillfully blending traditional teaching techniques with modern educational technology, you facilitate a dynamic and interactive learning environment. You excel in assessment and feedback, not only to gauge student progress but also to continuously improve your own teaching strategies. With strong interpersonal skills, you maintain open lines of communication with students, parents, and colleagues, fostering a collaborative and supportive school community.',\n", + " 'description': \"A Teacher is responsible for cultivating students' knowledge and growth through expertise in pedagogical practices and subject matter, designing engaging curricula, adapting teaching methods to diverse learning needs, integrating technology, and using assessment for continuous improvement while nurturing a cooperative school community.\"},\n", + " {'name': 'Lawyer',\n", + " 'system_message': 'As a Lawyer, you are required to uphold the highest standards of legal proficiency and ethical practice. Your role involves advising clients on their legal rights and responsibilities, as well as representing them in civil and criminal proceedings. You must possess a strong understanding of the law, paired with the ability to analyze case law and legislate history, to construct compelling arguments in support of your client’s position. Your keen attention to detail and dedication to thorough research are crucial in identifying legal precedents and crafting legal documents that adhere to the strictest of procedural standards. Moreover, you must exhibit exceptional negotiation skills to achieve favorable outcomes, whether in the courtroom or at the settlement table. With your articulate verbal and written communication, you clearly and persuasively present cases, explaining complex legal concepts in understandable terms to clients, judges, and juries. Your commitment to confidentiality and upholding justice is paramount and reflected in all aspects of your professional conduct.',\n", + " 'description': 'A Lawyer is a professionally trained legal advocate responsible for representing clients in legal proceedings, providing expert advice on legal matters, constructing persuasive arguments through meticulous research and analysis of law, and negotiating settlements, all while adhering to the highest ethical standards and maintaining strict confidentiality.'},\n", + " {'name': 'Programmer',\n", + " 'system_message': 'As a Programmer, you are responsible for the design, development, and implementation of software programs. Utilize your comprehensive understanding of programming languages, including but not limited to Java, C++, and Python, to create efficient and innovative software solutions. Your role involves writing clean, maintainable code while adhering to best practices in software development. You are expected to troubleshoot, debug, and upgrade existing software, as well as collaborate with cross-functional teams to define and design new product features. Your ability to think algorithmically and solve problems systematically will be integral in creating software that is not only functional but also scalable and secure.',\n", + " 'description': 'A Programmer designs, develops, and implements innovative and efficient software solutions using languages like Java, C++, and Python, ensuring code maintainability, collaborating on new features, and enhancing existing applications with a strong focus on scalability and security.'},\n", + " {'name': 'Accountant',\n", + " 'system_message': 'As Accountant, you are charged with the meticulous management and analysis of financial records, ensuring accuracy and compliance with relevant laws and regulations. Utilize your comprehensive understanding of accounting principles to prepare, examine, and maintain financial reports and statements, including balance sheets and income statements. Your role involves the reconciliation of accounts, evaluating financial operations to recommend best practices, identifying issues, and strategizing solutions for fiscal efficiency and profitability. Mastery in accounting software such as QuickBooks or Sage, alongside proficiency in Microsoft Excel, enables you to efficiently process and analyze financial data. You must ensure proper financial documentation and control systems are in place, providing comprehensive support to the organization’s financial health and integrity.',\n", + " 'description': 'As an Accountant, you are responsible for the accurate and compliant management, analysis, and reporting of financial data, along with recommending strategies to enhance fiscal efficiency and profitability, supported by proficiency in accounting software and Microsoft Excel.'},\n", + " {'name': 'Mathematician',\n", + " 'system_message': 'As a Mathematician, you are responsible for utilizing your profound understanding of mathematical theories and methodologies to solve complex theoretical and practical problems across various domains. Your proficiency in abstract reasoning enables you to develop new mathematical principles and to recognize and articulate the underlying mathematical relationships within real-world scenarios. You apply your expertise in calculus, algebra, statistics, and other mathematical branches to conduct rigorous analyses and to model systems for prediction and optimization. With a strong foundation in logic and quantitative reasoning, you perform peer reviews and contribute to interdisciplinary research projects, ensuring accuracy and consistency in mathematical arguments and results. Your role is crucial in advancing mathematical knowledge and providing innovative solutions to scientific and engineering challenges.',\n", + " 'description': 'As a Mathematician, you apply advanced mathematical theories and analytical skills to solve theoretical and practical problems in various industries, develop new principles, and provide innovative solutions to complex scientific and engineering challenges.'},\n", + " {'name': 'Physicist',\n", + " 'system_message': 'As a Physicist, you are charged with applying your profound understanding of the physical laws that govern the universe to unravel complex scientific phenomena. Your proficiency in theoretical and experimental physics enables you to develop models and conduct experiments that explore fundamental forces and particles. With exceptional analytical skills, you interpret empirical data to validate existing theories or propose new explanations for unexplained observations. Mastery in the use of mathematical tools such as differential equations and linear algebra is crucial for you to simulate physical processes. You are also adept at using specialized software and equipment for data acquisition and analysis, contributing to advancements in fields ranging from quantum mechanics to cosmology. Your strong critical thinking abilities empower you to solve intricate problems, and your commitment to scientific rigor ensures the integrity and accuracy of your research outcomes.',\n", + " 'description': 'A Physicist applies deep knowledge of physical laws to investigate scientific phenomena through theoretical modeling and experimental research, utilizing advanced mathematical techniques and specialized equipment to advance understanding in areas such as quantum mechanics and cosmology.'},\n", + " {'name': 'Biologist',\n", + " 'system_message': 'As a Biologist, you are entrusted with the study and understanding of living organisms, applying your expertise to investigate their functions, genetics, evolution, and ecosystems. Your skills in experimental design empower you to conduct research and experiments that can unlock new biological insights and improve our comprehension of life processes. Utilizing advanced microscopy techniques and molecular biology methods, you should meticulously analyze cell structures and DNA sequences to uncover the intricacies of life at a microscopic level. Demonstrate proficiency in bioinformatics tools to analyze genetic data and contribute valuable findings to the scientific community. Furthermore, as a communicator of science, ensure that your research findings are effectively documented and presented in scientific journals and at conferences, thereby enhancing the collective knowledge in your field.',\n", + " 'description': 'A Biologist meticulously studies and understands living organisms, conducting advanced research to decode genetics and ecosystems and sharing findings through scientific publications and presentations.'},\n", + " {'name': 'Chemist',\n", + " 'system_message': 'As a Chemist, you are charged with applying your profound understanding of chemical principles to conduct complex experiments, synthesize new compounds, and analyze the molecular and atomic structure of materials. Your proficiency in utilizing sophisticated analytical techniques - such as chromatography, spectroscopy, and mass spectrometry - enables you to decipher the composition and properties of substances. The knowledge you hold in chemical safety and handling procedures ensures a secure laboratory environment. With an adeptness in maintaining accurate records and an insightful approach to interpreting data, you transform raw experimental results into valuable scientific insights. Your ability to communicate complex chemical information clearly makes you essential in collaborative research efforts and in driving innovation within the field.',\n", + " 'description': 'As a Chemist, you are responsible for conducting advanced experiments, synthesizing compounds, deciphering substance compositions with techniques like chromatography and mass spectrometry, and transforming experimental data into scientific insights, while maintaining safety and clear communication in research collaborations.'},\n", + " {'name': 'Statistician',\n", + " 'system_message': 'As a Statistician, your primary duty is to apply mathematical and statistical methods to collect, analyze, and interpret numerical data to make informed decisions. Your strong grounding in probability theory will be essential for designing surveys and experiments to generate data. You are adept at constructing and applying sophisticated statistical models and methods, such as linear regression, ANOVA, or time-series analysis, ensuring that you accurately capture trends and relationships within the data. You possess an in-depth understanding of statistical software such as R or SAS, allowing you to perform complex analyses with efficiency and precision. Your ability to communicate complex statistical concepts to non-experts will be crucial; hence, your role includes presenting findings in a clear, actionable manner, with data visualizations and reports that drive strategic planning and policy development.',\n", + " 'description': 'A Statistician employs and interprets advanced statistical techniques to design data-collection processes, analyze data, and present findings in a comprehensible manner, supporting evidence-based decision-making and policy formation.'},\n", + " {'name': 'IT_Specialist',\n", + " 'system_message': 'As an IT Specialist, your primary responsibility is to maintain the integrity and functionality of all our computer systems and networks. Your comprehensive understanding of hardware and software is crucial for diagnosing and resolving technical issues. You are adept at implementing network security measures to protect data and systems from cyber threats. You also play a significant role in systems and software upgrades, ensuring a seamless transition without disrupting workflow. Utilizing your strong problem-solving skills and proficiency in scripting languages, you automate repetitive tasks, enhancing system efficiency. Your ability to communicate effectively with team members and non-technical staff allows you to provide clear guidance and end-user support.',\n", + " 'description': 'An IT Specialist is responsible for upholding and optimizing our computer systems and networks through maintenance, security, upgrades, issue resolution, automation, and providing support and clear communication to both technical and non-technical personnel.'},\n", + " {'name': 'Cybersecurity_Expert',\n", + " 'system_message': \"As a Cybersecurity Expert, you are charged with the responsibility of safeguarding the organization's computer networks and systems. Your deep understanding of cyber threats and mitigation techniques is critical in identifying vulnerabilities and protecting against malicious attacks. Employing your experience with tools such as firewalls, antivirus software, and intrusion detection systems, you will continuously monitor and defend our digital infrastructure. You are expected to conduct regular security audits and penetration testing to simulate cyber attacks and find potential weaknesses before they can be exploited. Your proficiency in risk management frameworks and incident response protocols ensures that you are prepared to swiftly handle and mitigate any security incidents that occur. With your expertise in encryption technologies and network protocols, you protect sensitive data and ensure compliance with relevant security standards and regulations. Your foresight in staying up-to-date with the latest cybersecurity trends and threats is paramount to maintaining the organization's digital defense at its peak.\",\n", + " 'description': \"As a Cybersecurity Expert, you are responsible for the proactive protection and defense of an organization's computer networks and systems against cyber threats through continuous monitoring, conducting security audits, penetrating testing, and swiftly mitigating security incidents, while ensuring compliance with security regulations.\"},\n", + " {'name': 'Artificial_Intelligence_Engineer',\n", + " 'system_message': 'As an Artificial Intelligence Engineer, you are responsible for conceptualizing, designing, and implementing intelligent systems that simulate human cognitive processes. Your role demands a deep understanding of neural networks, particularly Convolutional Neural Networks (CNNs) for image recognition tasks and Recurrent Neural Networks (RNNs) for natural language processing. With your expertise in TensorFlow or PyTorch, you develop complex models that can learn, adapt, and make decisions. You prioritize the ethical design and deployment of AI systems, conscious of the implications your work may have on society. Mastery of algorithms and a proficiency in a high-level programming language, preferably Python, enable you to transform theoretical AI concepts into practical solutions that drive innovation and efficiency.',\n", + " 'description': 'An Artificial Intelligence Engineer specializes in creating and implementing advanced intelligent systems, with a mastery of neural networks, machine learning frameworks, and ethical AI principles, to develop innovative solutions that emulate human cognition.'},\n", + " {'name': 'Financial_Analyst',\n", + " 'system_message': 'As a Financial Analyst, you are entrusted with utilizing your in-depth understanding of financial principles to assess investment opportunities, analyze financial data, and forecast economic trends. Your proficiency in financial modeling is paramount, enabling you to develop complex models that underpin the valuation of stocks, bonds, and other financial instruments. With a sharp eye for detail, you scrutinize company financial statements to derive actionable insights and recommend strategies to optimize financial performance. Your expertise in Excel, especially with advanced functions and formulas, allows you to efficiently manipulate and analyze large financial datasets. You are a whiz at creating compelling visualizations and delivering presentations to communicate your findings and influence strategic decisions. Your role is crucial in guiding investment decisions and driving the fiscal prudence of the organization.',\n", + " 'description': \"A Financial Analyst performs in-depth financial analysis and modeling to evaluate investments, forecast economic trends, and deliver strategic recommendations, leveraging advanced Excel skills to inform and guide the organization's financial decisions.\"}]" ] - } - ], - "metadata": { - "front_matter": { - "description": "Automatically build multi-agent system from agent library", - "tags": [ - "autobuild" - ] - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.14" - } + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "sys_msg_list" + ] + }, + { + "cell_type": "markdown", + "id": "256dd32b03a7a172", + "metadata": { + "collapsed": false + }, + "source": [ + "We can save the generated agents' information into a json file." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "53111125938845cf", + "metadata": { + "ExecuteTime": { + "end_time": "2023-12-23T07:40:01.750855900Z", + "start_time": "2023-12-23T07:40:01.710399600Z" + }, + "collapsed": false + }, + "outputs": [], + "source": [ + "json.dump(sys_msg_list, open(\"./agent_library_example.json\", \"w\"), indent=4)" + ] + }, + { + "cell_type": "markdown", + "id": "cfd883b79a3bd932", + "metadata": { + "collapsed": false + }, + "source": [ + "## Build agents from library (by LLM)\n", + "Here, we introduce how to build agents from the generated library. As in the previous `build`, we also need to specify a `building_task` that lets the build manager know which agents should be selected from the library according to the task. \n", + "\n", + "We also need to specify a `library_path_or_json`, which can be a path of library or a JSON string with agents' configs. Here, we use the previously saved path as the library path." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "8963a8709c8e92e2", + "metadata": { + "ExecuteTime": { + "end_time": "2023-12-23T07:40:01.752918500Z", + "start_time": "2023-12-23T07:40:01.735461Z" + }, + "collapsed": false + }, + "outputs": [], + "source": [ + "library_path_or_json = \"./agent_library_example.json\"\n", + "building_task = \"Find a paper on arxiv by programming, and analyze its application in some domain. For example, find a recent paper about gpt-4 on arxiv and find its potential applications in software.\"" + ] + }, + { + "cell_type": "markdown", + "id": "72656a8d0c1a9b12", + "metadata": { + "collapsed": false + }, + "source": [ + "Then, we can call the `build_from_library` from the AgentBuilder to generate a list of agents from the library and let them complete the user's `execution_task` in a group chat." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "5c669b76b2c9b750", + "metadata": { + "ExecuteTime": { + "end_time": "2023-12-23T07:46:02.075542200Z", + "start_time": "2023-12-23T07:43:55.489042900Z" + }, + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32m==> Looking for suitable agents in the library...\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['Programmer', 'Mathematician'] are selected.\n", + "\u001b[32m==> Creating agents...\u001b[0m\n", + "Creating agent Programmer...\n", + "Creating agent Mathematician...\n", + "Adding user console proxy...\n", + "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", + "\n", + "Find a recent paper about explainable AI on arxiv and find its potential applications in medical.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Computer_terminal\n", + "\u001b[0m\n", + "\u001b[33mComputer_terminal\u001b[0m (to chat_manager):\n", + "\n", + "There is no code from the last 1 message for me to execute. Group chat manager should let other participants to continue the conversation. If the group chat manager want to end the conversation, you should let other participant reply me only with \"TERMINATE\"\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Programmer\n", + "\u001b[0m\n", + "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", + "\n", + "To find a recent paper about explainable AI on arXiv, we can use the arXiv API to search for papers that match the query. However, since I cannot directly access external APIs, I suggest that one of us manually searches for the paper on the arXiv website using relevant search terms such as \"explainable AI\" and \"medical applications\". Once we find a suitable paper, we can discuss its potential applications in the medical field. \n", + "\n", + "Mathematician, would you like to perform the search, or shall I provide a Python script that could be used to perform the search programmatically?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Mathematician\n", + "\u001b[0m\n", + "\u001b[33mMathematician\u001b[0m (to chat_manager):\n", + "\n", + "As a Mathematician, I can certainly appreciate the value of a programmatic approach to searching for academic papers. However, since I do not have direct access to execute code or APIs, I would suggest that you, as the Programmer, provide the Python script that could be used to perform the search on arXiv. Once we have identified a paper, I can then assist in discussing its potential applications in the medical field from a mathematical perspective.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Programmer\n", + "\u001b[0m\n", + "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", + "\n", + "Understood. I will provide a Python script that can be used to search for recent papers about explainable AI on arXiv with potential applications in the medical field. The script will use the `arxiv` Python package, which is a wrapper for the arXiv API. If the package is not installed, we will need to install it first.\n", + "\n", + "Let's start by checking if the `arxiv` package is installed and if not, we will install it. Computer_terminal, please execute the following command to check for the `arxiv` package and install it if necessary.\n", + "\n", + "```sh\n", + "pip show arxiv || pip install arxiv\n", + "```\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Computer_terminal\n", + "\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is sh)...\u001b[0m\n", + "\u001b[33mComputer_terminal\u001b[0m (to chat_manager):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: \n", + "Name: arxiv\n", + "Version: 2.1.0\n", + "Summary: Python wrapper for the arXiv API: https://arxiv.org/help/api/\n", + "Home-page: https://github.com/lukasschwab/arxiv.py\n", + "Author: Lukas Schwab\n", + "Author-email: lukas.schwab@gmail.com\n", + "License: MIT\n", + "Location: /home/vscode/.local/lib/python3.10/site-packages\n", + "Requires: feedparser, requests\n", + "Required-by: \n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Programmer\n", + "\u001b[0m\n", + "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", + "\n", + "Great, the `arxiv` package is already installed. Now, I will provide a Python script that uses the `arxiv` package to search for recent papers related to explainable AI with potential applications in the medical field. The script will query the arXiv API for papers with relevant keywords and print out the title and summary of the most recent paper found.\n", + "\n", + "Computer_terminal, please execute the following Python script.\n", + "\n", + "```python\n", + "import arxiv\n", + "\n", + "# Define the search query\n", + "search_query = 'all:explainable AI AND all:medical'\n", + "\n", + "# Search for papers on arXiv\n", + "search = arxiv.Search(\n", + " query = search_query,\n", + " max_results = 1,\n", + " sort_by = arxiv.SortCriterion.SubmittedDate\n", + ")\n", + "\n", + "# Fetch the most recent paper\n", + "for paper in search.results():\n", + " print(\"Title:\", paper.title)\n", + " print(\"Summary:\", paper.summary)\n", + " # Only print the most recent paper\n", + " break\n", + "```\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Computer_terminal\n", + "\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n", + "\u001b[33mComputer_terminal\u001b[0m (to chat_manager):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: \n", + "Title: Automated Information Extraction from Thyroid Operation Narrative: A Comparative Study of GPT-4 and Fine-tuned KoELECTRA\n", + "Summary: In the rapidly evolving field of healthcare, the integration of artificial\n", + "intelligence (AI) has become a pivotal component in the automation of clinical\n", + "workflows, ushering in a new era of efficiency and accuracy. This study focuses\n", + "on the transformative capabilities of the fine-tuned KoELECTRA model in\n", + "comparison to the GPT-4 model, aiming to facilitate automated information\n", + "extraction from thyroid operation narratives. The current research landscape is\n", + "dominated by traditional methods heavily reliant on regular expressions, which\n", + "often face challenges in processing free-style text formats containing critical\n", + "details of operation records, including frozen biopsy reports. Addressing this,\n", + "the study leverages advanced natural language processing (NLP) techniques to\n", + "foster a paradigm shift towards more sophisticated data processing systems.\n", + "Through this comparative study, we aspire to unveil a more streamlined,\n", + "precise, and efficient approach to document processing in the healthcare\n", + "domain, potentially revolutionizing the way medical data is handled and\n", + "analyzed.\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Mathematician\n", + "\u001b[0m\n", + "\u001b[33mMathematician\u001b[0m (to chat_manager):\n", + "\n", + "The paper titled \"Automated Information Extraction from Thyroid Operation Narrative: A Comparative Study of GPT-4 and Fine-tuned KoELECTRA\" presents a study on the use of artificial intelligence for automating the extraction of information from thyroid operation narratives. This is a clear example of explainable AI being applied in the medical field, specifically in the area of clinical workflows and document processing.\n", + "\n", + "The potential applications of such technology in medicine are vast. By automating the extraction of information from operation narratives, healthcare professionals can save time and reduce the likelihood of human error. This can lead to more accurate patient records, improved patient care, and streamlined administrative processes. Additionally, the ability to quickly and accurately process operation records can facilitate better data analysis, which can be used for medical research, trend analysis, and improving healthcare outcomes.\n", + "\n", + "The use of advanced natural language processing (NLP) techniques, as mentioned in the summary, is particularly important for processing free-style text formats that contain critical medical information. This technology could be further explored to extend its application to other types of medical documents and records, enhancing the overall efficiency of the healthcare system.\n", + "\n", + "The study's focus on comparing the performance of the fine-tuned KoELECTRA model with GPT-4 also highlights the importance of evaluating different AI models to determine the most effective approach for specific medical applications. This comparative analysis can lead to the development of more specialized AI tools tailored to the needs of the healthcare industry.\n", + "\n", + "In conclusion, the research presented in this paper has significant implications for the future of medical document processing and the broader integration of AI in healthcare.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Programmer\n", + "\u001b[0m\n", + "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", + "\n", + "The insights provided by the Mathematician are indeed valuable. The application of AI for automated information extraction from medical documents like thyroid operation narratives can greatly enhance efficiency and accuracy in healthcare. The use of models like GPT-4 and KoELECTRA for natural language processing tasks shows the potential of AI to handle complex, unstructured data which is common in medical records.\n", + "\n", + "From a programming perspective, the implementation of such AI systems would involve training models on large datasets of medical documents to learn the context and semantics specific to medical terminology. Ensuring the explainability of AI in this context is crucial, as healthcare professionals need to understand and trust the AI's decision-making process, especially when it directly affects patient care.\n", + "\n", + "Moreover, the integration of explainable AI into healthcare systems must adhere to strict privacy and security regulations to protect sensitive patient data. This requires careful design and implementation of data handling procedures within the AI system.\n", + "\n", + "The potential applications extend beyond just document processing to diagnostic assistance, personalized treatment plans, and predictive analytics for patient outcomes. As AI technology continues to evolve, its role in supporting and enhancing the capabilities of healthcare professionals will undoubtedly expand.\n", + "\n", + "Given the importance of the topic and the potential impact on healthcare, it would be beneficial to keep an eye on further developments in this field. If there are no further questions or points to discuss, we can conclude our conversation on this topic.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Mathematician\n", + "\u001b[0m\n", + "\u001b[33mMathematician\u001b[0m (to chat_manager):\n", + "\n", + "I agree with the Programmer's assessment. The implementation of AI in healthcare does indeed require careful consideration of the models used, the training data, and the explainability of the AI's decisions. The ethical implications, particularly concerning patient privacy and data security, are paramount and must be addressed with the utmost rigor.\n", + "\n", + "The potential for AI to assist in diagnostics, treatment planning, and predictive analytics is a promising development for the future of medicine. It is essential that these systems are developed in collaboration with healthcare professionals to ensure they meet the real-world needs of the field.\n", + "\n", + "The interdisciplinary nature of this work, combining expertise in mathematics, computer science, and medicine, is a testament to the collaborative efforts needed to advance healthcare technology. It has been a pleasure discussing the potential applications of explainable AI in medicine with you.\n", + "\n", + "If there are no further points to add, I believe we have reached a natural conclusion to our conversation.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Computer_terminal\n", + "\u001b[0m\n", + "\u001b[33mComputer_terminal\u001b[0m (to chat_manager):\n", + "\n", + "There is no code from the last 1 message for me to execute. Group chat manager should let other participants to continue the conversation. If the group chat manager want to end the conversation, you should let other participant reply me only with \"TERMINATE\"\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mAll agents have been cleared.\u001b[0m\n" + ] + } + ], + "source": [ + "new_builder = AgentBuilder(\n", + " config_file_or_env=config_file_or_env, builder_model=\"gpt-4-1106-preview\", agent_model=\"gpt-4-1106-preview\"\n", + ")\n", + "agent_list, _ = new_builder.build_from_library(building_task, library_path_or_json, llm_config)\n", + "start_task(\n", + " execution_task=\"Find a recent paper about explainable AI on arxiv and find its potential applications in medical.\",\n", + " agent_list=agent_list,\n", + ")\n", + "new_builder.clear_all_agents()" + ] + }, + { + "cell_type": "markdown", + "id": "c7a10e6fa00a5a0d", + "metadata": { + "collapsed": false + }, + "source": [ + "## Build agents from library (by description-task similarity)\n", + "We also support using embedding similarity to select agents. You can use a [Sentence-Transformers model](https://www.sbert.net/docs/pretrained_models.html) as an embedding extractor, and AgentBuilder will select agents with profiles that are the most similar to the building task from the library by comparing their embedding similarity. This will reduce the use of LLMs but may have less accuracy." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "521dc5f961efde59", + "metadata": { + "ExecuteTime": { + "end_time": "2023-12-26T17:01:29.333975100Z", + "start_time": "2023-12-26T16:58:11.070813500Z" + }, + "collapsed": false + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32m==> Looking for suitable agents in the library...\u001b[0m\n" + ] }, - "nbformat": 4, - "nbformat_minor": 5 + { + "name": "stdout", + "output_type": "stream", + "text": [ + "['Programmer', 'Mathematician'] are selected.\n", + "\u001b[32m==> Creating agents...\u001b[0m\n", + "Creating agent Programmer...\n", + "Creating agent Mathematician...\n", + "Adding user console proxy...\n", + "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", + "\n", + "Find a recent paper about gpt-4 on arxiv and find its potential applications in software.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Computer_terminal\n", + "\u001b[0m\n", + "\u001b[33mComputer_terminal\u001b[0m (to chat_manager):\n", + "\n", + "There is no code from the last 1 message for me to execute. Group chat manager should let other participants to continue the conversation. If the group chat manager want to end the conversation, you should let other participant reply me only with \"TERMINATE\"\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Programmer\n", + "\u001b[0m\n", + "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", + "\n", + "To find a recent paper about GPT-4 on arXiv, we can use the arXiv API to search for papers. However, since I can't directly access external APIs, I can write a Python script that you can run on your local machine to perform this search. Would you like me to provide you with such a script?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Computer_terminal\n", + "\u001b[0m\n", + "\u001b[33mComputer_terminal\u001b[0m (to chat_manager):\n", + "\n", + "There is no code from the last 1 message for me to execute. Group chat manager should let other participants to continue the conversation. If the group chat manager want to end the conversation, you should let other participant reply me only with \"TERMINATE\"\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Programmer\n", + "\u001b[0m\n", + "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", + "\n", + "I apologize for the confusion. I will provide a Python script that can be executed by the Computer_terminal to search for recent papers about GPT-4 on arXiv. Let's proceed with that.\n", + "\n", + "```python\n", + "import requests\n", + "from xml.etree import ElementTree\n", + "\n", + "# Define the search parameters and URL for the arXiv API\n", + "search_query = 'all:gpt-4'\n", + "start = 0\n", + "max_results = 5\n", + "sort_by = 'submittedDate'\n", + "sort_order = 'descending'\n", + "url = f'http://export.arxiv.org/api/query?search_query={search_query}&start={start}&max_results={max_results}&sortBy={sort_by}&sortOrder={sort_order}'\n", + "\n", + "# Send a GET request to the arXiv API\n", + "response = requests.get(url)\n", + "\n", + "# Parse the response if it was successful\n", + "if response.status_code == 200:\n", + " root = ElementTree.fromstring(response.content)\n", + " # Find and print the entries (papers)\n", + " for entry in root.findall('{http://www.w3.org/2005/Atom}entry'):\n", + " title = entry.find('{http://www.w3.org/2005/Atom}title').text\n", + " summary = entry.find('{http://www.w3.org/2005/Atom}summary').text\n", + " published = entry.find('{http://www.w3.org/2005/Atom}published').text\n", + " print(f\"Title: {title}\\nSummary: {summary}\\nPublished Date: {published}\\n\")\n", + "else:\n", + " print(f\"Failed to fetch data from arXiv. Status code: {response.status_code}\")\n", + "```\n", + "\n", + "This script will fetch the most recent papers related to GPT-4 from the arXiv API and print out their titles, summaries, and publication dates. Please execute this script to find the information we need.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Computer_terminal\n", + "\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n", + "\u001b[33mComputer_terminal\u001b[0m (to chat_manager):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: \n", + "Title: What If We Recaption Billions of Web Images with LLaMA-3?\n", + "Summary: Web-crawled image-text pairs are inherently noisy. Prior studies demonstrate\n", + "that semantically aligning and enriching textual descriptions of these pairs\n", + "can significantly enhance model training across various vision-language tasks,\n", + "particularly text-to-image generation. However, large-scale investigations in\n", + "this area remain predominantly closed-source. Our paper aims to bridge this\n", + "community effort, leveraging the powerful and \\textit{open-sourced} LLaMA-3, a\n", + "GPT-4 level LLM. Our recaptioning pipeline is simple: first, we fine-tune a\n", + "LLaMA-3-8B powered LLaVA-1.5 and then employ it to recaption 1.3 billion images\n", + "from the DataComp-1B dataset. Our empirical results confirm that this enhanced\n", + "dataset, Recap-DataComp-1B, offers substantial benefits in training advanced\n", + "vision-language models. For discriminative models like CLIP, we observe\n", + "enhanced zero-shot performance in cross-modal retrieval tasks. For generative\n", + "models like text-to-image Diffusion Transformers, the generated images exhibit\n", + "a significant improvement in alignment with users' text instructions,\n", + "especially in following complex queries. Our project page is\n", + "https://www.haqtu.me/Recap-Datacomp-1B/\n", + "\n", + "Published Date: 2024-06-12T17:59:07Z\n", + "\n", + "Title: DafnyBench: A Benchmark for Formal Software Verification\n", + "Summary: We introduce DafnyBench, the largest benchmark of its kind for training and\n", + "evaluating machine learning systems for formal software verification. We test\n", + "the ability of LLMs such as GPT-4 and Claude 3 to auto-generate enough hints\n", + "for the Dafny formal verification engine to successfully verify over 750\n", + "programs with about 53,000 lines of code. The best model and prompting scheme\n", + "achieved 68% success rate, and we quantify how this rate improves when retrying\n", + "with error message feedback and how it deteriorates with the amount of required\n", + "code and hints. We hope that DafnyBench will enable rapid improvements from\n", + "this baseline as LLMs and verification techniques grow in quality.\n", + "\n", + "Published Date: 2024-06-12T17:53:31Z\n", + "\n", + "Title: A Sociotechnical Lens for Evaluating Computer Vision Models: A Case\n", + " Study on Detecting and Reasoning about Gender and Emotion\n", + "Summary: In the evolving landscape of computer vision (CV) technologies, the automatic\n", + "detection and interpretation of gender and emotion in images is a critical area\n", + "of study. This paper investigates social biases in CV models, emphasizing the\n", + "limitations of traditional evaluation metrics such as precision, recall, and\n", + "accuracy. These metrics often fall short in capturing the complexities of\n", + "gender and emotion, which are fluid and culturally nuanced constructs. Our\n", + "study proposes a sociotechnical framework for evaluating CV models,\n", + "incorporating both technical performance measures and considerations of social\n", + "fairness. Using a dataset of 5,570 images related to vaccination and climate\n", + "change, we empirically compared the performance of various CV models, including\n", + "traditional models like DeepFace and FER, and generative models like GPT-4\n", + "Vision. Our analysis involved manually validating the gender and emotional\n", + "expressions in a subset of images to serve as benchmarks. Our findings reveal\n", + "that while GPT-4 Vision outperforms other models in technical accuracy for\n", + "gender classification, it exhibits discriminatory biases, particularly in\n", + "response to transgender and non-binary personas. Furthermore, the model's\n", + "emotion detection skew heavily towards positive emotions, with a notable bias\n", + "towards associating female images with happiness, especially when prompted by\n", + "male personas. These findings underscore the necessity of developing more\n", + "comprehensive evaluation criteria that address both validity and discriminatory\n", + "biases in CV models. Our proposed framework provides guidelines for researchers\n", + "to critically assess CV tools, ensuring their application in communication\n", + "research is both ethical and effective. The significant contribution of this\n", + "study lies in its emphasis on a sociotechnical approach, advocating for CV\n", + "technologies that support social good and mitigate biases rather than\n", + "perpetuate them.\n", + "\n", + "Published Date: 2024-06-12T13:52:30Z\n", + "\n", + "Title: Supportiveness-based Knowledge Rewriting for Retrieval-augmented\n", + " Language Modeling\n", + "Summary: Retrieval-augmented language models (RALMs) have recently shown great\n", + "potential in mitigating the limitations of implicit knowledge in LLMs, such as\n", + "untimely updating of the latest expertise and unreliable retention of long-tail\n", + "knowledge. However, since the external knowledge base, as well as the\n", + "retriever, can not guarantee reliability, potentially leading to the knowledge\n", + "retrieved not being helpful or even misleading for LLM generation. In this\n", + "paper, we introduce Supportiveness-based Knowledge Rewriting (SKR), a robust\n", + "and pluggable knowledge rewriter inherently optimized for LLM generation.\n", + "Specifically, we introduce the novel concept of \"supportiveness\"--which\n", + "represents how effectively a knowledge piece facilitates downstream tasks--by\n", + "considering the perplexity impact of augmented knowledge on the response text\n", + "of a white-box LLM. Based on knowledge supportiveness, we first design a\n", + "training data curation strategy for our rewriter model, effectively identifying\n", + "and filtering out poor or irrelevant rewrites (e.g., with low supportiveness\n", + "scores) to improve data efficacy. We then introduce the direct preference\n", + "optimization (DPO) algorithm to align the generated rewrites to optimal\n", + "supportiveness, guiding the rewriter model to summarize augmented content that\n", + "better improves the final response. Comprehensive evaluations across six\n", + "popular knowledge-intensive tasks and four LLMs have demonstrated the\n", + "effectiveness and superiority of SKR. With only 7B parameters, SKR has shown\n", + "better knowledge rewriting capability over GPT-4, the current state-of-the-art\n", + "general-purpose LLM.\n", + "\n", + "Published Date: 2024-06-12T11:52:35Z\n", + "\n", + "Title: Automated Information Extraction from Thyroid Operation Narrative: A\n", + " Comparative Study of GPT-4 and Fine-tuned KoELECTRA\n", + "Summary: In the rapidly evolving field of healthcare, the integration of artificial\n", + "intelligence (AI) has become a pivotal component in the automation of clinical\n", + "workflows, ushering in a new era of efficiency and accuracy. This study focuses\n", + "on the transformative capabilities of the fine-tuned KoELECTRA model in\n", + "comparison to the GPT-4 model, aiming to facilitate automated information\n", + "extraction from thyroid operation narratives. The current research landscape is\n", + "dominated by traditional methods heavily reliant on regular expressions, which\n", + "often face challenges in processing free-style text formats containing critical\n", + "details of operation records, including frozen biopsy reports. Addressing this,\n", + "the study leverages advanced natural language processing (NLP) techniques to\n", + "foster a paradigm shift towards more sophisticated data processing systems.\n", + "Through this comparative study, we aspire to unveil a more streamlined,\n", + "precise, and efficient approach to document processing in the healthcare\n", + "domain, potentially revolutionizing the way medical data is handled and\n", + "analyzed.\n", + "\n", + "Published Date: 2024-06-12T06:44:05Z\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Programmer\n", + "\u001b[0m\n", + "\u001b[33mProgrammer\u001b[0m (to chat_manager):\n", + "\n", + "Based on the search results from the arXiv API, we have found several papers that discuss potential applications of GPT-4 in software:\n", + "\n", + "1. **Recaptioning Web Images with LLaMA-3 and GPT-4**: This paper discusses the use of GPT-4 level LLMs for recaptioning web images, which can enhance model training across various vision-language tasks. This has implications for improving the quality of datasets used in machine learning and could be particularly beneficial for text-to-image generation and cross-modal retrieval tasks.\n", + "\n", + "2. **DafnyBench: A Benchmark for Formal Software Verification**: This paper introduces a benchmark for training and evaluating machine learning systems for formal software verification. It tests the ability of LLMs such as GPT-4 to auto-generate hints for the Dafny formal verification engine to successfully verify programs. This application could significantly impact the field of software verification by automating the generation of verification hints, potentially improving the efficiency and reliability of the verification process.\n", + "\n", + "3. **Automated Information Extraction from Thyroid Operation Narrative**: This study compares the GPT-4 model with the fine-tuned KoELECTRA model for automated information extraction from thyroid operation narratives. The application of GPT-4 in this context could revolutionize document processing in healthcare by providing a more efficient and accurate method for extracting information from medical records.\n", + "\n", + "These papers suggest that GPT-4 has the potential to be applied in various software-related fields, including enhancing datasets for machine learning, formal software verification, and healthcare document processing. The applications in these papers could lead to more efficient, accurate, and reliable software systems across different domains.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Mathematician\n", + "\u001b[0m\n", + "\u001b[33mMathematician\u001b[0m (to chat_manager):\n", + "\n", + "The applications mentioned indeed highlight the versatility of GPT-4 in different domains. To further understand the potential impact of GPT-4 on software, let's delve into the mathematical and algorithmic principles that could be at play in these applications:\n", + "\n", + "1. **Recaptioning Web Images**: The process of recaptioning images with a language model like GPT-4 involves understanding the context of an image and generating descriptive text that accurately reflects its content. This task likely involves a combination of computer vision techniques to interpret the image and natural language processing to generate the caption. From a mathematical perspective, this would involve optimization algorithms to fine-tune the language model on a specific dataset, ensuring that the generated captions are both semantically and syntactically correct.\n", + "\n", + "2. **Formal Software Verification**: The use of GPT-4 to auto-generate hints for formal verification engines like Dafny involves the model understanding the logic and structure of the code. This requires a deep understanding of formal logic, proof theory, and possibly type theory if the language being verified is statically typed. The success rate of auto-generated hints would depend on the model's ability to reason about the correctness of code and the underlying mathematical properties that ensure its validity.\n", + "\n", + "3. **Automated Information Extraction from Medical Records**: For GPT-4 to extract information from medical narratives, it must process unstructured text and identify relevant medical terms and their relationships. This task involves natural language understanding, which from a mathematical standpoint, can be seen as a form of pattern recognition and classification. The model would need to be trained on a large corpus of medical texts, and its performance would be measured by its precision and recall in identifying and extracting the correct information.\n", + "\n", + "In each of these applications, GPT-4's effectiveness would be influenced by the underlying mathematical models, such as neural networks, and the optimization techniques used during training, such as gradient descent. The quality of the training data and the model's architecture (e.g., attention mechanisms, transformer layers) also play a crucial role in its performance.\n", + "\n", + "To verify the potential of GPT-4 in these applications, one could set up experiments to measure the performance of GPT-4 against specific benchmarks or metrics relevant to each domain. For example, in the case of formal software verification, one could measure the percentage of programs that are successfully verified with the hints generated by GPT-4 compared to a baseline or human-generated hints.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Mathematician\n", + "\u001b[0m\n", + "\u001b[33mMathematician\u001b[0m (to chat_manager):\n", + "\n", + "To further verify the potential applications of GPT-4 in software, we can consider the following:\n", + "\n", + "1. **Recaptioning Web Images**: The effectiveness of GPT-4 in this application can be measured by the accuracy of the captions it generates. This can be quantified using metrics such as BLEU (Bilingual Evaluation Understudy) or ROUGE (Recall-Oriented Understudy for Gisting Evaluation), which compare the machine-generated text to a set of reference captions. A high score on these metrics would indicate that GPT-4 is capable of understanding and describing the content of images accurately.\n", + "\n", + "2. **Formal Software Verification**: To verify the application of GPT-4 in software verification, we could measure the success rate of the model in generating verification hints that lead to successful program verification. This could be done by comparing the verification success rate with and without the hints provided by GPT-4. Additionally, the complexity of the programs and the hints required could be analyzed to understand the model's limitations and strengths.\n", + "\n", + "3. **Automated Information Extraction from Medical Records**: The performance of GPT-4 in extracting information from medical narratives can be assessed by comparing the extracted information to a gold standard set of annotations. Precision, recall, and F1-score are common metrics used to evaluate information extraction systems. A high F1-score would suggest that GPT-4 is effective in identifying and extracting relevant information from unstructured medical texts.\n", + "\n", + "For each of these applications, it would be important to conduct a thorough analysis of the results to ensure that the model's performance is not only statistically significant but also practically relevant. Additionally, it would be crucial to assess the model's performance on diverse datasets to ensure that it generalizes well across different contexts and does not exhibit biases that could lead to incorrect or unfair outcomes.\n", + "\n", + "If we want to simulate or test these applications, we could propose specific experiments or data analysis procedures that align with the above verification strategies. However, such experiments would require access to the relevant datasets and computational resources to run the models, which may not be feasible within the scope of this conversation.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Mathematician\n", + "\u001b[0m\n", + "\u001b[33mMathematician\u001b[0m (to chat_manager):\n", + "\n", + "To simulate or test these applications, we could propose specific experiments or data analysis procedures that align with the above verification strategies. However, such experiments would require access to the relevant datasets and computational resources to run the models, which may not be feasible within the scope of this conversation.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Mathematician\n", + "\u001b[0m\n", + "\u001b[33mMathematician\u001b[0m (to chat_manager):\n", + "\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mAll agents have been cleared.\u001b[0m\n" + ] + } + ], + "source": [ + "new_builder = AgentBuilder(\n", + " config_file_or_env=config_file_or_env, builder_model=\"gpt-4-1106-preview\", agent_model=\"gpt-4-1106-preview\"\n", + ")\n", + "agent_list, _ = new_builder.build_from_library(\n", + " building_task, library_path_or_json, llm_config, embedding_model=\"all-mpnet-base-v2\"\n", + ")\n", + "start_task(\n", + " execution_task=\"Find a recent paper about gpt-4 on arxiv and find its potential applications in software.\",\n", + " agent_list=agent_list,\n", + ")\n", + "new_builder.clear_all_agents()" + ] + } + ], + "metadata": { + "front_matter": { + "description": "Automatically build multi-agent system from agent library", + "tags": [ + "autobuild" + ] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/notebook/tools_interoperability.ipynb b/notebook/tools_interoperability.ipynb new file mode 100644 index 0000000000..00469be0d4 --- /dev/null +++ b/notebook/tools_interoperability.ipynb @@ -0,0 +1,418 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cross-Framework LLM Tool Integration with AG2\n", + "\n", + "In this tutorial, we demonstrate how to integrate LLM tools from various frameworks—including [LangChain Tools](https://python.langchain.com/v0.1/docs/modules/tools), [CrewAI Tools](https://github.com/crewAIInc/crewAI-tools/tree/main), and [PydanticAI Tools](https://ai.pydantic.dev/tools/) into the AG2 framework. This process enables smooth interoperability between these systems, allowing developers to leverage the unique capabilities of each toolset within AG2's flexible agent-based architecture. By the end of this guide, you will understand how to configure agents, adapt these tools for use in AG2, and validate the integration through practical examples." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## LangChain Tools Integration\n", + "\n", + "LangChain is a popular framework that offers a wide range of tools to work with LLMs. LangChain has already implemented a variety of tools that can be easily integrated into AG2. You can explore the available tools in the [LangChain Community Tools](https://github.com/langchain-ai/langchain/tree/master/libs/community/langchain_community/tools) folder. These tools, such as those for querying APIs, web scraping, and text generation, can be quickly incorporated into AG2, providing powerful functionality for your agents.\n", + "\n", + "### Installation\n", + "To integrate LangChain tools into the AG2 framework, install the required dependencies:\n", + "\n", + "```bash\n", + "pip install ag2[interop-langchain]\n", + "```\n", + "\n", + "Additionally, this notebook uses LangChain's [Wikipedia Tool](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html), which requires the `wikipedia` package. Install it with:\n", + "\n", + "```bash\n", + "pip install wikipedia\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Imports\n", + "\n", + "Import necessary modules and tools.\n", + "- `WikipediaQueryRun` and `WikipediaAPIWrapper`: Tools for querying Wikipedia.\n", + "- `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework.\n", + "- `Interoperability`: This module acts as a bridge, making it easier to integrate LangChain tools with AG2’s architecture." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from langchain_community.tools import WikipediaQueryRun\n", + "from langchain_community.utilities import WikipediaAPIWrapper\n", + "\n", + "from autogen import AssistantAgent, UserProxyAgent\n", + "from autogen.interop import Interoperability" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Agent Configuration\n", + "\n", + "Configure the agents for the interaction.\n", + "- `config_list` defines the LLM configurations, including the model and API key.\n", + "- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`).\n", + "- `AssistantAgent` represents the AI agent, configured with the LLM settings." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "config_list = [{\"model\": \"gpt-4o\", \"api_key\": os.environ[\"OPENAI_API_KEY\"]}]\n", + "user_proxy = UserProxyAgent(\n", + " name=\"User\",\n", + " human_input_mode=\"NEVER\",\n", + ")\n", + "\n", + "chatbot = AssistantAgent(\n", + " name=\"chatbot\",\n", + " llm_config={\"config_list\": config_list},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Tool Integration\n", + "\n", + "- Initialize and register the LangChain tool with AG2.\n", + "- `WikipediaAPIWrapper`: Configured to fetch the top 1 result from Wikipedia with a maximum of 1000 characters per document.\n", + "- `WikipediaQueryRun`: A LangChain tool that executes Wikipedia queries.\n", + "- `LangchainInteroperability`: Converts the LangChain tool into a format compatible with the AG2 framework.\n", + "- `ag2_tool.register_for_execution(user_proxy)`: Registers the tool for use by the user_proxy agent.\n", + "- `ag2_tool.register_for_llm(chatbot)`: Registers the tool for integration with the chatbot agent.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "api_wrapper = WikipediaAPIWrapper(top_k_results=1, doc_content_chars_max=1000)\n", + "langchain_tool = WikipediaQueryRun(api_wrapper=api_wrapper)\n", + "\n", + "interop = Interoperability()\n", + "ag2_tool = interop.convert_tool(tool=langchain_tool, type=\"langchain\")\n", + "\n", + "ag2_tool.register_for_execution(user_proxy)\n", + "ag2_tool.register_for_llm(chatbot)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "message = \"Tell me about the history of the United States\"\n", + "user_proxy.initiate_chat(recipient=chatbot, message=message, max_turns=2)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## CrewAI Tools Integration\n", + "\n", + "CrewAI provides a variety of powerful tools designed for tasks such as web scraping, search, code interpretation, and more. These tools are easy to integrate into the AG2 framework, allowing you to enhance your agents with advanced capabilities. You can explore the full list of available tools in the [CrewAI Tools](https://github.com/crewAIInc/crewAI-tools/tree/main) repository.\n", + "\n", + "### Installation\n", + "Install the required packages for integrating CrewAI tools into the AG2 framework.\n", + "This ensures all dependencies for both frameworks are installed.\n", + "\n", + "```bash\n", + "pip install ag2[interop-crewai]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Imports\n", + "\n", + "Import necessary modules and tools.\n", + "- `ScrapeWebsiteTool` are the CrewAI tools for web scraping\n", + "- `AssistantAgent` and `UserProxyAgent` are core AG2 classes.\n", + "- `Interoperability`: This module acts as a bridge, making it easier to integrate CrewAI tools with AG2’s architecture." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from crewai_tools import ScrapeWebsiteTool\n", + "\n", + "from autogen import AssistantAgent, UserProxyAgent\n", + "from autogen.interop import Interoperability" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Agent Configuration\n", + "\n", + "Configure the agents for the interaction.\n", + "- `config_list` defines the LLM configurations, including the model and API key.\n", + "- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`).\n", + "- `AssistantAgent` represents the AI agent, configured with the LLM settings." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "config_list = [{\"model\": \"gpt-4o\", \"api_key\": os.environ[\"OPENAI_API_KEY\"]}]\n", + "user_proxy = UserProxyAgent(\n", + " name=\"User\",\n", + " human_input_mode=\"NEVER\",\n", + ")\n", + "\n", + "chatbot = AssistantAgent(\n", + " name=\"chatbot\",\n", + " llm_config={\"config_list\": config_list},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Tool Integration\n", + "\n", + "Initialize and register the CrewAI tool with AG2.\n", + "- `crewai_tool` is an instance of the `ScrapeWebsiteTool` from CrewAI.\n", + "- `Interoperability` converts the CrewAI tool to make it usable in AG2.\n", + "- `register_for_execution` and `register_for_llm` allow the tool to work with the UserProxyAgent and AssistantAgent." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "interop = Interoperability()\n", + "crewai_tool = ScrapeWebsiteTool()\n", + "ag2_tool = interop.convert_tool(tool=crewai_tool, type=\"crewai\")\n", + "\n", + "ag2_tool.register_for_execution(user_proxy)\n", + "ag2_tool.register_for_llm(chatbot)\n", + "\n", + "message = \"Scrape the website https://ag2.ai/\"\n", + "\n", + "chat_result = user_proxy.initiate_chat(recipient=chatbot, message=message, max_turns=2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "print(chat_result.summary)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## PydanticAI Tools Integration\n", + "\n", + "[PydanticAI](https://ai.pydantic.dev/) is a newer framework that offers powerful capabilities for working with LLMs. Although it currently does not have a repository with pre-built tools, it provides features like **dependency injection**, allowing you to inject a \"Context\" into a tool for better execution without relying on LLMs. This context can be used for passing parameters or managing state during the execution of a tool. While the framework is still growing, you can integrate its tools into AG2 to enhance agent capabilities, especially for tasks that involve structured data and context-driven logic.\n", + "\n", + "### Installation\n", + "To integrate LangChain tools into the AG2 framework, install the required dependencies:\n", + "\n", + "```bash\n", + "pip install ag2[interop-pydantic-ai]\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Imports\n", + "\n", + "Import necessary modules and tools.\n", + "- `BaseModel`: Used to define data structures for tool inputs and outputs.\n", + "- `RunContext`: Provides context during the execution of tools.\n", + "- `PydanticAITool`: Represents a tool in the PydanticAI framework.\n", + "- `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework.\n", + "- `Interoperability`: This module acts as a bridge, making it easier to integrate PydanticAI tools with AG2’s architecture." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from typing import Optional\n", + "\n", + "from pydantic import BaseModel\n", + "from pydantic_ai import RunContext\n", + "from pydantic_ai.tools import Tool as PydanticAITool\n", + "\n", + "from autogen import AssistantAgent, UserProxyAgent\n", + "from autogen.interop import Interoperability" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Agent Configuration\n", + "\n", + "Configure the agents for the interaction.\n", + "- `config_list` defines the LLM configurations, including the model and API key.\n", + "- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`).\n", + "- `AssistantAgent` represents the AI agent, configured with the LLM settings." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "config_list = [{\"model\": \"gpt-4o\", \"api_key\": os.environ[\"OPENAI_API_KEY\"]}]\n", + "user_proxy = UserProxyAgent(\n", + " name=\"User\",\n", + " human_input_mode=\"NEVER\",\n", + ")\n", + "\n", + "chatbot = AssistantAgent(\n", + " name=\"chatbot\",\n", + " llm_config={\"config_list\": config_list},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Tool Integration\n", + "\n", + "Integrate the PydanticAI tool with AG2.\n", + "\n", + "- Define a `Player` model using `BaseModel` to structure the input data.\n", + "- Use `RunContext` to securely inject dependencies (like the `Player` instance) into the tool function without exposing them to the LLM.\n", + "- Implement `get_player` to define the tool's functionality, accessing `ctx.deps` for injected data.\n", + "- Convert the tool to an AG2-compatible format with `Interoperability` and register it for execution and LLM communication.\n", + "- Convert the PydanticAI tool into an AG2-compatible format using `convert_tool`.\n", + "- Register the tool for both execution and communication with the LLM by associating it with the `user_proxy` and `chatbot`." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [], + "source": [ + "class Player(BaseModel):\n", + " name: str\n", + " age: int\n", + "\n", + "\n", + "def get_player(ctx: RunContext[Player], additional_info: Optional[str] = None) -> str: # type: ignore[valid-type]\n", + " \"\"\"Get the player's name.\n", + "\n", + " Args:\n", + " additional_info: Additional information which can be used.\n", + " \"\"\"\n", + " return f\"Name: {ctx.deps.name}, Age: {ctx.deps.age}, Additional info: {additional_info}\" # type: ignore[attr-defined]\n", + "\n", + "\n", + "interop = Interoperability()\n", + "pydantic_ai_tool = PydanticAITool(get_player, takes_ctx=True)\n", + "\n", + "# player will be injected as a dependency\n", + "player = Player(name=\"Luka\", age=25)\n", + "ag2_tool = interop.convert_tool(tool=pydantic_ai_tool, type=\"pydanticai\", deps=player)\n", + "\n", + "ag2_tool.register_for_execution(user_proxy)\n", + "ag2_tool.register_for_llm(chatbot)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Initiate a conversation between the `UserProxyAgent` and the `AssistantAgent`.\n", + "\n", + "- Use the `initiate_chat` method to send a message from the `user_proxy` to the `chatbot`.\n", + "- In this example, the user requests the chatbot to retrieve player information, providing \"goal keeper\" as additional context.\n", + "- The `Player` instance is securely injected into the tool using `RunContext`, ensuring the chatbot can retrieve and use this data during the interaction." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "user_proxy.initiate_chat(\n", + " recipient=chatbot, message=\"Get player, for additional information use 'goal keeper'\", max_turns=3\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.16" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/pyproject.toml b/pyproject.toml index c7d7f08b75..8f1db523ec 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -60,9 +60,13 @@ files = [ "autogen/_pydantic.py", "autogen/function_utils.py", "autogen/io", + "autogen/tools", + "autogen/interop", "test/test_pydantic.py", "test/test_function_utils.py", "test/io", + "test/tools", + "test/interop", ] exclude = [ "autogen/math_utils\\.py", @@ -87,7 +91,8 @@ no_implicit_optional = true check_untyped_defs = true warn_return_any = true show_error_codes = true -warn_unused_ignores = true + +warn_unused_ignores = false disallow_incomplete_defs = true disallow_untyped_decorators = true diff --git a/setup.py b/setup.py index f9d3dd4d8d..04a89a00e0 100644 --- a/setup.py +++ b/setup.py @@ -59,8 +59,6 @@ "ipykernel>=6.29.0", ] -types = ["mypy==1.9.0"] + test + jupyter_executor - retrieve_chat = [ "protobuf==4.25.3", "chromadb==0.5.3", @@ -82,6 +80,13 @@ "llama-index-core==0.12.5", ] +interop_crewai = ["crewai[tools]>=0.86,<1; python_version>='3.10' and python_version<'3.13'"] +interop_langchain = ["langchain-community>=0.3.12,<1"] +interop_pydantic_ai = ["pydantic-ai==0.0.13"] +interop = interop_crewai + interop_langchain + interop_pydantic_ai + +types = ["mypy==1.9.0"] + test + jupyter_executor + interop + if current_os in ["Windows", "Darwin"]: retrieve_chat_pgvector.extend(["psycopg[binary]>=3.1.18"]) elif current_os == "Linux": @@ -126,6 +131,10 @@ "cohere": ["cohere>=5.5.8"], "ollama": ["ollama>=0.3.3", "fix_busted_json>=0.0.18"], "bedrock": ["boto3>=1.34.149"], + "interop-crewai": interop_crewai, + "interop-langchain": interop_langchain, + "interop-pydantic-ai": interop_pydantic_ai, + "interop": interop, "neo4j": neo4j, } diff --git a/setup_ag2.py b/setup_ag2.py index eebda43f2e..da6e6aa79b 100644 --- a/setup_ag2.py +++ b/setup_ag2.py @@ -54,6 +54,10 @@ "cohere": ["pyautogen[cohere]==" + __version__], "ollama": ["pyautogen[ollama]==" + __version__], "bedrock": ["pyautogen[bedrock]==" + __version__], + "interop-crewai": ["pyautogen[interop-crewai]==" + __version__], + "interop-langchain": ["pyautogen[interop-langchain]==" + __version__], + "interop-pydantic-ai": ["pyautogen[interop-pydantic-ai]==" + __version__], + "interop": ["pyautogen[interop]==" + __version__], "neo4j": ["pyautogen[neo4j]==" + __version__], }, url="https://github.com/ag2ai/ag2", diff --git a/setup_autogen.py b/setup_autogen.py index 369e82e204..5b56a7c12a 100644 --- a/setup_autogen.py +++ b/setup_autogen.py @@ -54,6 +54,10 @@ "cohere": ["pyautogen[cohere]==" + __version__], "ollama": ["pyautogen[ollama]==" + __version__], "bedrock": ["pyautogen[bedrock]==" + __version__], + "interop-crewai": ["pyautogen[interop-crewai]==" + __version__], + "interop-langchain": ["pyautogen[interop-langchain]==" + __version__], + "interop-pydantic-ai": ["pyautogen[interop-pydantic-ai]==" + __version__], + "interop": ["pyautogen[interop]==" + __version__], "neo4j": ["pyautogen[neo4j]==" + __version__], }, url="https://github.com/ag2ai/ag2", diff --git a/test/interop/__init__.py b/test/interop/__init__.py new file mode 100644 index 0000000000..bcd5401d54 --- /dev/null +++ b/test/interop/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/test/interop/crewai/__init__.py b/test/interop/crewai/__init__.py new file mode 100644 index 0000000000..bcd5401d54 --- /dev/null +++ b/test/interop/crewai/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/test/interop/crewai/test_crewai.py b/test/interop/crewai/test_crewai.py new file mode 100644 index 0000000000..0d39926638 --- /dev/null +++ b/test/interop/crewai/test_crewai.py @@ -0,0 +1,106 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +from tempfile import TemporaryDirectory +from unittest.mock import MagicMock + +import pytest +from conftest import reason, skip_openai + +if sys.version_info >= (3, 10) and sys.version_info < (3, 13): + from crewai_tools import FileReadTool +else: + FileReadTool = MagicMock() + +from autogen import AssistantAgent, UserProxyAgent +from autogen.interop import Interoperable + +if sys.version_info >= (3, 10) and sys.version_info < (3, 13): + from autogen.interop.crewai import CrewAIInteroperability +else: + CrewAIInteroperability = MagicMock() + + +# skip if python version is not in [3.10, 3.11, 3.12] +@pytest.mark.skipif( + sys.version_info < (3, 10) or sys.version_info >= (3, 13), reason="Only Python 3.10, 3.11, 3.12 are supported" +) +class TestCrewAIInteroperability: + @pytest.fixture(autouse=True) + def setup(self) -> None: + + crewai_tool = FileReadTool() + self.model_type = crewai_tool.args_schema + self.tool = CrewAIInteroperability.convert_tool(crewai_tool) + + def test_type_checks(self) -> None: + # mypy should fail if the type checks are not correct + interop: Interoperable = CrewAIInteroperability() + + # runtime check + assert isinstance(interop, Interoperable) + + def test_convert_tool(self) -> None: + with TemporaryDirectory() as tmp_dir: + file_path = f"{tmp_dir}/test.txt" + with open(file_path, "w") as file: + file.write("Hello, World!") + + assert self.tool.name == "Read_a_file_s_content" + assert ( + self.tool.description + == "A tool that can be used to read a file's content. (IMPORTANT: When using arguments, put them all in an `args` dictionary)" + ) + + args = self.model_type(file_path=file_path) + + assert self.tool.func(args=args) == "Hello, World!" + + @pytest.mark.skipif(skip_openai, reason=reason) + def test_with_llm(self) -> None: + config_list = [{"model": "gpt-4o", "api_key": os.environ["OPENAI_API_KEY"]}] + user_proxy = UserProxyAgent( + name="User", + human_input_mode="NEVER", + ) + + chatbot = AssistantAgent( + name="chatbot", + llm_config={"config_list": config_list}, + ) + + self.tool.register_for_execution(user_proxy) + self.tool.register_for_llm(chatbot) + + with TemporaryDirectory() as tmp_dir: + file_path = f"{tmp_dir}/test.txt" + with open(file_path, "w") as file: + file.write("Hello, World!") + + user_proxy.initiate_chat( + recipient=chatbot, message=f"Read the content of the file at {file_path}", max_turns=2 + ) + + for message in user_proxy.chat_messages[chatbot]: + if "tool_responses" in message: + assert message["tool_responses"][0]["content"] == "Hello, World!" + return + + assert False, "Tool response not found in chat messages" + + def test_get_unsupported_reason(self) -> None: + assert CrewAIInteroperability.get_unsupported_reason() is None + + +@pytest.mark.skipif( + sys.version_info >= (3, 10) or sys.version_info < (3, 13), reason="Crew AI Interoperability is supported" +) +class TestCrewAIInteroperabilityIfNotSupported: + def test_get_unsupported_reason(self) -> None: + assert ( + CrewAIInteroperability.get_unsupported_reason() + == "This submodule is only supported for Python versions 3.10, 3.11, and 3.12" + ) diff --git a/test/interop/langchain/__init__.py b/test/interop/langchain/__init__.py new file mode 100644 index 0000000000..bcd5401d54 --- /dev/null +++ b/test/interop/langchain/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/test/interop/langchain/test_langchain.py b/test/interop/langchain/test_langchain.py new file mode 100644 index 0000000000..be0a2f6bfc --- /dev/null +++ b/test/interop/langchain/test_langchain.py @@ -0,0 +1,139 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import sys +import unittest + +import pytest +from conftest import reason, skip_openai +from pydantic import BaseModel, Field + +from autogen import AssistantAgent, UserProxyAgent +from autogen.interop import Interoperable + +if sys.version_info >= (3, 9): + from langchain.tools import tool as langchain_tool +else: + langchain_tool = unittest.mock.MagicMock() + +from autogen.interop.langchain import LangChainInteroperability + + +# skip if python version is not >= 3.9 +@pytest.mark.skipif( + sys.version_info < (3, 9), reason="Only Python 3.9 and above are supported for LangchainInteroperability" +) +class TestLangChainInteroperability: + @pytest.fixture(autouse=True) + def setup(self) -> None: + class SearchInput(BaseModel): + query: str = Field(description="should be a search query") + + @langchain_tool("search-tool", args_schema=SearchInput, return_direct=True) # type: ignore[misc] + def search(query: SearchInput) -> str: + """Look up things online.""" + return "LangChain Integration" + + self.model_type = search.args_schema + self.tool = LangChainInteroperability.convert_tool(search) + + def test_type_checks(self) -> None: + # mypy should fail if the type checks are not correct + interop: Interoperable = LangChainInteroperability() + + # runtime check + assert isinstance(interop, Interoperable) + + def test_convert_tool(self) -> None: + assert self.tool.name == "search-tool" + assert self.tool.description == "Look up things online." + + tool_input = self.model_type(query="LangChain") # type: ignore[misc] + assert self.tool.func(tool_input=tool_input) == "LangChain Integration" + + @pytest.mark.skipif(skip_openai, reason=reason) + def test_with_llm(self) -> None: + config_list = [{"model": "gpt-4o", "api_key": os.environ["OPENAI_API_KEY"]}] + user_proxy = UserProxyAgent( + name="User", + human_input_mode="NEVER", + ) + + chatbot = AssistantAgent( + name="chatbot", + llm_config={"config_list": config_list}, + ) + + self.tool.register_for_execution(user_proxy) + self.tool.register_for_llm(chatbot) + + user_proxy.initiate_chat(recipient=chatbot, message="search for LangChain", max_turns=2) + + for message in user_proxy.chat_messages[chatbot]: + if "tool_responses" in message: + assert message["tool_responses"][0]["content"] == "LangChain Integration" + return + + assert False, "No tool response found in chat messages" + + def test_get_unsupported_reason(self) -> None: + assert LangChainInteroperability.get_unsupported_reason() is None + + +# skip if python version is not >= 3.9 +@pytest.mark.skipif( + sys.version_info < (3, 9), reason="Only Python 3.9 and above are supported for LangchainInteroperability" +) +class TestLangChainInteroperabilityWithoutPydanticInput: + @pytest.fixture(autouse=True) + def setup(self) -> None: + @langchain_tool + def search(query: str, max_length: int) -> str: + """Look up things online.""" + return f"LangChain Integration, max_length: {max_length}" + + self.tool = LangChainInteroperability.convert_tool(search) + self.model_type = search.args_schema + + def test_convert_tool(self) -> None: + assert self.tool.name == "search" + assert self.tool.description == "Look up things online." + + tool_input = self.model_type(query="LangChain", max_length=100) # type: ignore[misc] + assert self.tool.func(tool_input=tool_input) == "LangChain Integration, max_length: 100" + + @pytest.mark.skipif(skip_openai, reason=reason) + def test_with_llm(self) -> None: + config_list = [{"model": "gpt-4o", "api_key": os.environ["OPENAI_API_KEY"]}] + user_proxy = UserProxyAgent( + name="User", + human_input_mode="NEVER", + ) + + chatbot = AssistantAgent( + name="chatbot", + llm_config={"config_list": config_list}, + ) + + self.tool.register_for_execution(user_proxy) + self.tool.register_for_llm(chatbot) + + user_proxy.initiate_chat(recipient=chatbot, message="search for LangChain, Use max 100 characters", max_turns=2) + + for message in user_proxy.chat_messages[chatbot]: + if "tool_responses" in message: + assert message["tool_responses"][0]["content"] == "LangChain Integration, max_length: 100" + return + + assert False, "No tool response found in chat messages" + + +@pytest.mark.skipif(sys.version_info >= (3, 9), reason="LangChain Interoperability is supported") +class TestLangChainInteroperabilityIfNotSupported: + def test_get_unsupported_reason(self) -> None: + assert ( + LangChainInteroperability.get_unsupported_reason() + == "This submodule is only supported for Python versions 3.9 and above" + ) diff --git a/test/interop/pydantic_ai/__init__.py b/test/interop/pydantic_ai/__init__.py new file mode 100644 index 0000000000..bcd5401d54 --- /dev/null +++ b/test/interop/pydantic_ai/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/test/interop/pydantic_ai/test_pydantic_ai.py b/test/interop/pydantic_ai/test_pydantic_ai.py new file mode 100644 index 0000000000..2840cdbc9a --- /dev/null +++ b/test/interop/pydantic_ai/test_pydantic_ai.py @@ -0,0 +1,235 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +import os +import random +import sys +import unittest +from inspect import signature +from typing import Any, Dict, Optional + +import pytest +from conftest import reason, skip_openai +from pydantic import BaseModel + +from autogen import AssistantAgent, UserProxyAgent +from autogen.interop import Interoperable + +if sys.version_info >= (3, 9): + from pydantic_ai import RunContext + from pydantic_ai.tools import Tool as PydanticAITool + +else: + RunContext = unittest.mock.MagicMock() + PydanticAITool = unittest.mock.MagicMock() + +from autogen.interop.pydantic_ai import PydanticAIInteroperability + + +# skip if python version is not >= 3.9 +@pytest.mark.skipif( + sys.version_info < (3, 9), reason="Only Python 3.9 and above are supported for LangchainInteroperability" +) +class TestPydanticAIInteroperabilityWithotContext: + @pytest.fixture(autouse=True) + def setup(self) -> None: + def roll_dice() -> str: + """Roll a six-sided dice and return the result.""" + return str(random.randint(1, 6)) + + pydantic_ai_tool = PydanticAITool(roll_dice, max_retries=3) # type: ignore[var-annotated] + self.tool = PydanticAIInteroperability.convert_tool(pydantic_ai_tool) + + def test_type_checks(self) -> None: + # mypy should fail if the type checks are not correct + interop: Interoperable = PydanticAIInteroperability() + # runtime check + assert isinstance(interop, Interoperable) + + def test_convert_tool(self) -> None: + assert self.tool.name == "roll_dice" + assert self.tool.description == "Roll a six-sided dice and return the result." + assert self.tool.func() in ["1", "2", "3", "4", "5", "6"] + + @pytest.mark.skipif(skip_openai, reason=reason) + def test_with_llm(self) -> None: + config_list = [{"model": "gpt-4o", "api_key": os.environ["OPENAI_API_KEY"]}] + user_proxy = UserProxyAgent( + name="User", + human_input_mode="NEVER", + ) + + chatbot = AssistantAgent( + name="chatbot", + llm_config={"config_list": config_list}, + ) + + self.tool.register_for_execution(user_proxy) + self.tool.register_for_llm(chatbot) + + user_proxy.initiate_chat(recipient=chatbot, message="roll a dice", max_turns=2) + + for message in user_proxy.chat_messages[chatbot]: + if "tool_responses" in message: + assert message["tool_responses"][0]["content"] in ["1", "2", "3", "4", "5", "6"] + return + + assert False, "No tool response found in chat messages" + + +@pytest.mark.skipif( + sys.version_info < (3, 9), reason="Only Python 3.9 and above are supported for LangchainInteroperability" +) +class TestPydanticAIInteroperabilityDependencyInjection: + + def test_dependency_injection(self) -> None: + def f( + ctx: RunContext[int], # type: ignore[valid-type] + city: str, + date: str, + ) -> str: + """Random function for testing.""" + return f"{city} {date} {ctx.deps}" # type: ignore[attr-defined] + + ctx = RunContext( + deps=123, + retry=0, + messages=None, # type: ignore[arg-type] + tool_name=f.__name__, + ) + pydantic_ai_tool = PydanticAITool(f, takes_ctx=True) # type: ignore[var-annotated] + g = PydanticAIInteroperability.inject_params( + ctx=ctx, + tool=pydantic_ai_tool, + ) + assert list(signature(g).parameters.keys()) == ["city", "date"] + kwargs: Dict[str, Any] = {"city": "Zagreb", "date": "2021-01-01"} + assert g(**kwargs) == "Zagreb 2021-01-01 123" + + def test_dependency_injection_with_retry(self) -> None: + def f( + ctx: RunContext[int], # type: ignore[valid-type] + city: str, + date: str, + ) -> str: + """Random function for testing.""" + raise ValueError("Retry") + + ctx = RunContext( + deps=123, + retry=0, + messages=None, # type: ignore[arg-type] + tool_name=f.__name__, + ) + + pydantic_ai_tool = PydanticAITool(f, takes_ctx=True, max_retries=3) # type: ignore[var-annotated] + g = PydanticAIInteroperability.inject_params( + ctx=ctx, + tool=pydantic_ai_tool, + ) + + for i in range(3): + with pytest.raises(ValueError, match="Retry"): + g(city="Zagreb", date="2021-01-01") + assert pydantic_ai_tool.current_retry == i + 1 + assert ctx.retry == i + + with pytest.raises(ValueError, match="f failed after 3 retries"): + g(city="Zagreb", date="2021-01-01") + assert pydantic_ai_tool.current_retry == 3 + + +@pytest.mark.skipif( + sys.version_info < (3, 9), reason="Only Python 3.9 and above are supported for LangchainInteroperability" +) +class TestPydanticAIInteroperabilityWithContext: + @pytest.fixture(autouse=True) + def setup(self) -> None: + class Player(BaseModel): + name: str + age: int + + def get_player(ctx: RunContext[Player], additional_info: Optional[str] = None) -> str: # type: ignore[valid-type] + """Get the player's name. + + Args: + additional_info: Additional information which can be used. + """ + return f"Name: {ctx.deps.name}, Age: {ctx.deps.age}, Additional info: {additional_info}" # type: ignore[attr-defined] + + self.pydantic_ai_tool = PydanticAITool(get_player, takes_ctx=True) # type: ignore[var-annotated] + player = Player(name="Luka", age=25) + self.tool = PydanticAIInteroperability.convert_tool(tool=self.pydantic_ai_tool, deps=player) + + def test_convert_tool_raises_error_if_take_ctx_is_true_and_deps_is_none(self) -> None: + with pytest.raises(ValueError, match="If the tool takes a context, the `deps` argument must be provided"): + PydanticAIInteroperability.convert_tool(tool=self.pydantic_ai_tool, deps=None) + + def test_expected_tools(self) -> None: + config_list = [{"model": "gpt-4o", "api_key": "abc"}] + chatbot = AssistantAgent( + name="chatbot", + llm_config={"config_list": config_list}, + ) + self.tool.register_for_llm(chatbot) + + expected_tools = [ + { + "type": "function", + "function": { + "name": "get_player", + "description": "Get the player's name.", + "parameters": { + "properties": { + "additional_info": { + "anyOf": [{"type": "string"}, {"type": "null"}], + "description": "Additional information which can be used.", + "title": "Additional Info", + } + }, + "required": ["additional_info"], + "type": "object", + "additionalProperties": False, + }, + }, + } + ] + + assert chatbot.llm_config["tools"] == expected_tools # type: ignore[index] + + @pytest.mark.skipif(skip_openai, reason=reason) + def test_with_llm(self) -> None: + config_list = [{"model": "gpt-4o", "api_key": os.environ["OPENAI_API_KEY"]}] + user_proxy = UserProxyAgent( + name="User", + human_input_mode="NEVER", + ) + + chatbot = AssistantAgent( + name="chatbot", + llm_config={"config_list": config_list}, + ) + + self.tool.register_for_execution(user_proxy) + self.tool.register_for_llm(chatbot) + + user_proxy.initiate_chat( + recipient=chatbot, message="Get player, for additional information use 'goal keeper'", max_turns=3 + ) + + for message in user_proxy.chat_messages[chatbot]: + if "tool_responses" in message: + assert message["tool_responses"][0]["content"] == "Name: Luka, Age: 25, Additional info: goal keeper" + return + + assert False, "No tool response found in chat messages" + + +@pytest.mark.skipif(sys.version_info >= (3, 9), reason="LangChain Interoperability is supported") +class TestPydanticAIInteroperabilityIfNotSupported: + def test_get_unsupported_reason(self) -> None: + assert ( + PydanticAIInteroperability.get_unsupported_reason() + == "This submodule is only supported for Python versions 3.9 and above" + ) diff --git a/test/interop/pydantic_ai/test_pydantic_ai_tool.py b/test/interop/pydantic_ai/test_pydantic_ai_tool.py new file mode 100644 index 0000000000..f1ae38389e --- /dev/null +++ b/test/interop/pydantic_ai/test_pydantic_ai_tool.py @@ -0,0 +1,74 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +import sys +import unittest + +import pytest + +from autogen import AssistantAgent + +if sys.version_info >= (3, 9): + from pydantic_ai.tools import Tool as PydanticAITool + +else: + PydanticAITool = unittest.mock.MagicMock() + +from autogen.interop.pydantic_ai.pydantic_ai_tool import PydanticAITool as AG2PydanticAITool + + +# skip if python version is not >= 3.9 +@pytest.mark.skipif( + sys.version_info < (3, 9), reason="Only Python 3.9 and above are supported for LangchainInteroperability" +) +class TestPydanticAITool: + def test_register_for_llm(self) -> None: + def foobar(a: int, b: str, c: dict[str, list[float]]) -> str: # type: ignore[misc] + """Get me foobar. + + Args: + a: apple pie + b: banana cake + c: carrot smoothie + """ + return f"{a} {b} {c}" + + tool = PydanticAITool(foobar) # type: ignore[var-annotated] + ag2_tool = AG2PydanticAITool( + name=tool.name, + description=tool.description, + func=tool.function, + parameters_json_schema=tool._parameters_json_schema, + ) + config_list = [{"model": "gpt-4o", "api_key": "abc"}] + chatbot = AssistantAgent( + name="chatbot", + llm_config={"config_list": config_list}, + ) + ag2_tool.register_for_llm(chatbot) + expected_tools = [ + { + "type": "function", + "function": { + "name": "foobar", + "description": "Get me foobar.", + "parameters": { + "properties": { + "a": {"description": "apple pie", "title": "A", "type": "integer"}, + "b": {"description": "banana cake", "title": "B", "type": "string"}, + "c": { + "additionalProperties": {"items": {"type": "number"}, "type": "array"}, + "description": "carrot smoothie", + "title": "C", + "type": "object", + }, + }, + "required": ["a", "b", "c"], + "type": "object", + "additionalProperties": False, + }, + }, + } + ] + assert chatbot.llm_config["tools"] == expected_tools # type: ignore[index] diff --git a/test/interop/test_interoperability.py b/test/interop/test_interoperability.py new file mode 100644 index 0000000000..3925f056f1 --- /dev/null +++ b/test/interop/test_interoperability.py @@ -0,0 +1,62 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +import sys +from tempfile import TemporaryDirectory +from typing import Any + +import pytest + +from autogen.interop import Interoperability + + +class TestInteroperability: + def test_supported_types(self) -> None: + actual = Interoperability.get_supported_types() + + if sys.version_info < (3, 9): + assert actual == [] + + if sys.version_info >= (3, 9) and sys.version_info < (3, 10): + assert actual == ["langchain", "pydanticai"] + + if sys.version_info >= (3, 10) and sys.version_info < (3, 13): + assert actual == ["crewai", "langchain", "pydanticai"] + + if sys.version_info >= (3, 13): + assert actual == ["langchain", "pydanticai"] + + @pytest.mark.skipif( + sys.version_info < (3, 10) or sys.version_info >= (3, 13), reason="Only Python 3.10, 3.11, 3.12 are supported" + ) + def test_crewai(self) -> None: + from crewai_tools import FileReadTool + + crewai_tool = FileReadTool() + + tool = Interoperability.convert_tool(type="crewai", tool=crewai_tool) + + with TemporaryDirectory() as tmp_dir: + file_path = f"{tmp_dir}/test.txt" + with open(file_path, "w") as file: + file.write("Hello, World!") + + assert tool.name == "Read_a_file_s_content" + assert ( + tool.description + == "A tool that can be used to read a file's content. (IMPORTANT: When using arguments, put them all in an `args` dictionary)" + ) + + model_type = crewai_tool.args_schema + + args = model_type(file_path=file_path) + + assert tool.func(args=args) == "Hello, World!" + + @pytest.mark.skipif( + sys.version_info < (3, 9), reason="Only Python 3.9 and above are supported for LangchainInteroperability" + ) + @pytest.mark.skip(reason="This test is not yet implemented") + def test_langchain(self) -> None: + raise NotImplementedError("This test is not yet implemented") diff --git a/test/interop/test_interoperable.py b/test/interop/test_interoperable.py new file mode 100644 index 0000000000..164854f6aa --- /dev/null +++ b/test/interop/test_interoperable.py @@ -0,0 +1,9 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +from autogen.interop import Interoperable + + +def test_interoperable() -> None: + assert Interoperable is not None diff --git a/test/tools/__init__.py b/test/tools/__init__.py new file mode 100644 index 0000000000..bcd5401d54 --- /dev/null +++ b/test/tools/__init__.py @@ -0,0 +1,3 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 diff --git a/test/tools/test_tool.py b/test/tools/test_tool.py new file mode 100644 index 0000000000..b21edebd45 --- /dev/null +++ b/test/tools/test_tool.py @@ -0,0 +1,59 @@ +# Copyright (c) 2023 - 2024, Owners of https://github.com/ag2ai +# +# SPDX-License-Identifier: Apache-2.0 + +import os + +import pytest + +from autogen import AssistantAgent, UserProxyAgent +from autogen.tools import Tool + + +class TestTool: + @pytest.fixture(autouse=True) + def setup(self) -> None: + def f(x: str) -> str: + return x + "!" + + self.tool = Tool(name="test_tool", description="A test tool", func=f) + + def test_init(self) -> None: + assert self.tool.name == "test_tool" + assert self.tool.description == "A test tool" + + def test_register_for_llm(self) -> None: + config_list = [{"model": "gpt-4", "api_key": os.environ["OPENAI_API_KEY"]}] + + agent = AssistantAgent( + name="agent", + llm_config={"config_list": config_list}, + ) + + self.tool.register_for_llm(agent=agent) + + expected_tools = [ + { + "type": "function", + "function": { + "description": "A test tool", + "name": "test_tool", + "parameters": { + "type": "object", + "properties": {"x": {"type": "string", "description": "x"}}, + "required": ["x"], + }, + }, + } + ] + + assert agent.llm_config["tools"] == expected_tools # type: ignore[index] + + def test_register_for_execution(self) -> None: + user_proxy = UserProxyAgent( + name="user", + ) + + self.tool.register_for_execution(user_proxy) + assert user_proxy.can_execute_function("test_tool") + assert user_proxy.function_map["test_tool"]("Hello") == "Hello!" diff --git a/website/blog/2024-12-18-Tools-interoperability/index.mdx b/website/blog/2024-12-18-Tools-interoperability/index.mdx new file mode 100644 index 0000000000..3f6ae0962d --- /dev/null +++ b/website/blog/2024-12-18-Tools-interoperability/index.mdx @@ -0,0 +1,425 @@ +--- +title: Cross-Framework LLM Tool Integration with AG2 +authors: + - rjambrecic +tags: [LLM, tools, langchain, crewai, pydanticai] +--- + +**TL;DR** +AG2 lets you bring in tools from different frameworks like **LangChain**, **CrewAI**, and **PydanticAI**. + +- **LangChain**: Useful for tasks like API querying and web scraping. +- **CrewAI**: Offers a variety of tools for web scraping, search, and more. +- **PydanticAI**: Adds context-driven tools and structured data processing. + +With AG2, you can combine these tools and enhance your agents' capabilities. + +In this post, we’ll walk through how to integrate tools from various frameworks—like [LangChain Tools](https://python.langchain.com/v0.1/docs/modules/tools), +[CrewAI Tools](https://github.com/crewAIInc/crewAI-tools/tree/main), and [PydanticAI Tools](https://ai.pydantic.dev/tools/)—into AG2. +This allows you to use tools from different frameworks within AG2, giving your agents more power and flexibility. You’ll see how to set up agents, adapt the tools, and validate the integration through examples.his post, you will understand how to configure agents, adapt these tools for use in AG2, and validate the integration through practical examples. + +## LangChain Tools Integration + +LangChain is a popular framework that offers a wide range of tools to work with LLMs. LangChain has already implemented a variety of tools that can be easily integrated into AG2. You can explore the available tools in the [LangChain Community Tools](https://github.com/langchain-ai/langchain/tree/master/libs/community/langchain_community/tools) folder. These tools, such as those for querying APIs, web scraping, and text generation, can be quickly incorporated into AG2, providing powerful functionality for your agents. + +### Installation +To integrate LangChain tools into the AG2 framework, install the required dependencies: + +```bash +pip install ag2[interop-langchain] +``` + +Additionally, this notebook uses LangChain's [Wikipedia Tool](https://api.python.langchain.com/en/latest/tools/langchain_community.tools.wikipedia.tool.WikipediaQueryRun.html), which requires the `wikipedia` package. Install it with: + +```bash +pip install wikipedia +``` + +### Imports + +Import necessary modules and tools. +- `WikipediaQueryRun` and `WikipediaAPIWrapper`: Tools for querying Wikipedia. +- `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework. +- `Interoperability`: This module acts as a bridge, making it easier to integrate LangChain tools with AG2’s architecture. + +```python +import os + +from langchain_community.tools import WikipediaQueryRun +from langchain_community.utilities import WikipediaAPIWrapper + +from autogen import AssistantAgent, UserProxyAgent +from autogen.interop import Interoperability +``` + +### Agent Configuration + +Configure the agents for the interaction. +- `config_list` defines the LLM configurations, including the model and API key. +- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`). +- `AssistantAgent` represents the AI agent, configured with the LLM settings. + +```python +config_list = [{"model": "gpt-4o", "api_key": os.environ["OPENAI_API_KEY"]}] +user_proxy = UserProxyAgent( + name="User", + human_input_mode="NEVER", +) + +chatbot = AssistantAgent( + name="chatbot", + llm_config={"config_list": config_list}, +) +``` + +### Tool Integration + +- Initialize and register the LangChain tool with AG2. +- `WikipediaAPIWrapper`: Configured to fetch the top 1 result from Wikipedia with a maximum of 1000 characters per document. +- `WikipediaQueryRun`: A LangChain tool that executes Wikipedia queries. +- `LangchainInteroperability`: Converts the LangChain tool into a format compatible with the AG2 framework. +- `ag2_tool.register_for_execution(user_proxy)`: Registers the tool for use by the user_proxy agent. +- `ag2_tool.register_for_llm(chatbot)`: Registers the tool for integration with the chatbot agent. + +```python +api_wrapper = WikipediaAPIWrapper(top_k_results=1, doc_content_chars_max=1000) +langchain_tool = WikipediaQueryRun(api_wrapper=api_wrapper) + +interop = Interoperability() +ag2_tool = interop.convert_tool(tool=langchain_tool, type="langchain") + +ag2_tool.register_for_execution(user_proxy) +ag2_tool.register_for_llm(chatbot) +``` + +### Initiate Chat +```python +message = "Tell me about the history of the United States" +user_proxy.initiate_chat(recipient=chatbot, message=message, max_turns=2) +``` + +Output: + +```console +User (to chatbot): + +Tell me about the history of the United States + +-------------------------------------------------------------------------------- +chatbot (to User): + +***** Suggested tool call (call_hhy2G43ymytUFmJlDsK9J0tk): wikipedia ***** +Arguments: +{"tool_input":{"query":"history of the United States"}} +************************************************************************** + +-------------------------------------------------------------------------------- + +>>>>>>>> EXECUTING FUNCTION wikipedia... +User (to chatbot): + +***** Response from calling tool (call_hhy2G43ymytUFmJlDsK9J0tk) ***** +Page: History of the United States +Summary: The history of the lands that became the United States began with the arrival of the first people in the Americas around 15,000 BC. After European colonization of North America began in the late 15th century, wars and epidemics decimated Indigenous societies. By the 1760s, the thirteen British colonies were established. The Southern Colonies built an agricultural system on slave labor, enslaving millions from Africa. After defeating France, the British Parliament imposed a series of taxes; resistance to these taxes, especially the Boston Tea Party in 1773, led to Parliament issuing the Intolerable Acts designed to end self-government. +In 1776, the United States declared its independence. Led by General George Washington, it won the Revolutionary War in 1783. The Constitution was adopted in 1789, and a Bill of Rights was added in 1791 to guarantee inalienable rights. Washington, the first president, and his adviser Alexander Hamilton created a +********************************************************************** + +-------------------------------------------------------------------------------- +chatbot (to User): + +The history of the United States begins with the arrival of the first peoples in the Americas around 15,000 BC. This pre-Columbian era was followed by European colonization, beginning in the late 15th century, which dramatically altered the indigenous societies through wars and epidemics. + +By the 1760s, thirteen British colonies were established along the Atlantic seaboard. In the Southern Colonies, an agricultural economy heavily reliant on enslaved labor from Africa was developed. The British victory over France in the Seven Years' War led Parliament to impose various taxes on the colonies. Resistance to these taxes, exemplified by the Boston Tea Party in 1773, prompted the Parliament to enact the Intolerable Acts, seeking to curtail colonial self-governance. + +The United States declared independence in 1776. Under the leadership of General George Washington, the American Revolutionary War concluded successfully in 1783. Subsequently, the U.S. Constitution was adopted in 1789, with the Bill of Rights added in 1791 to ensure inalienable rights. During this early period, President George Washington and his advisor Alexander Hamilton played significant roles in forming the young nation's governmental and economic foundations. + +This overview covers the early formation and foundational moments of what became the United States, setting the stage for the country's subsequent expansion and development. TERMINATE + +-------------------------------------------------------------------------------- +``` + +## CrewAI Tools Integration + +CrewAI provides a variety of powerful tools designed for tasks such as web scraping, search, code interpretation, and more. These tools are easy to integrate into the AG2 framework, allowing you to enhance your agents with advanced capabilities. You can explore the full list of available tools in the [CrewAI Tools](https://github.com/crewAIInc/crewAI-tools/tree/main) repository. + +### Installation +Install the required packages for integrating CrewAI tools into the AG2 framework. +This ensures all dependencies for both frameworks are installed. + +```bash +pip install ag2[interop-crewai] +``` + +### Imports + +Import necessary modules and tools. +- `ScrapeWebsiteTool` are the CrewAI tools for web scraping +- `AssistantAgent` and `UserProxyAgent` are core AG2 classes. +- `Interoperability`: This module acts as a bridge, making it easier to integrate CrewAI tools with AG2’s architecture. + +```python +import os + +from crewai_tools import ScrapeWebsiteTool + +from autogen import AssistantAgent, UserProxyAgent +from autogen.interop import Interoperability +``` + +### Agent Configuration + +Configure the agents for the interaction. +- `config_list` defines the LLM configurations, including the model and API key. +- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`). +- `AssistantAgent` represents the AI agent, configured with the LLM settings. + +```python +config_list = [{"model": "gpt-4o", "api_key": os.environ["OPENAI_API_KEY"]}] +user_proxy = UserProxyAgent( + name="User", + human_input_mode="NEVER", +) + +chatbot = AssistantAgent( + name="chatbot", + llm_config={"config_list": config_list}, +) +``` + +### Tool Integration + +Initialize and register the CrewAI tool with AG2. +- `crewai_tool` is an instance of the `ScrapeWebsiteTool` from CrewAI. +- `Interoperability` converts the CrewAI tool to make it usable in AG2. +- `register_for_execution` and `register_for_llm` allow the tool to work with the UserProxyAgent and AssistantAgent. + +```python +interop = Interoperability() +crewai_tool = ScrapeWebsiteTool() +ag2_tool = interop.convert_tool(tool=crewai_tool, type="crewai") + +ag2_tool.register_for_execution(user_proxy) +ag2_tool.register_for_llm(chatbot) + +message = "Scrape the website https://ag2.ai/" + +chat_result = user_proxy.initiate_chat(recipient=chatbot, message=message, max_turns=2) +``` + +Output: + +```console +User (to chatbot): + +Scrape the website https://ag2.ai/ + +-------------------------------------------------------------------------------- +chatbot (to User): + +***** Suggested tool call (call_ZStuwmexfN7j56uJKOi6BCid): Read_website_content ***** +Arguments: +{"args":{"website_url":"https://ag2.ai/"}} +************************************************************************************* + +-------------------------------------------------------------------------------- + +>>>>>>>> EXECUTING FUNCTION Read_website_content... +Using Tool: Read website content +User (to chatbot): + +***** Response from calling tool (call_ZStuwmexfN7j56uJKOi6BCid) ***** + +AgentOS +Join our growing community of over 20,000 agent builders Join our growing community of over 20,000 agent builders The Open-Source AgentOS Build production-ready multi-agent systems in minutes, not months. Github Discord The End-to-End Platform for Multi-Agent Automation The End-to-End Platform for Multi-Agent Automation Flexible Agent Construction and Orchestration Create specialized agents that work together seamlessly. AG2 makes it easy to define roles, configure behaviors, and orchestrate collaboration - all through simple, intuitive code. → Assistant agents for problem-solving → Executor agents for taking action → Critic agents for validation → Group chat managers for coordination Built-in Conversation Patterns Built-in Conversation Patterns Stop wrestling with agent coordination. AG2 handles message routing, state management, and conversation flow automatically. → Two-agent conversations → Group chats with dynamic speaker selection → Sequential chats with context carryover → Nested conversations for modularity Seamless Human-AI collaboration Seamless Human-AI collaboration Seamlessly integrate human oversight and input into your agent workflows. → Configurable human input modes → Flexible intervention points → Optional human approval workflows → Interactive conversation interfaces → Context-aware human handoff Roadmap AG2 STUDIO → Visual agent system design → Real-time testing and debugging → One-click deployment to production → Perfect for prototyping and MVPs AG2 STUDIO → Visual agent system design → Real-time testing and debugging → One-click deployment to production → Perfect for prototyping and MVPs AG2 STUDIO → Visual agent system design → Real-time testing and debugging → One-click deployment to production → Perfect for prototyping and MVPs AG2 MARKETPLACE → Share and monetize your agents → Discover pre-built solution templates → Quick-start your agent development → Connect with other builders AG2 MARKETPLACE → Share and monetize your agents → Discover pre-built solution templates → Quick-start your agent development → Connect with other builders AG2 MARKETPLACE → Share and monetize your agents → Discover pre-built solution templates → Quick-start your agent development → Connect with other builders SCALING TOOLS → Zero to production deployment guides → Usage analytics and cost optimization → Team collaboration features → Enterprise-ready security controls SCALING TOOLS → Zero to production deployment guides → Usage analytics and cost optimization → Team collaboration features → Enterprise-ready security controls SCALING TOOLS → Zero to production deployment guides → Usage analytics and cost optimization → Team collaboration features → Enterprise-ready security controls AG2 STUDIO → Visual agent system design → Real-time testing and debugging → One-click deployment to production → Perfect for prototyping and MVPs AG2 STUDIO → Visual agent system design → Real-time testing and debugging → One-click deployment to production → Perfect for prototyping and MVPs AG2 MARKETPLACE → Share and monetize your agents → Discover pre-built solution templates → Quick-start your agent development → Connect with other builders AG2 MARKETPLACE → Share and monetize your agents → Discover pre-built solution templates → Quick-start your agent development → Connect with other builders SCALING TOOLS → Zero to production deployment guides → Usage analytics and cost optimization → Team collaboration features → Enterprise-ready security controls SCALING TOOLS → Zero to production deployment guides → Usage analytics and cost optimization → Team collaboration features → Enterprise-ready security controls Whether you're a solo founder prototyping the next big AI product, or an enterprise team deploying at scale we're building AG2 for you. This is AgentOS - making multi-agent development accessible to everyone. Github Join Our Growing Community Join Our Growing Community → 20,000+ active agent builders → Daily technical discussions → Weekly community calls → Open RFC process → Regular contributor events (Coming soon) Discord Problem Features Roadmap Community Documentation Problem Features Roadmap Community Documentation Problem Features Roadmap Community Documentation + +********************************************************************** + +-------------------------------------------------------------------------------- +chatbot (to User): + +The website "https://ag2.ai/" promotes a platform named AgentOS, which is designed for building multi-agent systems efficiently. Key highlights from the website are: + +- **Community**: They have a growing community of over 20,000 agent builders. + +- **End-to-End Platform**: AG2 is described as an end-to-end platform for multi-agent automation. It supports flexible agent construction and orchestration, helping to define roles, configure behaviors, and orchestrate collaboration. + +- **Agent Types**: It includes assistant agents for problem-solving, executor agents for taking action, critic agents for validation, and group chat managers for coordination. + +- **Built-in Conversation Patterns**: AG2 offers capabilities for message routing, state management, and conversation flow management, supporting various conversation types like two-agent conversations, group chats, and nested conversations. + +- **Human-AI Collaboration**: The platform facilitates seamless integration of human oversight and input, with options for human intervention and approval workflows. + +- **AG2 Studio**: This feature provides visual agent system design, real-time testing, debugging, and one-click deployment, suited for prototyping and MVPs. + +- **AG2 Marketplace**: Provides a place to share, monetize agents, discover pre-built solution templates, and connect with other builders. + +- **Scaling Tools**: Includes guides for deployment, analytics, cost optimization, team collaboration features, and enterprise-ready security controls. + +- **Community and Documentation**: They encourage connecting through GitHub and Discord and have regular community calls and events planned. + +This comprehensive platform seems to aim at both individual developers and enterprise teams looking to deploy multi-agent systems effectively and collaboratively. TERMINATE + +-------------------------------------------------------------------------------- +``` + +```python +print(chat_result.summary) +``` + +Output: + +```console +The website "https://ag2.ai/" promotes a platform named AgentOS, which is designed for building multi-agent systems efficiently. Key highlights from the website are: + +- **Community**: They have a growing community of over 20,000 agent builders. + +- **End-to-End Platform**: AG2 is described as an end-to-end platform for multi-agent automation. It supports flexible agent construction and orchestration, helping to define roles, configure behaviors, and orchestrate collaboration. + +- **Agent Types**: It includes assistant agents for problem-solving, executor agents for taking action, critic agents for validation, and group chat managers for coordination. + +- **Built-in Conversation Patterns**: AG2 offers capabilities for message routing, state management, and conversation flow management, supporting various conversation types like two-agent conversations, group chats, and nested conversations. + +- **Human-AI Collaboration**: The platform facilitates seamless integration of human oversight and input, with options for human intervention and approval workflows. + +- **AG2 Studio**: This feature provides visual agent system design, real-time testing, debugging, and one-click deployment, suited for prototyping and MVPs. + +- **AG2 Marketplace**: Provides a place to share, monetize agents, discover pre-built solution templates, and connect with other builders. + +- **Scaling Tools**: Includes guides for deployment, analytics, cost optimization, team collaboration features, and enterprise-ready security controls. + +- **Community and Documentation**: They encourage connecting through GitHub and Discord and have regular community calls and events planned. + +This comprehensive platform seems to aim at both individual developers and enterprise teams looking to deploy multi-agent systems effectively and collaboratively. +``` + +## PydanticAI Tools Integration + +[PydanticAI](https://ai.pydantic.dev/) is a newer framework that offers powerful capabilities for working with LLMs. Although it currently does not have a repository with pre-built tools, it provides features like **dependency injection**, allowing you to inject a "Context" into a tool for better execution without relying on LLMs. This context can be used for passing parameters or managing state during the execution of a tool. While the framework is still growing, you can integrate its tools into AG2 to enhance agent capabilities, especially for tasks that involve structured data and context-driven logic. + +### Installation +To integrate LangChain tools into the AG2 framework, install the required dependencies: + +```bash +pip install ag2[interop-pydantic-ai] +``` + + +### Imports + +Import necessary modules and tools. +- `BaseModel`: Used to define data structures for tool inputs and outputs. +- `RunContext`: Provides context during the execution of tools. +- `PydanticAITool`: Represents a tool in the PydanticAI framework. +- `AssistantAgent` and `UserProxyAgent`: Agents that facilitate communication in the AG2 framework. +- `Interoperability`: This module acts as a bridge, making it easier to integrate PydanticAI tools with AG2’s architecture. + +```python +import os +from typing import Optional + +from pydantic import BaseModel +from pydantic_ai import RunContext +from pydantic_ai.tools import Tool as PydanticAITool + +from autogen import AssistantAgent, UserProxyAgent +from autogen.interop import Interoperability +``` + +### Agent Configuration + +Configure the agents for the interaction. +- `config_list` defines the LLM configurations, including the model and API key. +- `UserProxyAgent` simulates user inputs without requiring actual human interaction (set to `NEVER`). +- `AssistantAgent` represents the AI agent, configured with the LLM settings. + +```python +config_list = [{"model": "gpt-4o", "api_key": os.environ["OPENAI_API_KEY"]}] +user_proxy = UserProxyAgent( + name="User", + human_input_mode="NEVER", +) + +chatbot = AssistantAgent( + name="chatbot", + llm_config={"config_list": config_list}, +) +``` + +### Tool Integration + +Integrate the PydanticAI tool with AG2. + +- Define a `Player` model using `BaseModel` to structure the input data. +- Use `RunContext` to securely inject dependencies (like the `Player` instance) into the tool function without exposing them to the LLM. +- Implement `get_player` to define the tool's functionality, accessing `ctx.deps` for injected data. +- Convert the tool to an AG2-compatible format with `Interoperability` and register it for execution and LLM communication. +- Convert the PydanticAI tool into an AG2-compatible format using `convert_tool`. +- Register the tool for both execution and communication with the LLM by associating it with the `user_proxy` and `chatbot`. + +```python +class Player(BaseModel): + name: str + age: int + + +def get_player(ctx: RunContext[Player], additional_info: Optional[str] = None) -> str: # type: ignore[valid-type] + """Get the player's name. + + Args: + additional_info: Additional information which can be used. + """ + return f"Name: {ctx.deps.name}, Age: {ctx.deps.age}, Additional info: {additional_info}" # type: ignore[attr-defined] + + +interop = Interoperability() +pydantic_ai_tool = PydanticAITool(get_player, takes_ctx=True) + +# player will be injected as a dependency +player = Player(name="Luka", age=25) +ag2_tool = interop.convert_tool(tool=pydantic_ai_tool, type="pydanticai", deps=player) + +ag2_tool.register_for_execution(user_proxy) +ag2_tool.register_for_llm(chatbot) +``` + +Initiate a conversation between the `UserProxyAgent` and the `AssistantAgent`. + +- Use the `initiate_chat` method to send a message from the `user_proxy` to the `chatbot`. +- In this example, the user requests the chatbot to retrieve player information, providing "goal keeper" as additional context. +- The `Player` instance is securely injected into the tool using `RunContext`, ensuring the chatbot can retrieve and use this data during the interaction. + +```python +user_proxy.initiate_chat( + recipient=chatbot, message="Get player, for additional information use 'goal keeper'", max_turns=3 +) +``` + +Output: + +```console +User (to chatbot): + +Get player, for additional information use 'goal keeper' + +-------------------------------------------------------------------------------- +chatbot (to User): + +***** Suggested tool call (call_lPXIohFiJfnjmgwDnNFPQCzc): get_player ***** +Arguments: +{"additional_info":"goal keeper"} +*************************************************************************** + +-------------------------------------------------------------------------------- + +>>>>>>>> EXECUTING FUNCTION get_player... +User (to chatbot): + +***** Response from calling tool (call_lPXIohFiJfnjmgwDnNFPQCzc) ***** +Name: Luka, Age: 25, Additional info: goal keeper +********************************************************************** + +-------------------------------------------------------------------------------- +chatbot (to User): + +The player's name is Luka, who is a 25-year-old goalkeeper. TERMINATE +``` diff --git a/website/blog/authors.yml b/website/blog/authors.yml index f93f3aee16..f689a1dfd6 100644 --- a/website/blog/authors.yml +++ b/website/blog/authors.yml @@ -188,3 +188,9 @@ AgentGenie: title: AG2 Contributor url: https://github.com/AgentGenie image_url: https://github.com/AgentGenie.png + +rjambrecic: + name: Robert Jambrecic + title: Machine Learning Engineer at Airt + url: https://github.com/rjambrecic + image_url: https://github.com/rjambrecic.png