diff --git a/autogen/agentchat/realtime_agent/function_observer.py b/autogen/agentchat/realtime_agent/function_observer.py index 7d0f2fe9c3..8818e27d4a 100644 --- a/autogen/agentchat/realtime_agent/function_observer.py +++ b/autogen/agentchat/realtime_agent/function_observer.py @@ -21,35 +21,23 @@ async def update(self, response): await self.call_function(response["call_id"], **json.loads(response["arguments"])) async def call_function(self, call_id, location): - function_result = { - "type": "conversation.item.create", - "item": { - "type": "function_call_output", - "call_id": call_id, - "output": "The weather is cloudy." if location == "Seattle" else "The weather is sunny.", - }, - } - await self.client.openai_ws.send(json.dumps(function_result)) - await self.client.openai_ws.send(json.dumps({"type": "response.create"})) + result = "The weather is cloudy." if location == "Seattle" else "The weather is sunny." + await self.client.function_result(call_id, result) - async def run(self, openai_ws): - await self.initialize_session(openai_ws) + async def run(self): + await self.initialize_session() - async def initialize_session(self, openai_ws): + async def initialize_session(self): """Add tool to OpenAI.""" session_update = { - "type": "session.update", - "session": { - "tools": [ - { - "name": "get_weather", - "description": "Get the current weather", - "parameters": {"type": "object", "properties": {"location": {"type": "string"}}}, - "type": "function", - } - ], - "tool_choice": "auto", - }, + "tools": [ + { + "name": "get_weather", + "description": "Get the current weather", + "parameters": {"type": "object", "properties": {"location": {"type": "string"}}}, + "type": "function", + } + ], + "tool_choice": "auto", } - print("Sending session update:", json.dumps(session_update)) - await openai_ws.send(json.dumps(session_update)) + await self.client.session_update(session_update) diff --git a/autogen/agentchat/realtime_agent/realtime_agent.py b/autogen/agentchat/realtime_agent/realtime_agent.py index 12e8b0f9e3..c6dc3ea0e7 100644 --- a/autogen/agentchat/realtime_agent/realtime_agent.py +++ b/autogen/agentchat/realtime_agent/realtime_agent.py @@ -66,9 +66,21 @@ async def notify_observers(self, message): for observer in self.observers: await observer.update(message) - async def _read_from_client(self, openai_ws): + async def function_result(self, call_id, result): + result_item = { + "type": "conversation.item.create", + "item": { + "type": "function_call_output", + "call_id": call_id, + "output": result, + }, + } + await self.openai_ws.send(json.dumps(result_item)) + await self.openai_ws.send(json.dumps({"type": "response.create"})) + + async def _read_from_client(self): try: - async for openai_message in openai_ws: + async for openai_message in self.openai_ws: response = json.loads(openai_message) await self.notify_observers(response) except Exception as e: @@ -82,20 +94,26 @@ async def run(self): self.openai_ws = openai_ws await self.initialize_session() await asyncio.gather( - self._read_from_client(openai_ws), *[observer.run(openai_ws) for observer in self.observers] + self._read_from_client(), *[observer.run() for observer in self.observers] ) async def initialize_session(self): """Control initial session with OpenAI.""" session_update = { + "turn_detection": {"type": "server_vad"}, + "voice": self.voice, + "instructions": self.system_message, + "modalities": ["text", "audio"], + "temperature": 0.8, + "input_audio_format": "g711_ulaw", + "output_audio_format": "g711_ulaw", + } + await self.session_update(session_update) + + async def session_update(self, session_options): + update = { "type": "session.update", - "session": { - "turn_detection": {"type": "server_vad"}, - "voice": self.voice, - "instructions": self.system_message, - "modalities": ["text", "audio"], - "temperature": 0.8, - }, + "session": session_options } - print("Sending session update:", json.dumps(session_update)) - await self.openai_ws.send(json.dumps(session_update)) + print("Sending session update:", json.dumps(update)) + await self.openai_ws.send(json.dumps(update)) diff --git a/autogen/agentchat/realtime_agent/twilio_observer.py b/autogen/agentchat/realtime_agent/twilio_observer.py index aa1152febb..2082c60976 100644 --- a/autogen/agentchat/realtime_agent/twilio_observer.py +++ b/autogen/agentchat/realtime_agent/twilio_observer.py @@ -29,7 +29,7 @@ class TwilioAudioAdapter(RealtimeObserver): def __init__(self, websocket): super().__init__() - + self.client = None self.websocket = websocket # Connection specific state @@ -101,8 +101,9 @@ async def send_mark(self): await self.websocket.send_json(mark_event) self.mark_queue.append("responsePart") - async def run(self, openai_ws): - await self.initialize_session(openai_ws) + async def run(self): + openai_ws = self.client.openai_ws + await self.initialize_session() async for message in self.websocket.iter_text(): data = json.loads(message) @@ -121,14 +122,10 @@ async def run(self, openai_ws): self.mark_queue.pop(0) - async def initialize_session(self, openai_ws): + async def initialize_session(self): """Control initial session with OpenAI.""" session_update = { - "type": "session.update", - "session": { - "input_audio_format": "g711_ulaw", - "output_audio_format": "g711_ulaw", - }, + "input_audio_format": "g711_ulaw", + "output_audio_format": "g711_ulaw", } - print("Sending session update:", json.dumps(session_update)) - await openai_ws.send(json.dumps(session_update)) + await self.client.session_update(session_update)