Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

removing openai_ws from observers #198

Merged
merged 1 commit into from
Dec 13, 2024
Merged

removing openai_ws from observers #198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 15 additions & 27 deletions autogen/agentchat/realtime_agent/function_observer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,35 +21,23 @@ async def update(self, response):
await self.call_function(response["call_id"], **json.loads(response["arguments"]))

async def call_function(self, call_id, location):
function_result = {
"type": "conversation.item.create",
"item": {
"type": "function_call_output",
"call_id": call_id,
"output": "The weather is cloudy." if location == "Seattle" else "The weather is sunny.",
},
}
await self.client.openai_ws.send(json.dumps(function_result))
await self.client.openai_ws.send(json.dumps({"type": "response.create"}))
result = "The weather is cloudy." if location == "Seattle" else "The weather is sunny."
await self.client.function_result(call_id, result)

async def run(self, openai_ws):
await self.initialize_session(openai_ws)
async def run(self):
await self.initialize_session()

async def initialize_session(self, openai_ws):
async def initialize_session(self):
"""Add tool to OpenAI."""
session_update = {
"type": "session.update",
"session": {
"tools": [
{
"name": "get_weather",
"description": "Get the current weather",
"parameters": {"type": "object", "properties": {"location": {"type": "string"}}},
"type": "function",
}
],
"tool_choice": "auto",
},
"tools": [
{
"name": "get_weather",
"description": "Get the current weather",
"parameters": {"type": "object", "properties": {"location": {"type": "string"}}},
"type": "function",
}
],
"tool_choice": "auto",
}
print("Sending session update:", json.dumps(session_update))
await openai_ws.send(json.dumps(session_update))
await self.client.session_update(session_update)
42 changes: 30 additions & 12 deletions autogen/agentchat/realtime_agent/realtime_agent.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,9 +66,21 @@ async def notify_observers(self, message):
for observer in self.observers:
await observer.update(message)

async def _read_from_client(self, openai_ws):
async def function_result(self, call_id, result):
result_item = {
"type": "conversation.item.create",
"item": {
"type": "function_call_output",
"call_id": call_id,
"output": result,
},
}
await self.openai_ws.send(json.dumps(result_item))
await self.openai_ws.send(json.dumps({"type": "response.create"}))

async def _read_from_client(self):
try:
async for openai_message in openai_ws:
async for openai_message in self.openai_ws:
response = json.loads(openai_message)
await self.notify_observers(response)
except Exception as e:
Expand All @@ -82,20 +94,26 @@ async def run(self):
self.openai_ws = openai_ws
await self.initialize_session()
await asyncio.gather(
self._read_from_client(openai_ws), *[observer.run(openai_ws) for observer in self.observers]
self._read_from_client(), *[observer.run() for observer in self.observers]
)

async def initialize_session(self):
"""Control initial session with OpenAI."""
session_update = {
"turn_detection": {"type": "server_vad"},
"voice": self.voice,
"instructions": self.system_message,
"modalities": ["text", "audio"],
"temperature": 0.8,
"input_audio_format": "g711_ulaw",
"output_audio_format": "g711_ulaw",
}
await self.session_update(session_update)

async def session_update(self, session_options):
update = {
"type": "session.update",
"session": {
"turn_detection": {"type": "server_vad"},
"voice": self.voice,
"instructions": self.system_message,
"modalities": ["text", "audio"],
"temperature": 0.8,
},
"session": session_options
}
print("Sending session update:", json.dumps(session_update))
await self.openai_ws.send(json.dumps(session_update))
print("Sending session update:", json.dumps(update))
await self.openai_ws.send(json.dumps(update))
19 changes: 8 additions & 11 deletions autogen/agentchat/realtime_agent/twilio_observer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
class TwilioAudioAdapter(RealtimeObserver):
def __init__(self, websocket):
super().__init__()

self.client = None
self.websocket = websocket

# Connection specific state
Expand Down Expand Up @@ -101,8 +101,9 @@ async def send_mark(self):
await self.websocket.send_json(mark_event)
self.mark_queue.append("responsePart")

async def run(self, openai_ws):
await self.initialize_session(openai_ws)
async def run(self):
openai_ws = self.client.openai_ws
await self.initialize_session()

async for message in self.websocket.iter_text():
data = json.loads(message)
Expand All @@ -121,14 +122,10 @@ async def run(self, openai_ws):
self.mark_queue.pop(0)


async def initialize_session(self, openai_ws):
async def initialize_session(self):
"""Control initial session with OpenAI."""
session_update = {
"type": "session.update",
"session": {
"input_audio_format": "g711_ulaw",
"output_audio_format": "g711_ulaw",
},
"input_audio_format": "g711_ulaw",
"output_audio_format": "g711_ulaw",
}
print("Sending session update:", json.dumps(session_update))
await openai_ws.send(json.dumps(session_update))
await self.client.session_update(session_update)
Loading