Feat: voice chat with AI

enjoy/package.json

@@ -92,6 +92,7 @@
     "@radix-ui/react-alert-dialog": "^1.1.1",
     "@radix-ui/react-aspect-ratio": "^1.1.0",
     "@radix-ui/react-avatar": "^1.1.0",
+    "@radix-ui/react-checkbox": "^1.1.1",
     "@radix-ui/react-collapsible": "^1.1.0",
     "@radix-ui/react-dropdown-menu": "^2.1.1",
     "@radix-ui/react-hover-card": "^1.1.1",
@@ -112,6 +113,8 @@
     "@radix-ui/react-toggle": "^1.1.0",
     "@radix-ui/react-tooltip": "^1.1.2",
     "@rails/actioncable": "7.1.3",
+    "@ricky0123/vad-react": "^0.0.24",
+    "@ricky0123/vad-web": "^0.0.18",
     "@sentry/electron": "^5.3.0",
     "@uidotdev/usehooks": "^2.4.1",
     "@vidstack/react": "^1.11.30",
@@ -156,7 +159,9 @@
     "postcss": "^8.4.40",
     "proxy-agent": "^6.4.0",
     "react": "^18.3.1",
-    "react-activity-calendar": "^2.3.1",
+    "react-activity-calendar": "^2.2.11",
+    "react-audio-visualize": "^1.1.3",
+    "react-audio-voice-recorder": "^2.2.0",
     "react-dom": "^18.3.1",
     "react-hook-form": "^7.52.2",
     "react-hotkeys-hook": "^4.5.0",

enjoy/src/commands/index.ts

@@ -7,3 +7,4 @@ export * from "./analyze.command";
 export * from "./punctuate.command";
 export * from "./summarize-topic.command";
 export * from "./text.command";
+export * from "./refine.command";

enjoy/src/commands/refine.command.ts

@@ -0,0 +1,39 @@
+import { ChatPromptTemplate } from "@langchain/core/prompts";
+import { textCommand } from "./text.command";
+import { LANGUAGES } from "@/constants";
+
+export const refineCommand = async (
+  text: string,
+  params: {
+    learningLanguage: string;
+    nativeLanguage: string;
+    context: string;
+  },
+  options: {
+    key: string;
+    modelName?: string;
+    temperature?: number;
+    baseUrl?: string;
+  }
+): Promise<string> => {
+  if (!text) throw new Error("Text is required");
+
+  const { learningLanguage, nativeLanguage, context = "None" } = params;
+  const prompt = await ChatPromptTemplate.fromMessages([
+    ["system", SYSTEM_PROMPT],
+    ["human", text],
+  ]).format({
+    learning_language: LANGUAGES.find((l) => l.code === learningLanguage).name,
+    native_language: LANGUAGES.find((l) => l.code === nativeLanguage).name,
+    context,
+  });
+
+  return textCommand(prompt, options);
+};
+
+const SYSTEM_PROMPT = `I speak {native_language}. You're my {learning_language} coach. I'll give you my expression in {learning_language}. And I may also provide some context about my expression. 
+
+Please try to understand my true meaning and provide several refined expressions in the native way. And explain them in {native_language}.
+
+[Context]
+{context}`;

enjoy/src/constants/index.ts

@@ -1,11 +1,11 @@
-export * from './gpt-presets';
-export * from './ipa';
+export * from "./gpt-presets";
+export * from "./ipa";
 
 // https://hf-mirror.com/ggerganov/whisper.cpp/tree/main
-import whisperModels from './whisper-models.json';
+import whisperModels from "./whisper-models.json";
 export const WHISPER_MODELS_OPTIONS = whisperModels;
 
-import languages from './languages.json';
+import languages from "./languages.json";
 export const LANGUAGES = languages;
 
 export const DATABASE_NAME = "enjoy_database";
@@ -22,7 +22,8 @@ export const AI_WORKER_ENDPOINT = "https://ai-worker.enjoy.bot";
 export const WEB_API_URL = "https://enjoy.bot";
 export const WS_URL = "wss://enjoy.bot";
 
-export const REPO_URL = "https://github.com/zuodaotech/everyone-can-use-english";
+export const REPO_URL =
+  "https://github.com/zuodaotech/everyone-can-use-english";
 
 export const SENTRY_DSN =
   "https://d51056d7af7d14eae446c0c15b4f3d31@o1168905.ingest.us.sentry.io/4506969353289728";
@@ -55,4 +56,55 @@ export const NOT_SUPPORT_JSON_FORMAT_MODELS = [
   "gpt-4-vision-preview",
   "gpt-4",
   "gpt-4-32k",
-];
+];
+
+export const CHAT_SYSTEM_PROMPT_TEMPLATE = `You are {name}. {agent_prompt}
+You are chatting in an online chat room.
+{agent_chat_prompt}
+
+[Rules must be followed]
+1. Always reply in {language}.
+2. Reply in your personality style and talk in casual way.
+3. Reply what you would say only, do not include any other format.
+
+[Chat Topic]
+{topic}
+
+[Chat Members]
+{members}
+
+[Chat History]
+{history}
+`;
+
+export const AGENT_FIXTURE_AVA = {
+  name: "Ava",
+  introduction: "I'm Ava, your English speaking teacher.",
+  language: "en-US",
+  config: {
+    engine: "enjoyai",
+    model: "gpt-4o",
+    prompt:
+      "You are an experienced English teacher who excels at improving students' speaking skills. You always use simple yet authentic words and sentences to help students understand.",
+    temperature: 1,
+    ttsEngine: "enjoyai",
+    ttsModel: "azure/speech",
+    ttsVoice: "en-US-AvaNeural",
+  },
+};
+
+export const AGENT_FIXTURE_ANDREW = {
+  name: "Andrew",
+  introduction: "I'm Andrew, your American friend.",
+  language: "en-US",
+  config: {
+    engine: "enjoyai",
+    model: "gpt-4o",
+    prompt:
+      "You're a native American who speaks authentic American English, familiar with the culture and customs of the U.S. You're warm and welcoming, eager to make friends from abroad and share all aspects of American life.",
+    temperature: 0.9,
+    ttsEngine: "enjoyai",
+    ttsModel: "azure/speech",
+    ttsVoice: "en-US-AndrewNeural",
+  },
+};

enjoy/src/i18n/en.json

@@ -129,10 +129,39 @@
         "missingBreak": "There is no break between two words when there is a punctuation between them.",
         "monotone": "The word is monotone."
       }
+    },
+    "chatAgent": {
+      "notFound": "Chat agent not found",
+      "created": "Chat agent created",
+      "updated": "Chat agent updated",
+      "deleted": "Chat agent deleted",
+      "name": "Name",
+      "introduction": "Introduction",
+      "language": "Language",
+      "prompt": "Prompt",
+      "engine": "AI Engine",
+      "model": "AI Model",
+      "temperature": "Temperature",
+      "temperatureDescription": "The higher the temperature, the more creative the result",
+      "ttsEngine": "TTS Engine",
+      "ttsModel": "TTS Model",
+      "ttsVoice": "TTS Voice"
+    },
+    "chat": {
+      "notFound": "Chat not found",
+      "created": "Chat created",
+      "updated": "Chat updated",
+      "deleted": "Chat deleted",
+      "name": "Name",
+      "language": "Language",
+      "topic": "Topic",
+      "members": "Members",
+      "memberConfig": "Member Configuration"
     }
   },
   "sidebar": {
     "home": "Home",
+    "chats": "Chats",
     "courses": "Courses",
     "community": "Community",
     "audios": "Audios",
@@ -660,5 +689,30 @@
   "platformInfo": "Platform Information",
   "networkState": "Network State",
   "connectError": "Connect Error",
-  "refresh": "refresh"
+  "refresh": "refresh",
+  "deleteChat": "Delete chat",
+  "deleteChatConfirmation": "Are you sure to delete this chat?",
+  "deleteChatAgent": "Delete agent",
+  "deleteChatAgentConfirmation": "Are you sure to delete this chat agent?",
+  "deleteMessage": "Delete message",
+  "deleteMessageConfirmation": "Are you sure to delete this message?",
+  "suggestion": "Suggestion",
+  "editChat": "Edit chat",
+  "newChat": "New chat",
+  "addChat": "Add chat",
+  "agentsManagement": "Agents management",
+  "newAgent": "New agent",
+  "editAgent": "Edit agent",
+  "addToChat": "Add to chat",
+  "introduction": "Introduction",
+  "introduceYourself": "Introduce yourself",
+  "prompt": "Prompt",
+  "extraPromptForChat": "Extra prompt for this chat",
+  "promptPreview": "Prompt preview",
+  "noChatSelected": "No chat selected",
+  "reRecord": "Re-record",
+  "quickStart": "Quick start",
+  "itsYourTurn": "It's your turn. Please start speaking.",
+  "displayContent": "Display content",
+  "hideContent": "Hide content"
 }

enjoy/src/i18n/zh-CN.json

@@ -129,10 +129,39 @@
         "missingBreak": "当两个单词之前存在标点符号时，单词之间没有停顿",
         "monotone": "这些单词以平淡且不兴奋的语调发音，没有任何节奏或表达"
       }
+    },
+    "chatAgent": {
+      "notFound": "未找到角色",
+      "created": "角色创建成功",
+      "updated": "角色更新成功",
+      "deleted": "角色删除成功",
+      "name": "角色名称",
+      "introduction": "角色介绍",
+      "language": "语言",
+      "prompt": "提示语",
+      "engine": "AI 引擎",
+      "model": "AI 模型",
+      "temperature": "随机性",
+      "temperatureDescription": "值越高，生成的文本越具创造性，反之则越稳定",
+      "ttsEngine": "TTS 引擎",
+      "ttsModel": "TTS 模型",
+      "ttsVoice": "TTS 音色"
+    },
+    "chat": {
+      "notFound": "未找到聊天",
+      "created": "聊天创建成功",
+      "updated": "聊天更新成功",
+      "deleted": "聊天删除成功",
+      "name": "聊天标题",
+      "language": "语言",
+      "topic": "话题",
+      "members": "成员",
+      "memberConfig": "成员配置"
     }
   },
   "sidebar": {
     "home": "主页",
+    "chats": "聊天",
     "courses": "课程",
     "community": "社区",
     "audios": "音频",
@@ -660,5 +689,30 @@
   "platformInfo": "设备信息",
   "networkState": "网络状态",
   "connectError": "连接错误",
-  "refresh": "刷新"
+  "refresh": "刷新",
+  "deleteChat": "删除聊天",
+  "deleteChatConfirmation": "您确定要删除此聊天吗？",
+  "deleteChatAgent": "删除角色",
+  "deleteChatAgentConfirmation": "您确定要删除此角色吗？",
+  "deleteMessage": "删除消息",
+  "deleteMessageConfirmation": "您确定要删除此消息吗？",
+  "suggestion": "修改建议",
+  "editChat": "编辑聊天",
+  "newChat": "新聊天",
+  "addChat": "添加聊天",
+  "agentsManagement": "角色管理",
+  "newAgent": "新角色",
+  "editAgent": "编辑角色",
+  "addToChat": "添加到聊天",
+  "introduction": "介绍",
+  "introduceYourself": "自我介绍",
+  "prompt": "提示语",
+  "extraPromptForChat": "额外提示语",
+  "promptPreview": "提示语预览",
+  "noChatSelected": "未选择聊天",
+  "reRecord": "重新录音",
+  "quickStart": "快速开始",
+  "itsYourTurn": "轮到您了，请开始说点什么",
+  "displayContent": "显示内容",
+  "hideContent": "隐藏内容"
 }

enjoy/src/main/db/handlers/chat-agents-handler.ts

@@ -0,0 +1,88 @@
+import { ipcMain, IpcMainEvent } from "electron";
+import { ChatAgent } from "@main/db/models";
+import { FindOptions, WhereOptions, Attributes, Op } from "sequelize";
+import log from "@main/logger";
+import { t } from "i18next";
+
+const logger = log.scope("db/handlers/chat-agents-handler");
+
+class ChatAgentsHandler {
+  private async findAll(
+    _event: IpcMainEvent,
+    options: FindOptions<Attributes<ChatAgent>> & { query?: string }
+  ) {
+    const { query, where = {} } = options || {};
+    delete options.query;
+    delete options.where;
+
+    if (query) {
+      (where as any).name = {
+        [Op.like]: `%${query}%`,
+      };
+    }
+    const agents = await ChatAgent.findAll({
+      order: [["name", "ASC"]],
+      where,
+      ...options,
+    });
+
+    if (!agents) {
+      return [];
+    }
+    return agents.map((agent) => agent.toJSON());
+  }
+
+  private async findOne(
+    _event: IpcMainEvent,
+    options: FindOptions<Attributes<ChatAgent>> & {
+      where: WhereOptions<ChatAgent>;
+    }
+  ) {
+    const agent = await ChatAgent.findOne(options);
+    return agent?.toJSON();
+  }
+
+  private async create(
+    _event: IpcMainEvent,
+    data: { name: string; description: string; language: string; config: any }
+  ) {
+    const agent = await ChatAgent.create(data);
+    return agent.toJSON();
+  }
+
+  private async update(
+    _event: IpcMainEvent,
+    id: string,
+    data: {
+      name: string;
+      description: string;
+      language: string;
+      config: any;
+    }
+  ) {
+    const agent = await ChatAgent.findByPk(id);
+    if (!agent) {
+      throw new Error(t("models.chatAgent.notFound"));
+    }
+    await agent.update(data);
+    return agent.toJSON();
+  }
+
+  private async destroy(_event: IpcMainEvent, id: string) {
+    const agent = await ChatAgent.findByPk(id);
+    if (!agent) {
+      throw new Error(t("models.chatAgent.notFound"));
+    }
+    agent.destroy();
+  }
+
+  register() {
+    ipcMain.handle("chat-agents-find-all", this.findAll);
+    ipcMain.handle("chat-agents-find-one", this.findOne);
+    ipcMain.handle("chat-agents-create", this.create);
+    ipcMain.handle("chat-agents-update", this.update);
+    ipcMain.handle("chat-agents-destroy", this.destroy);
+  }
+}
+
+export const chatAgentsHandler = new ChatAgentsHandler();