feat: realtime config

2025-05-20 04:30:17 +09:00 · 2024-11-07 21:28:23 +08:00 · 2024-11-07 21:28:23 +08:00 · e44ebe3f0e
commit e44ebe3f0e
parent 283caba8ce
7 changed files with 320 additions and 58 deletions
--- a/app/components/chat.tsx
+++ b/app/components/chat.tsx
@ -793,11 +793,13 @@ export function ChatActions(props: {
        )}
      </>
      <div className={styles["chat-input-actions-end"]}>
-        <ChatAction
-          onClick={() => props.setShowChatSidePanel(true)}
-          text={"Realtime Chat"}
-          icon={<HeadphoneIcon />}
-        />
+        {config.realtimeConfig.enable && (
+          <ChatAction
+            onClick={() => props.setShowChatSidePanel(true)}
+            text={"Realtime Chat"}
+            icon={<HeadphoneIcon />}
+          />
+        )}
      </div>
    </div>
  );
--- a/app/components/realtime-chat/realtime-chat.tsx
+++ b/app/components/realtime-chat/realtime-chat.tsx
@ -1,4 +1,3 @@
-import { useDebouncedCallback } from "use-debounce";
 import VoiceIcon from "@/app/icons/voice.svg";
 import VoiceOffIcon from "@/app/icons/voice-off.svg";
 import PowerIcon from "@/app/icons/power.svg";
@ -8,12 +7,7 @@ import clsx from "clsx";

 import { useState, useRef, useEffect } from "react";

-import {
-  useAccessStore,
-  useChatStore,
-  ChatMessage,
-  createMessage,
-} from "@/app/store";
+import { useChatStore, createMessage, useAppConfig } from "@/app/store";

 import { IconButton } from "@/app/components/button";

@ -23,7 +17,6 @@ import {
  RTInputAudioItem,
  RTResponse,
  TurnDetection,
-  Voice,
 } from "rt-client";
 import { AudioHandler } from "@/app/lib/audio";
 import { uploadImage } from "@/app/utils/chat";
@ -39,41 +32,40 @@ export function RealtimeChat({
  onStartVoice,
  onPausedVoice,
 }: RealtimeChatProps) {
-  const currentItemId = useRef<string>("");
-  const currentBotMessage = useRef<ChatMessage | null>();
-  const currentUserMessage = useRef<ChatMessage | null>();
-  const accessStore = useAccessStore.getState();
  const chatStore = useChatStore();
  const session = chatStore.currentSession();
-
+  const config = useAppConfig();
  const [status, setStatus] = useState("");
  const [isRecording, setIsRecording] = useState(false);
  const [isConnected, setIsConnected] = useState(false);
  const [isConnecting, setIsConnecting] = useState(false);
  const [modality, setModality] = useState("audio");
-  const [isAzure, setIsAzure] = useState(false);
-  const [endpoint, setEndpoint] = useState("");
-  const [deployment, setDeployment] = useState("");
  const [useVAD, setUseVAD] = useState(true);
-  const [voice, setVoice] = useState<Voice>("alloy");
-  const [temperature, setTemperature] = useState(0.9);

  const clientRef = useRef<RTClient | null>(null);
  const audioHandlerRef = useRef<AudioHandler | null>(null);
+  const initRef = useRef(false);

-  const apiKey = accessStore.openaiApiKey;
+  const temperature = config.realtimeConfig.temperature;
+  const apiKey = config.realtimeConfig.apiKey;
+  const model = config.realtimeConfig.model;
+  const azure = config.realtimeConfig.provider === "Azure";
+  const azureEndpoint = config.realtimeConfig.azure.endpoint;
+  const azureDeployment = config.realtimeConfig.azure.deployment;
+  const voice = config.realtimeConfig.voice;

  const handleConnect = async () => {
    if (isConnecting) return;
    if (!isConnected) {
      try {
        setIsConnecting(true);
-        clientRef.current = isAzure
-          ? new RTClient(new URL(endpoint), { key: apiKey }, { deployment })
-          : new RTClient(
+        clientRef.current = azure
+          ? new RTClient(
+              new URL(azureEndpoint),
              { key: apiKey },
-              { model: "gpt-4o-realtime-preview-2024-10-01" },
-            );
+              { deployment: azureDeployment },
+            )
+          : new RTClient({ key: apiKey }, { model });
        const modalities: Modality[] =
          modality === "audio" ? ["text", "audio"] : ["text"];
        const turnDetection: TurnDetection = useVAD
@ -191,7 +183,6 @@ export function RealtimeChat({
        const blob = audioHandlerRef.current?.savePlayFile();
        uploadImage(blob!).then((audio_url) => {
          botMessage.audio_url = audio_url;
-          // botMessage.date = new Date().toLocaleString();
          // update text and audio_url
          chatStore.updateTargetSession(session, (session) => {
            session.messages = session.messages.concat();
@ -258,31 +249,32 @@ export function RealtimeChat({
    }
  };

-  useEffect(
-    useDebouncedCallback(() => {
-      const initAudioHandler = async () => {
-        const handler = new AudioHandler();
-        await handler.initialize();
-        audioHandlerRef.current = handler;
-        await handleConnect();
-        await toggleRecording();
-      };
+  useEffect(() => {
+    // 防止重复初始化
+    if (initRef.current) return;
+    initRef.current = true;

-      initAudioHandler().catch((error) => {
-        setStatus(error);
-        console.error(error);
-      });
+    const initAudioHandler = async () => {
+      const handler = new AudioHandler();
+      await handler.initialize();
+      audioHandlerRef.current = handler;
+      await handleConnect();
+      await toggleRecording();
+    };

-      return () => {
-        if (isRecording) {
-          toggleRecording();
-        }
-        audioHandlerRef.current?.close().catch(console.error);
-        disconnect();
-      };
-    }),
-    [],
-  );
+    initAudioHandler().catch((error) => {
+      setStatus(error);
+      console.error(error);
+    });
+
+    return () => {
+      if (isRecording) {
+        toggleRecording();
+      }
+      audioHandlerRef.current?.close().catch(console.error);
+      disconnect();
+    };
+  }, []);

  // update session params
  useEffect(() => {
@ -304,7 +296,7 @@ export function RealtimeChat({
    <div className={styles["realtime-chat"]}>
      <div
        className={clsx(styles["circle-mic"], {
-          [styles["pulse"]]: true,
+          [styles["pulse"]]: isRecording,
        })}
      >
        <div className={styles["icon-center"]}></div>
@ -312,10 +304,11 @@ export function RealtimeChat({
      <div className={styles["bottom-icons"]}>
        <div>
          <IconButton
-            icon={isRecording ? <VoiceOffIcon /> : <VoiceIcon />}
+            icon={isRecording ? <VoiceIcon /> : <VoiceOffIcon />}
            onClick={toggleRecording}
            disabled={!isConnected}
-            type={isRecording ? "danger" : isConnected ? "primary" : null}
+            shadow
+            bordered
          />
        </div>
        <div className={styles["icon-center"]}>{status}</div>
@ -323,7 +316,8 @@ export function RealtimeChat({
          <IconButton
            icon={<PowerIcon />}
            onClick={handleClose}
-            type={isConnecting || isConnected ? "danger" : "primary"}
+            shadow
+            bordered
          />
        </div>
      </div>
--- a/app/components/realtime-chat/realtime-config.tsx
+++ b/app/components/realtime-chat/realtime-config.tsx
@ -0,0 +1,173 @@
+import { RealtimeConfig } from "@/app/store";
+
+import Locale from "@/app/locales";
+import { ListItem, Select, PasswordInput } from "@/app/components/ui-lib";
+
+import { InputRange } from "@/app/components/input-range";
+import { Voice } from "rt-client";
+import { ServiceProvider } from "@/app/constant";
+
+const providers = [ServiceProvider.OpenAI, ServiceProvider.Azure];
+
+const models = ["gpt-4o-realtime-preview-2024-10-01"];
+
+const voice = ["alloy", "shimmer", "echo"];
+
+export function RealtimeConfigList(props: {
+  realtimeConfig: RealtimeConfig;
+  updateConfig: (updater: (config: RealtimeConfig) => void) => void;
+}) {
+  const azureConfigComponent = props.realtimeConfig.provider ===
+    ServiceProvider.Azure && (
+    <>
+      <ListItem
+        title={Locale.Settings.Realtime.Azure.Endpoint.Title}
+        subTitle={Locale.Settings.Realtime.Azure.Endpoint.SubTitle}
+      >
+        <input
+          value={props.realtimeConfig?.azure?.endpoint}
+          type="text"
+          placeholder={Locale.Settings.Realtime.Azure.Endpoint.Title}
+          onChange={(e) => {
+            props.updateConfig(
+              (config) => (config.azure.endpoint = e.currentTarget.value),
+            );
+          }}
+        />
+      </ListItem>
+      <ListItem
+        title={Locale.Settings.Realtime.Azure.Deployment.Title}
+        subTitle={Locale.Settings.Realtime.Azure.Deployment.SubTitle}
+      >
+        <input
+          value={props.realtimeConfig?.azure?.deployment}
+          type="text"
+          placeholder={Locale.Settings.Realtime.Azure.Deployment.Title}
+          onChange={(e) => {
+            props.updateConfig(
+              (config) => (config.azure.deployment = e.currentTarget.value),
+            );
+          }}
+        />
+      </ListItem>
+    </>
+  );
+
+  return (
+    <>
+      <ListItem
+        title={Locale.Settings.Realtime.Enable.Title}
+        subTitle={Locale.Settings.Realtime.Enable.SubTitle}
+      >
+        <input
+          type="checkbox"
+          checked={props.realtimeConfig.enable}
+          onChange={(e) =>
+            props.updateConfig(
+              (config) => (config.enable = e.currentTarget.checked),
+            )
+          }
+        ></input>
+      </ListItem>
+
+      {props.realtimeConfig.enable && (
+        <>
+          <ListItem
+            title={Locale.Settings.Realtime.Provider.Title}
+            subTitle={Locale.Settings.Realtime.Provider.SubTitle}
+          >
+            <Select
+              aria-label={Locale.Settings.Realtime.Provider.Title}
+              value={props.realtimeConfig.provider}
+              onChange={(e) => {
+                props.updateConfig(
+                  (config) =>
+                    (config.provider = e.target.value as ServiceProvider),
+                );
+              }}
+            >
+              {providers.map((v, i) => (
+                <option value={v} key={i}>
+                  {v}
+                </option>
+              ))}
+            </Select>
+          </ListItem>
+          <ListItem
+            title={Locale.Settings.Realtime.Model.Title}
+            subTitle={Locale.Settings.Realtime.Model.SubTitle}
+          >
+            <Select
+              aria-label={Locale.Settings.Realtime.Model.Title}
+              value={props.realtimeConfig.model}
+              onChange={(e) => {
+                props.updateConfig((config) => (config.model = e.target.value));
+              }}
+            >
+              {models.map((v, i) => (
+                <option value={v} key={i}>
+                  {v}
+                </option>
+              ))}
+            </Select>
+          </ListItem>
+          <ListItem
+            title={Locale.Settings.Realtime.ApiKey.Title}
+            subTitle={Locale.Settings.Realtime.ApiKey.SubTitle}
+          >
+            <PasswordInput
+              aria={Locale.Settings.ShowPassword}
+              aria-label={Locale.Settings.Realtime.ApiKey.Title}
+              value={props.realtimeConfig.apiKey}
+              type="text"
+              placeholder={Locale.Settings.Realtime.ApiKey.Placeholder}
+              onChange={(e) => {
+                props.updateConfig(
+                  (config) => (config.apiKey = e.currentTarget.value),
+                );
+              }}
+            />
+          </ListItem>
+          {azureConfigComponent}
+          <ListItem
+            title={Locale.Settings.TTS.Voice.Title}
+            subTitle={Locale.Settings.TTS.Voice.SubTitle}
+          >
+            <Select
+              value={props.realtimeConfig.voice}
+              onChange={(e) => {
+                props.updateConfig(
+                  (config) => (config.voice = e.currentTarget.value as Voice),
+                );
+              }}
+            >
+              {voice.map((v, i) => (
+                <option value={v} key={i}>
+                  {v}
+                </option>
+              ))}
+            </Select>
+          </ListItem>
+          <ListItem
+            title={Locale.Settings.Realtime.Temperature.Title}
+            subTitle={Locale.Settings.Realtime.Temperature.SubTitle}
+          >
+            <InputRange
+              aria={Locale.Settings.Temperature.Title}
+              value={props.realtimeConfig?.temperature?.toFixed(1)}
+              min="0.6"
+              max="1"
+              step="0.1"
+              onChange={(e) => {
+                props.updateConfig(
+                  (config) =>
+                    (config.temperature = e.currentTarget.valueAsNumber),
+                );
+              }}
+            ></InputRange>
+          </ListItem>
+        </>
+      )}
+    </>
+  );
+}
--- a/app/components/settings.tsx
+++ b/app/components/settings.tsx
@ -85,6 +85,7 @@ import { nanoid } from "nanoid";
 import { useMaskStore } from "../store/mask";
 import { ProviderType } from "../utils/cloud";
 import { TTSConfigList } from "./tts-config";
+import { RealtimeConfigList } from "./realtime-chat/realtime-config";

 function EditPromptModal(props: { id: string; onClose: () => void }) {
  const promptStore = usePromptStore();
@ -1799,7 +1800,18 @@ export function Settings() {
        {shouldShowPromptModal && (
          <UserPromptModal onClose={() => setShowPromptModal(false)} />
        )}
-
+        <List>
+          <RealtimeConfigList
+            realtimeConfig={config.realtimeConfig}
+            updateConfig={(updater) => {
+              const realtimeConfig = { ...config.realtimeConfig };
+              updater(realtimeConfig);
+              config.update(
+                (config) => (config.realtimeConfig = realtimeConfig),
+              );
+            }}
+          />
+        </List>
        <List>
          <TTSConfigList
            ttsConfig={config.ttsConfig}
--- a/app/locales/cn.ts
+++ b/app/locales/cn.ts
@ -562,6 +562,39 @@ const cn = {
        SubTitle: "生成语音的速度",
      },
    },
+    Realtime: {
+      Enable: {
+        Title: "实时聊天",
+        SubTitle: "开启实时聊天功能",
+      },
+      Provider: {
+        Title: "模型服务商",
+        SubTitle: "切换不同的服务商",
+      },
+      Model: {
+        Title: "模型",
+        SubTitle: "选择一个模型",
+      },
+      ApiKey: {
+        Title: "API Key",
+        SubTitle: "API Key",
+        Placeholder: "API Key",
+      },
+      Azure: {
+        Endpoint: {
+          Title: "接口地址",
+          SubTitle: "接口地址",
+        },
+        Deployment: {
+          Title: "部署",
+          SubTitle: "Deployment",
+        },
+      },
+      Temperature: {
+        Title: "随机性 (temperature)",
+        SubTitle: "值越大，回复越随机",
+      },
+    },
  },
  Store: {
    DefaultTopic: "新的聊天",
--- a/app/locales/en.ts
+++ b/app/locales/en.ts
@ -570,6 +570,39 @@ const en: LocaleType = {
      },
      Engine: "TTS Engine",
    },
+    Realtime: {
+      Enable: {
+        Title: "Realtime Chat",
+        SubTitle: "Enable realtime chat feature",
+      },
+      Provider: {
+        Title: "Model Provider",
+        SubTitle: "Switch between different providers",
+      },
+      Model: {
+        Title: "Model",
+        SubTitle: "Select a model",
+      },
+      ApiKey: {
+        Title: "API Key",
+        SubTitle: "API Key",
+        Placeholder: "API Key",
+      },
+      Azure: {
+        Endpoint: {
+          Title: "Endpoint",
+          SubTitle: "Endpoint",
+        },
+        Deployment: {
+          Title: "Deployment",
+          SubTitle: "Deployment",
+        },
+      },
+      Temperature: {
+        Title: "Randomness (temperature)",
+        SubTitle: "Higher values result in more random responses",
+      },
+    },
  },
  Store: {
    DefaultTopic: "New Conversation",
--- a/app/store/config.ts
+++ b/app/store/config.ts
@ -15,6 +15,7 @@ import {
  ServiceProvider,
 } from "../constant";
 import { createPersistStore } from "../utils/store";
+import type { Voice } from "rt-client";

 export type ModelType = (typeof DEFAULT_MODELS)[number]["name"];
 export type TTSModelType = (typeof DEFAULT_TTS_MODELS)[number];
@ -90,12 +91,26 @@ export const DEFAULT_CONFIG = {
    voice: DEFAULT_TTS_VOICE,
    speed: 1.0,
  },
+
+  realtimeConfig: {
+    enable: false,
+    provider: "OpenAI" as ServiceProvider,
+    model: "gpt-4o-realtime-preview-2024-10-01",
+    apiKey: "",
+    azure: {
+      endpoint: "",
+      deployment: "",
+    },
+    temperature: 0.9,
+    voice: "alloy" as Voice,
+  },
 };

 export type ChatConfig = typeof DEFAULT_CONFIG;

 export type ModelConfig = ChatConfig["modelConfig"];
 export type TTSConfig = ChatConfig["ttsConfig"];
+export type RealtimeConfig = ChatConfig["realtimeConfig"];

 export function limitNumber(
  x: number,