feat: realtime chat ui

app/components/chat.module.scss

@@ -45,6 +45,14 @@
 .chat-input-actions {
   display: flex;
   flex-wrap: wrap;
+  justify-content: space-between;
+  gap: 5px;
+
+  &-end {
+    display: flex;
+    margin-left: auto;
+    gap: 5px;
+  }
 
   .chat-input-action {
     display: inline-flex;
@@ -62,10 +70,6 @@
     width: var(--icon-width);
     overflow: hidden;
 
-    &:not(:last-child) {
-      margin-right: 5px;
-    }
-
     .text {
       white-space: nowrap;
       padding-left: 5px;
@@ -231,10 +235,12 @@
 
   animation: slide-in ease 0.3s;
 
-  $linear: linear-gradient(to right,
+  $linear: linear-gradient(
-      rgba(0, 0, 0, 0),
+    to right,
-      rgba(0, 0, 0, 1),
+    rgba(0, 0, 0, 0),
-      rgba(0, 0, 0, 0));
+    rgba(0, 0, 0, 1),
+    rgba(0, 0, 0, 0)
+  );
   mask-image: $linear;
 
   @mixin show {
@@ -373,7 +379,7 @@
   }
 }
 
-.chat-message-user>.chat-message-container {
+.chat-message-user > .chat-message-container {
   align-items: flex-end;
 }
 
@@ -443,6 +449,25 @@
   transition: all ease 0.3s;
 }
 
+.chat-message-audio {
+  display: flex;
+  align-items: center;
+  justify-content: space-between;
+  border-radius: 10px;
+  background-color: rgba(0, 0, 0, 0.05);
+  border: var(--border-in-light);
+  position: relative;
+  transition: all ease 0.3s;
+  margin-top: 10px;
+  font-size: 14px;
+  user-select: text;
+  word-break: break-word;
+  box-sizing: border-box;
+  audio {
+    height: 30px; /* 调整高度 */
+  }
+}
+
 .chat-message-item-image {
   width: 100%;
   margin-top: 10px;
@@ -471,23 +496,27 @@
   border: rgba($color: #888, $alpha: 0.2) 1px solid;
 }
 
-
 @media only screen and (max-width: 600px) {
-  $calc-image-width: calc(100vw/3*2/var(--image-count));
+  $calc-image-width: calc(100vw / 3 * 2 / var(--image-count));
 
   .chat-message-item-image-multi {
     width: $calc-image-width;
     height: $calc-image-width;
   }
-  
+
   .chat-message-item-image {
-    max-width: calc(100vw/3*2);
+    max-width: calc(100vw / 3 * 2);
   }
 }
 
 @media screen and (min-width: 600px) {
-  $max-image-width: calc(calc(1200px - var(--sidebar-width))/3*2/var(--image-count));
+  $max-image-width: calc(
-  $image-width: calc(calc(var(--window-width) - var(--sidebar-width))/3*2/var(--image-count));
+    calc(1200px - var(--sidebar-width)) / 3 * 2 / var(--image-count)
+  );
+  $image-width: calc(
+    calc(var(--window-width) - var(--sidebar-width)) / 3 * 2 /
+      var(--image-count)
+  );
 
   .chat-message-item-image-multi {
     width: $image-width;
@@ -497,7 +526,7 @@
   }
 
   .chat-message-item-image {
-    max-width: calc(calc(1200px - var(--sidebar-width))/3*2);
+    max-width: calc(calc(1200px - var(--sidebar-width)) / 3 * 2);
   }
 }
 
@@ -515,7 +544,7 @@
   z-index: 1;
 }
 
-.chat-message-user>.chat-message-container>.chat-message-item {
+.chat-message-user > .chat-message-container > .chat-message-item {
   background-color: var(--second);
 
   &:hover {
@@ -626,7 +655,8 @@
   min-height: 68px;
 }
 
-.chat-input:focus {}
+.chat-input:focus {
+}
 
 .chat-input-send {
   background-color: var(--primary);
@@ -693,4 +723,31 @@
 .shortcut-key span {
   font-size: 12px;
   color: var(--black);
-}
+}
+
+.chat-main {
+  display: flex;
+  height: 100%;
+  width: 100%;
+  position: relative;
+  overflow: hidden;
+  .chat-body-container {
+    height: 100%;
+    display: flex;
+    flex-direction: column;
+    flex: 1;
+    width: 100%;
+  }
+  .chat-side-panel {
+    position: absolute;
+    inset: 0;
+    background: var(--white);
+    overflow: hidden;
+    z-index: 10;
+    transform: translateX(100%);
+    transition: all ease 0.3s;
+    &-show {
+      transform: translateX(0);
+    }
+  }
+}

app/components/realtime-chat/index.ts

@@ -0,0 +1 @@
+export * from "./realtime-chat";

app/components/realtime-chat/realtime-chat.module.scss

@@ -0,0 +1,73 @@
+.realtime-chat {
+  width: 100%;
+  justify-content: center;
+  align-items: center;
+  position: relative;
+  display: flex;
+  flex-direction: column;
+  height: 100%;
+  padding: 20px;
+  box-sizing: border-box;
+  .circle-mic {
+    width: 150px;
+    height: 150px;
+    border-radius: 50%;
+    background: linear-gradient(to bottom right, #a0d8ef, #f0f8ff);
+    display: flex;
+    justify-content: center;
+    align-items: center;
+  }
+  .icon-center {
+    font-size: 24px;
+  }
+
+  .bottom-icons {
+    display: flex;
+    justify-content: space-between;
+    width: 100%;
+    position: absolute;
+    bottom: 20px;
+    box-sizing: border-box;
+    padding: 0 20px;
+  }
+
+  .icon-left,
+  .icon-right {
+    width: 46px;
+    height: 46px;
+    font-size: 36px;
+    background: var(--second);
+    border-radius: 50%;
+    padding: 2px;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    cursor: pointer;
+    &:hover {
+      opacity: 0.8;
+    }
+  }
+
+  &.mobile {
+    display: none;
+  }
+}
+
+.pulse {
+  animation: pulse 1.5s infinite;
+}
+
+@keyframes pulse {
+  0% {
+    transform: scale(1);
+    opacity: 0.7;
+  }
+  50% {
+    transform: scale(1.1);
+    opacity: 1;
+  }
+  100% {
+    transform: scale(1);
+    opacity: 0.7;
+  }
+}

app/components/realtime-chat/realtime-chat.tsx

@@ -0,0 +1,257 @@
+import VoiceIcon from "@/app/icons/voice.svg";
+import VoiceOffIcon from "@/app/icons/voice-off.svg";
+import Close24Icon from "@/app/icons/close-24.svg";
+import styles from "./realtime-chat.module.scss";
+import clsx from "clsx";
+
+import { useState, useRef, useCallback } from "react";
+
+import { useAccessStore, useChatStore, ChatMessage } from "@/app/store";
+
+interface RealtimeChatProps {
+  onClose?: () => void;
+  onStartVoice?: () => void;
+  onPausedVoice?: () => void;
+  sampleRate?: number;
+}
+
+export function RealtimeChat({
+  onClose,
+  onStartVoice,
+  onPausedVoice,
+  sampleRate = 24000,
+}: RealtimeChatProps) {
+  const [isVoicePaused, setIsVoicePaused] = useState(true);
+  const clientRef = useRef<null>(null);
+  const currentItemId = useRef<string>("");
+  const currentBotMessage = useRef<ChatMessage | null>();
+  const currentUserMessage = useRef<ChatMessage | null>();
+  const accessStore = useAccessStore.getState();
+  const chatStore = useChatStore();
+
+  //   useEffect(() => {
+  //     if (
+  //       clientRef.current?.getTurnDetectionType() === "server_vad" &&
+  //       audioData
+  //     ) {
+  //       // console.log("appendInputAudio", audioData);
+  //       // 将录制的16PCM音频发送给openai
+  //       clientRef.current?.appendInputAudio(audioData);
+  //     }
+  //   }, [audioData]);
+
+  //   useEffect(() => {
+  //     console.log("isRecording", isRecording);
+  //     if (!isRecording.current) return;
+  //     if (!clientRef.current) {
+  //       const apiKey = accessStore.openaiApiKey;
+  //       const client = (clientRef.current = new RealtimeClient({
+  //         url: "wss://api.openai.com/v1/realtime",
+  //         apiKey,
+  //         dangerouslyAllowAPIKeyInBrowser: true,
+  //         debug: true,
+  //       }));
+  //       client
+  //         .connect()
+  //         .then(() => {
+  //           // TODO 设置真实的上下文
+  //           client.sendUserMessageContent([
+  //             {
+  //               type: `input_text`,
+  //               text: `Hi`,
+  //               // text: `For testing purposes, I want you to list ten car brands. Number each item, e.g. "one (or whatever number you are one): the item name".`
+  //             },
+  //           ]);
+
+  //           // 配置服务端判断说话人开启还是结束
+  //           client.updateSession({
+  //             turn_detection: { type: "server_vad" },
+  //           });
+
+  //           client.on("realtime.event", (realtimeEvent) => {
+  //             // 调试
+  //             console.log("realtime.event", realtimeEvent);
+  //           });
+
+  //           client.on("conversation.interrupted", async () => {
+  //             if (currentBotMessage.current) {
+  //               stopPlaying();
+  //               try {
+  //                 client.cancelResponse(
+  //                   currentBotMessage.current?.id,
+  //                   currentTime(),
+  //                 );
+  //               } catch (e) {
+  //                 console.error(e);
+  //               }
+  //             }
+  //           });
+  //           client.on("conversation.updated", async (event: any) => {
+  //             // console.log("currentSession", chatStore.currentSession());
+  //             // const items = client.conversation.getItems();
+  //             const content = event?.item?.content?.[0]?.transcript || "";
+  //             const text = event?.item?.content?.[0]?.text || "";
+  //             // console.log(
+  //             //   "conversation.updated",
+  //             //   event,
+  //             //   "content[0]",
+  //             //   event?.item?.content?.[0]?.transcript,
+  //             //   "formatted",
+  //             //   event?.item?.formatted?.transcript,
+  //             //   "content",
+  //             //   content,
+  //             //   "text",
+  //             //   text,
+  //             //   event?.item?.status,
+  //             //   event?.item?.role,
+  //             //   items.length,
+  //             //   items,
+  //             // );
+  //             const { item, delta } = event;
+  //             const { role, id, status, formatted } = item || {};
+  //             if (id && role == "assistant") {
+  //               if (
+  //                 !currentBotMessage.current ||
+  //                 currentBotMessage.current?.id != id
+  //               ) {
+  //                 // create assistant message and save to session
+  //                 currentBotMessage.current = createMessage({ id, role });
+  //                 chatStore.updateCurrentSession((session) => {
+  //                   session.messages = session.messages.concat([
+  //                     currentBotMessage.current!,
+  //                   ]);
+  //                 });
+  //               }
+  //               if (currentBotMessage.current?.id != id) {
+  //                 stopPlaying();
+  //               }
+  //               if (content) {
+  //                 currentBotMessage.current.content = content;
+  //                 chatStore.updateCurrentSession((session) => {
+  //                   session.messages = session.messages.concat();
+  //                 });
+  //               }
+  //               if (delta?.audio) {
+  //                 // typeof delta.audio is Int16Array
+  //                 // 直接播放
+  //                 addInt16PCM(delta.audio);
+  //               }
+  //               // console.log(
+  //               //   "updated try save wavFile",
+  //               //   status,
+  //               //   currentBotMessage.current?.audio_url,
+  //               //   formatted?.audio,
+  //               // );
+  //               if (
+  //                 status == "completed" &&
+  //                 !currentBotMessage.current?.audio_url &&
+  //                 formatted?.audio?.length
+  //               ) {
+  //                 // 转换为wav文件保存 TODO 使用mp3格式会更节省空间
+  //                 const botMessage = currentBotMessage.current;
+  //                 const wavFile = new WavPacker().pack(sampleRate, {
+  //                   bitsPerSample: 16,
+  //                   channelCount: 1,
+  //                   data: formatted?.audio,
+  //                 });
+  //                 // 这里将音频文件放到对象里面wavFile.url可以使用<audio>标签播放
+  //                 item.formatted.file = wavFile;
+  //                 uploadImageRemote(wavFile.blob).then((audio_url) => {
+  //                   botMessage.audio_url = audio_url;
+  //                   chatStore.updateCurrentSession((session) => {
+  //                     session.messages = session.messages.concat();
+  //                   });
+  //                 });
+  //               }
+  //               if (
+  //                 status == "completed" &&
+  //                 !currentBotMessage.current?.content
+  //               ) {
+  //                 chatStore.updateCurrentSession((session) => {
+  //                   session.messages = session.messages.filter(
+  //                     (m) => m.id !== currentBotMessage.current?.id,
+  //                   );
+  //                 });
+  //               }
+  //             }
+  //             if (id && role == "user" && !text) {
+  //               if (
+  //                 !currentUserMessage.current ||
+  //                 currentUserMessage.current?.id != id
+  //               ) {
+  //                 // create assistant message and save to session
+  //                 currentUserMessage.current = createMessage({ id, role });
+  //                 chatStore.updateCurrentSession((session) => {
+  //                   session.messages = session.messages.concat([
+  //                     currentUserMessage.current!,
+  //                   ]);
+  //                 });
+  //               }
+  //               if (content) {
+  //                 // 转换为wav文件保存 TODO 使用mp3格式会更节省空间
+  //                 const userMessage = currentUserMessage.current;
+  //                 const wavFile = new WavPacker().pack(sampleRate, {
+  //                   bitsPerSample: 16,
+  //                   channelCount: 1,
+  //                   data: formatted?.audio,
+  //                 });
+  //                 // 这里将音频文件放到对象里面wavFile.url可以使用<audio>标签播放
+  //                 item.formatted.file = wavFile;
+  //                 uploadImageRemote(wavFile.blob).then((audio_url) => {
+  //                   // update message content
+  //                   userMessage.content = content;
+  //                   // update message audio_url
+  //                   userMessage.audio_url = audio_url;
+  //                   chatStore.updateCurrentSession((session) => {
+  //                     session.messages = session.messages.concat();
+  //                   });
+  //                 });
+  //               }
+  //             }
+  //           });
+  //         })
+  //         .catch((e) => {
+  //           console.error("Error", e);
+  //         });
+  //     }
+  //     return () => {
+  //       stop();
+  //       // TODO close client
+  //       clientRef.current?.disconnect();
+  //     };
+  //   }, [isRecording.current]);
+
+  const handleStartVoice = useCallback(() => {
+    onStartVoice?.();
+    setIsVoicePaused(false);
+  }, []);
+
+  const handlePausedVoice = () => {
+    onPausedVoice?.();
+    setIsVoicePaused(true);
+  };
+
+  return (
+    <div className={styles["realtime-chat"]}>
+      <div
+        className={clsx(styles["circle-mic"], {
+          [styles["pulse"]]: true,
+        })}
+      >
+        <div className={styles["icon-center"]}></div>
+      </div>
+      <div className={styles["bottom-icons"]}>
+        <div className={styles["icon-left"]}>
+          {isVoicePaused ? (
+            <VoiceOffIcon onClick={handleStartVoice} />
+          ) : (
+            <VoiceIcon onClick={handlePausedVoice} />
+          )}
+        </div>
+        <div className={styles["icon-right"]} onClick={onClose}>
+          <Close24Icon />
+        </div>
+      </div>
+    </div>
+  );
+}

app/icons/close-24.svg

@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="24" height="24" viewBox="0 0 48 48" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path d="M8 8L40 40" stroke="#333" stroke-width="4" stroke-linecap="round"
+        stroke-linejoin="round" />
+    <path d="M8 40L40 8" stroke="#333" stroke-width="4" stroke-linecap="round"
+        stroke-linejoin="round" />
+</svg>

app/icons/headphone.svg

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="16" height="16" viewBox="0 0 48 48" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path d="M4 28C4 26.8954 4.89543 26 6 26H10V38H6C4.89543 38 4 37.1046 4 36V28Z" fill="none" />
+    <path d="M38 26H42C43.1046 26 44 26.8954 44 28V36C44 37.1046 43.1046 38 42 38H38V26Z"
+        fill="none" />
+    <path
+        d="M10 36V24C10 16.268 16.268 10 24 10C31.732 10 38 16.268 38 24V36M10 26H6C4.89543 26 4 26.8954 4 28V36C4 37.1046 4.89543 38 6 38H10V26ZM38 26H42C43.1046 26 44 26.8954 44 28V36C44 37.1046 43.1046 38 42 38H38V26Z"
+        stroke="#333" stroke-width="4" stroke-linecap="round" stroke-linejoin="round" />
+    <path d="M16 32H20L22 26L26 38L28 32H32" stroke="#333" stroke-width="4" stroke-linecap="round"
+        stroke-linejoin="round" />
+</svg>

app/icons/voice-off.svg

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="24" height="24" viewBox="0 0 48 48" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <path
+        d="M31 24V11C31 7.13401 27.866 4 24 4C20.134 4 17 7.13401 17 11V24C17 27.866 20.134 31 24 31C27.866 31 31 27.866 31 24Z"
+        stroke="#d0021b" stroke-width="4" stroke-linejoin="round" />
+    <path
+        d="M9 23C9 31.2843 15.7157 38 24 38C25.7532 38 27.4361 37.6992 29 37.1465M39 23C39 25.1333 38.5547 27.1626 37.7519 29"
+        stroke="#d0021b" stroke-width="4" stroke-linecap="round" stroke-linejoin="round" />
+    <path d="M24 38V44" stroke="#d0021b" stroke-width="4" stroke-linecap="round"
+        stroke-linejoin="round" />
+    <path d="M42 42L6 6" stroke="#d0021b" stroke-width="4" stroke-linecap="round"
+        stroke-linejoin="round" />
+</svg>

app/icons/voice.svg

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<svg width="24" height="24" viewBox="0 0 48 48" fill="none" xmlns="http://www.w3.org/2000/svg">
+    <rect x="17" y="4" width="14" height="27" rx="7" fill="none" stroke="#333" stroke-width="4"
+        stroke-linejoin="round" />
+    <path d="M9 23C9 31.2843 15.7157 38 24 38C32.2843 38 39 31.2843 39 23" stroke="#333"
+        stroke-width="4" stroke-linecap="round" stroke-linejoin="round" />
+    <path d="M24 38V44" stroke="#333" stroke-width="4" stroke-linecap="round"
+        stroke-linejoin="round" />
+</svg>

app/store/chat.ts

@@ -52,6 +52,7 @@ export type ChatMessage = RequestMessage & {
   id: string;
   model?: ModelType;
   tools?: ChatMessageTool[];
+  audio_url?: string;
 };
 
 export function createMessage(override: Partial<ChatMessage>): ChatMessage {