From 9f91c2d05c21c7fea604a88a0974679a07293c81 Mon Sep 17 00:00:00 2001 From: suruiqiang Date: Sun, 9 Feb 2025 16:52:46 +0800 Subject: [PATCH 1/9] fix avatar for export message preview and saved image --- app/components/exporter.tsx | 37 ++++++++++++++----------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/app/components/exporter.tsx b/app/components/exporter.tsx index 79ae87be2..69a73062a 100644 --- a/app/components/exporter.tsx +++ b/app/components/exporter.tsx @@ -23,7 +23,6 @@ import CopyIcon from "../icons/copy.svg"; import LoadingIcon from "../icons/three-dots.svg"; import ChatGptIcon from "../icons/chatgpt.png"; import ShareIcon from "../icons/share.svg"; -import BotIcon from "../icons/bot.png"; import DownloadIcon from "../icons/download.svg"; import { useEffect, useMemo, useRef, useState } from "react"; @@ -33,13 +32,13 @@ import dynamic from "next/dynamic"; import NextImage from "next/image"; import { toBlob, toPng } from "html-to-image"; -import { DEFAULT_MASK_AVATAR } from "../store/mask"; import { prettyObject } from "../utils/format"; import { EXPORT_MESSAGE_CLASS_NAME } from "../constant"; import { getClientConfig } from "../config/client"; import { type ClientApi, getClientApi } from "../client/api"; import { getMessageTextContent } from "../utils"; +import { MaskAvatar } from "./mask"; import clsx from "clsx"; const Markdown = dynamic(async () => (await import("./markdown")).Markdown, { @@ -407,22 +406,6 @@ export function PreviewActions(props: { ); } -function ExportAvatar(props: { avatar: string }) { - if (props.avatar === DEFAULT_MASK_AVATAR) { - return ( - bot - ); - } - - return ; -} - export function ImagePreviewer(props: { messages: ChatMessage[]; topic: string; @@ -546,9 +529,12 @@ export function ImagePreviewer(props: { github.com/ChatGPTNextWeb/ChatGPT-Next-Web
- + & - +
@@ -576,9 +562,14 @@ export function ImagePreviewer(props: { key={i} >
- + {m.role === "user" ? ( + + ) : ( + + )}
From 0bfc6480855640032ec3593960b434fc5e1c1de5 Mon Sep 17 00:00:00 2001 From: Shenghang Tsai Date: Sun, 9 Feb 2025 18:47:57 +0800 Subject: [PATCH 2/9] fix model icon on siliconflow --- app/components/emoji.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/components/emoji.tsx b/app/components/emoji.tsx index ecb1c6581..19fb1400e 100644 --- a/app/components/emoji.tsx +++ b/app/components/emoji.tsx @@ -66,11 +66,11 @@ export function Avatar(props: { model?: ModelType; avatar?: string }) { LlmIcon = BotIconGemma; } else if (modelName.startsWith("claude")) { LlmIcon = BotIconClaude; - } else if (modelName.startsWith("llama")) { + } else if (modelName.includes("llama")) { LlmIcon = BotIconMeta; } else if (modelName.startsWith("mixtral")) { LlmIcon = BotIconMistral; - } else if (modelName.startsWith("deepseek")) { + } else if (modelName.includes("deepseek")) { LlmIcon = BotIconDeepseek; } else if (modelName.startsWith("moonshot")) { LlmIcon = BotIconMoonshot; @@ -85,7 +85,7 @@ export function Avatar(props: { model?: ModelType; avatar?: string }) { } else if (modelName.startsWith("doubao") || modelName.startsWith("ep-")) { LlmIcon = BotIconDoubao; } else if ( - modelName.startsWith("glm") || + modelName.includes("glm") || modelName.startsWith("cogview-") || modelName.startsWith("cogvideox-") ) { From 18fa2cc30d96fbb452efd9226db7ca6021cacb3e Mon Sep 17 00:00:00 2001 From: Shenghang Tsai Date: Sun, 9 Feb 2025 18:49:26 +0800 Subject: [PATCH 3/9] fix model icon on siliconflow --- app/components/emoji.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/components/emoji.tsx b/app/components/emoji.tsx index 19fb1400e..1bf39ac1d 100644 --- a/app/components/emoji.tsx +++ b/app/components/emoji.tsx @@ -66,11 +66,11 @@ export function Avatar(props: { model?: ModelType; avatar?: string }) { LlmIcon = BotIconGemma; } else if (modelName.startsWith("claude")) { LlmIcon = BotIconClaude; - } else if (modelName.includes("llama")) { + } else if (modelName.toLowerCase().includes("llama")) { LlmIcon = BotIconMeta; } else if (modelName.startsWith("mixtral")) { LlmIcon = BotIconMistral; - } else if (modelName.includes("deepseek")) { + } else if (modelName.toLowerCase().includes("deepseek")) { LlmIcon = BotIconDeepseek; } else if (modelName.startsWith("moonshot")) { LlmIcon = BotIconMoonshot; @@ -85,7 +85,7 @@ export function Avatar(props: { model?: ModelType; avatar?: string }) { } else if (modelName.startsWith("doubao") || modelName.startsWith("ep-")) { LlmIcon = BotIconDoubao; } else if ( - modelName.includes("glm") || + modelName.toLowerCase().includes("glm") || modelName.startsWith("cogview-") || modelName.startsWith("cogvideox-") ) { From 2137aa65bfaeda33bdbfad7f1ae36bfdde8c9edf Mon Sep 17 00:00:00 2001 From: Shenghang Tsai Date: Mon, 10 Feb 2025 11:03:49 +0800 Subject: [PATCH 4/9] Model listing of SiliconFlow --- app/client/platforms/siliconflow.ts | 44 +++++++++++++++++++++++++++-- app/constant.ts | 1 + 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/app/client/platforms/siliconflow.ts b/app/client/platforms/siliconflow.ts index 1ad316a61..8cf9ad3b1 100644 --- a/app/client/platforms/siliconflow.ts +++ b/app/client/platforms/siliconflow.ts @@ -5,6 +5,7 @@ import { SILICONFLOW_BASE_URL, SiliconFlow, REQUEST_TIMEOUT_MS_FOR_THINKING, + DEFAULT_MODELS, } from "@/app/constant"; import { useAccessStore, @@ -27,10 +28,19 @@ import { getMessageTextContentWithoutThinking, } from "@/app/utils"; import { RequestPayload } from "./openai"; + import { fetch } from "@/app/utils/stream"; +export interface SiliconFlowListModelResponse { + object: string; + data: Array<{ + id: string; + object: string; + root: string; + }>; +} export class SiliconflowApi implements LLMApi { - private disableListModels = true; + private disableListModels = false; path(path: string): string { const accessStore = useAccessStore.getState(); @@ -238,6 +248,36 @@ export class SiliconflowApi implements LLMApi { } async models(): Promise { - return []; + if (this.disableListModels) { + return DEFAULT_MODELS.slice(); + } + + const res = await fetch(this.path(SiliconFlow.ListModelPath), { + method: "GET", + headers: { + ...getHeaders(), + }, + }); + + const resJson = (await res.json()) as SiliconFlowListModelResponse; + const chatModels = resJson.data; + console.log("[Models]", chatModels); + + if (!chatModels) { + return []; + } + + let seq = 1000; //同 Constant.ts 中的排序保持一致 + return chatModels.map((m) => ({ + name: m.id, + available: true, + sorted: seq++, + provider: { + id: "siliconflow", + providerName: "SiliconFlow", + providerType: "siliconflow", + sorted: 14, + }, + })); } } diff --git a/app/constant.ts b/app/constant.ts index 09eec44b6..5d0640d1c 100644 --- a/app/constant.ts +++ b/app/constant.ts @@ -258,6 +258,7 @@ export const ChatGLM = { export const SiliconFlow = { ExampleEndpoint: SILICONFLOW_BASE_URL, ChatPath: "v1/chat/completions", + ListModelPath: "v1/models?&sub_type=chat", }; export const DEFAULT_INPUT_TEMPLATE = `{{input}}`; // input / time / model / lang From 86f86962fb0725b888cee6ebd9eb9f818a0c9cee Mon Sep 17 00:00:00 2001 From: Shenghang Tsai Date: Mon, 10 Feb 2025 13:37:48 +0800 Subject: [PATCH 5/9] Support VLM on SiliconFlow --- app/client/platforms/siliconflow.ts | 8 ++++++-- app/constant.ts | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/app/client/platforms/siliconflow.ts b/app/client/platforms/siliconflow.ts index 1ad316a61..17650a9c6 100644 --- a/app/client/platforms/siliconflow.ts +++ b/app/client/platforms/siliconflow.ts @@ -13,7 +13,7 @@ import { ChatMessageTool, usePluginStore, } from "@/app/store"; -import { streamWithThink } from "@/app/utils/chat"; +import { preProcessImageContent, streamWithThink } from "@/app/utils/chat"; import { ChatOptions, getHeaders, @@ -25,6 +25,7 @@ import { getClientConfig } from "@/app/config/client"; import { getMessageTextContent, getMessageTextContentWithoutThinking, + isVisionModel, } from "@/app/utils"; import { RequestPayload } from "./openai"; import { fetch } from "@/app/utils/stream"; @@ -71,13 +72,16 @@ export class SiliconflowApi implements LLMApi { } async chat(options: ChatOptions) { + const visionModel = isVisionModel(options.config.model); const messages: ChatOptions["messages"] = []; for (const v of options.messages) { if (v.role === "assistant") { const content = getMessageTextContentWithoutThinking(v); messages.push({ role: v.role, content }); } else { - const content = getMessageTextContent(v); + const content = visionModel + ? await preProcessImageContent(v.content) + : getMessageTextContent(v); messages.push({ role: v.role, content }); } } diff --git a/app/constant.ts b/app/constant.ts index 09eec44b6..d9cb62bf9 100644 --- a/app/constant.ts +++ b/app/constant.ts @@ -462,6 +462,7 @@ export const VISION_MODEL_REGEXES = [ /gpt-4-turbo(?!.*preview)/, // Matches "gpt-4-turbo" but not "gpt-4-turbo-preview" /^dall-e-3$/, // Matches exactly "dall-e-3" /glm-4v/, + /vl/i, ]; export const EXCLUDE_VISION_MODEL_REGEXES = [/claude-3-5-haiku-20241022/]; From 98a11e56d2c55d7d89dfc4c8905045781863bf98 Mon Sep 17 00:00:00 2001 From: suruiqiang Date: Tue, 11 Feb 2025 12:46:46 +0800 Subject: [PATCH 6/9] support alibaba and bytedance's reasoning_content --- app/client/platforms/alibaba.ts | 220 ++++++++++++++---------------- app/client/platforms/bytedance.ts | 205 +++++++++++++--------------- 2 files changed, 200 insertions(+), 225 deletions(-) diff --git a/app/client/platforms/alibaba.ts b/app/client/platforms/alibaba.ts index 6fe69e87a..13cb558f9 100644 --- a/app/client/platforms/alibaba.ts +++ b/app/client/platforms/alibaba.ts @@ -5,8 +5,14 @@ import { ALIBABA_BASE_URL, REQUEST_TIMEOUT_MS, } from "@/app/constant"; -import { useAccessStore, useAppConfig, useChatStore } from "@/app/store"; - +import { + useAccessStore, + useAppConfig, + useChatStore, + ChatMessageTool, + usePluginStore, +} from "@/app/store"; +import { streamWithThink } from "@/app/utils/chat"; import { ChatOptions, getHeaders, @@ -15,14 +21,11 @@ import { SpeechOptions, MultimodalContent, } from "../api"; -import Locale from "../../locales"; -import { - EventStreamContentType, - fetchEventSource, -} from "@fortaine/fetch-event-source"; -import { prettyObject } from "@/app/utils/format"; import { getClientConfig } from "@/app/config/client"; -import { getMessageTextContent } from "@/app/utils"; +import { + getMessageTextContent, + getMessageTextContentWithoutThinking, +} from "@/app/utils"; import { fetch } from "@/app/utils/stream"; export interface OpenAIListModelResponse { @@ -92,7 +95,10 @@ export class QwenApi implements LLMApi { async chat(options: ChatOptions) { const messages = options.messages.map((v) => ({ role: v.role, - content: getMessageTextContent(v), + content: + v.role === "assistant" + ? getMessageTextContentWithoutThinking(v) + : getMessageTextContent(v), })); const modelConfig = { @@ -122,15 +128,17 @@ export class QwenApi implements LLMApi { options.onController?.(controller); try { + const headers = { + ...getHeaders(), + "X-DashScope-SSE": shouldStream ? "enable" : "disable", + }; + const chatPath = this.path(Alibaba.ChatPath); const chatPayload = { method: "POST", body: JSON.stringify(requestPayload), signal: controller.signal, - headers: { - ...getHeaders(), - "X-DashScope-SSE": shouldStream ? "enable" : "disable", - }, + headers: headers, }; // make a fetch request @@ -140,116 +148,96 @@ export class QwenApi implements LLMApi { ); if (shouldStream) { - let responseText = ""; - let remainText = ""; - let finished = false; - let responseRes: Response; - - // animate response to make it looks smooth - function animateResponseText() { - if (finished || controller.signal.aborted) { - responseText += remainText; - console.log("[Response Animation] finished"); - if (responseText?.length === 0) { - options.onError?.(new Error("empty response from server")); - } - return; - } - - if (remainText.length > 0) { - const fetchCount = Math.max(1, Math.round(remainText.length / 60)); - const fetchText = remainText.slice(0, fetchCount); - responseText += fetchText; - remainText = remainText.slice(fetchCount); - options.onUpdate?.(responseText, fetchText); - } - - requestAnimationFrame(animateResponseText); - } - - // start animaion - animateResponseText(); - - const finish = () => { - if (!finished) { - finished = true; - options.onFinish(responseText + remainText, responseRes); - } - }; - - controller.signal.onabort = finish; - - fetchEventSource(chatPath, { - fetch: fetch as any, - ...chatPayload, - async onopen(res) { - clearTimeout(requestTimeoutId); - const contentType = res.headers.get("content-type"); - console.log( - "[Alibaba] request response content type: ", - contentType, - ); - responseRes = res; - - if (contentType?.startsWith("text/plain")) { - responseText = await res.clone().text(); - return finish(); + const [tools, funcs] = usePluginStore + .getState() + .getAsTools( + useChatStore.getState().currentSession().mask?.plugin || [], + ); + return streamWithThink( + chatPath, + requestPayload, + headers, + tools as any, + funcs, + controller, + // parseSSE + (text: string, runTools: ChatMessageTool[]) => { + // console.log("parseSSE", text, runTools); + const json = JSON.parse(text); + const choices = json.output.choices as Array<{ + message: { + content: string | null; + tool_calls: ChatMessageTool[]; + reasoning_content: string | null; + }; + }>; + const tool_calls = choices[0]?.message?.tool_calls; + if (tool_calls?.length > 0) { + const index = tool_calls[0]?.index; + const id = tool_calls[0]?.id; + const args = tool_calls[0]?.function?.arguments; + if (id) { + runTools.push({ + id, + type: tool_calls[0]?.type, + function: { + name: tool_calls[0]?.function?.name as string, + arguments: args, + }, + }); + } else { + // @ts-ignore + runTools[index]["function"]["arguments"] += args; + } } + const reasoning = choices[0]?.message?.reasoning_content; + const content = choices[0]?.message?.content; + // Skip if both content and reasoning_content are empty or null if ( - !res.ok || - !res.headers - .get("content-type") - ?.startsWith(EventStreamContentType) || - res.status !== 200 + (!reasoning || reasoning.trim().length === 0) && + (!content || content.trim().length === 0) ) { - const responseTexts = [responseText]; - let extraInfo = await res.clone().text(); - try { - const resJson = await res.clone().json(); - extraInfo = prettyObject(resJson); - } catch {} - - if (res.status === 401) { - responseTexts.push(Locale.Error.Unauthorized); - } - - if (extraInfo) { - responseTexts.push(extraInfo); - } - - responseText = responseTexts.join("\n\n"); - - return finish(); + return { + isThinking: false, + content: "", + }; } - }, - onmessage(msg) { - if (msg.data === "[DONE]" || finished) { - return finish(); - } - const text = msg.data; - try { - const json = JSON.parse(text); - const choices = json.output.choices as Array<{ - message: { content: string }; - }>; - const delta = choices[0]?.message?.content; - if (delta) { - remainText += delta; - } - } catch (e) { - console.error("[Request] parse error", text, msg); + + if (reasoning && reasoning.trim().length > 0) { + return { + isThinking: true, + content: reasoning, + }; + } else if (content && content.trim().length > 0) { + return { + isThinking: false, + content: content, + }; } + + return { + isThinking: false, + content: "", + }; }, - onclose() { - finish(); + // processToolMessage, include tool_calls message and tool call results + ( + requestPayload: RequestPayload, + toolCallMessage: any, + toolCallResult: any[], + ) => { + // @ts-ignore + requestPayload?.messages?.splice( + // @ts-ignore + requestPayload?.messages?.length, + 0, + toolCallMessage, + ...toolCallResult, + ); }, - onerror(e) { - options.onError?.(e); - throw e; - }, - openWhenHidden: true, - }); + options, + ); } else { const res = await fetch(chatPath, chatPayload); clearTimeout(requestTimeoutId); diff --git a/app/client/platforms/bytedance.ts b/app/client/platforms/bytedance.ts index c2f128128..5d7ddebeb 100644 --- a/app/client/platforms/bytedance.ts +++ b/app/client/platforms/bytedance.ts @@ -5,7 +5,13 @@ import { BYTEDANCE_BASE_URL, REQUEST_TIMEOUT_MS, } from "@/app/constant"; -import { useAccessStore, useAppConfig, useChatStore } from "@/app/store"; +import { + useAccessStore, + useAppConfig, + useChatStore, + ChatMessageTool, + usePluginStore, +} from "@/app/store"; import { ChatOptions, @@ -15,14 +21,11 @@ import { MultimodalContent, SpeechOptions, } from "../api"; -import Locale from "../../locales"; -import { - EventStreamContentType, - fetchEventSource, -} from "@fortaine/fetch-event-source"; -import { prettyObject } from "@/app/utils/format"; + +import { streamWithThink } from "@/app/utils/chat"; import { getClientConfig } from "@/app/config/client"; import { preProcessImageContent } from "@/app/utils/chat"; +import { getMessageTextContentWithoutThinking } from "@/app/utils"; import { fetch } from "@/app/utils/stream"; export interface OpenAIListModelResponse { @@ -86,7 +89,10 @@ export class DoubaoApi implements LLMApi { async chat(options: ChatOptions) { const messages: ChatOptions["messages"] = []; for (const v of options.messages) { - const content = await preProcessImageContent(v.content); + const content = + v.role === "assistant" + ? getMessageTextContentWithoutThinking(v) + : await preProcessImageContent(v.content); messages.push({ role: v.role, content }); } @@ -128,115 +134,96 @@ export class DoubaoApi implements LLMApi { ); if (shouldStream) { - let responseText = ""; - let remainText = ""; - let finished = false; - let responseRes: Response; - - // animate response to make it looks smooth - function animateResponseText() { - if (finished || controller.signal.aborted) { - responseText += remainText; - console.log("[Response Animation] finished"); - if (responseText?.length === 0) { - options.onError?.(new Error("empty response from server")); - } - return; - } - - if (remainText.length > 0) { - const fetchCount = Math.max(1, Math.round(remainText.length / 60)); - const fetchText = remainText.slice(0, fetchCount); - responseText += fetchText; - remainText = remainText.slice(fetchCount); - options.onUpdate?.(responseText, fetchText); - } - - requestAnimationFrame(animateResponseText); - } - - // start animaion - animateResponseText(); - - const finish = () => { - if (!finished) { - finished = true; - options.onFinish(responseText + remainText, responseRes); - } - }; - - controller.signal.onabort = finish; - - fetchEventSource(chatPath, { - fetch: fetch as any, - ...chatPayload, - async onopen(res) { - clearTimeout(requestTimeoutId); - const contentType = res.headers.get("content-type"); - console.log( - "[ByteDance] request response content type: ", - contentType, - ); - responseRes = res; - if (contentType?.startsWith("text/plain")) { - responseText = await res.clone().text(); - return finish(); + const [tools, funcs] = usePluginStore + .getState() + .getAsTools( + useChatStore.getState().currentSession().mask?.plugin || [], + ); + return streamWithThink( + chatPath, + requestPayload, + getHeaders(), + tools as any, + funcs, + controller, + // parseSSE + (text: string, runTools: ChatMessageTool[]) => { + // console.log("parseSSE", text, runTools); + const json = JSON.parse(text); + const choices = json.choices as Array<{ + delta: { + content: string | null; + tool_calls: ChatMessageTool[]; + reasoning_content: string | null; + }; + }>; + const tool_calls = choices[0]?.delta?.tool_calls; + if (tool_calls?.length > 0) { + const index = tool_calls[0]?.index; + const id = tool_calls[0]?.id; + const args = tool_calls[0]?.function?.arguments; + if (id) { + runTools.push({ + id, + type: tool_calls[0]?.type, + function: { + name: tool_calls[0]?.function?.name as string, + arguments: args, + }, + }); + } else { + // @ts-ignore + runTools[index]["function"]["arguments"] += args; + } } + const reasoning = choices[0]?.delta?.reasoning_content; + const content = choices[0]?.delta?.content; + // Skip if both content and reasoning_content are empty or null if ( - !res.ok || - !res.headers - .get("content-type") - ?.startsWith(EventStreamContentType) || - res.status !== 200 + (!reasoning || reasoning.trim().length === 0) && + (!content || content.trim().length === 0) ) { - const responseTexts = [responseText]; - let extraInfo = await res.clone().text(); - try { - const resJson = await res.clone().json(); - extraInfo = prettyObject(resJson); - } catch {} - - if (res.status === 401) { - responseTexts.push(Locale.Error.Unauthorized); - } - - if (extraInfo) { - responseTexts.push(extraInfo); - } - - responseText = responseTexts.join("\n\n"); - - return finish(); + return { + isThinking: false, + content: "", + }; } - }, - onmessage(msg) { - if (msg.data === "[DONE]" || finished) { - return finish(); - } - const text = msg.data; - try { - const json = JSON.parse(text); - const choices = json.choices as Array<{ - delta: { content: string }; - }>; - const delta = choices[0]?.delta?.content; - if (delta) { - remainText += delta; - } - } catch (e) { - console.error("[Request] parse error", text, msg); + + if (reasoning && reasoning.trim().length > 0) { + return { + isThinking: true, + content: reasoning, + }; + } else if (content && content.trim().length > 0) { + return { + isThinking: false, + content: content, + }; } + + return { + isThinking: false, + content: "", + }; }, - onclose() { - finish(); + // processToolMessage, include tool_calls message and tool call results + ( + requestPayload: RequestPayload, + toolCallMessage: any, + toolCallResult: any[], + ) => { + // @ts-ignore + requestPayload?.messages?.splice( + // @ts-ignore + requestPayload?.messages?.length, + 0, + toolCallMessage, + ...toolCallResult, + ); }, - onerror(e) { - options.onError?.(e); - throw e; - }, - openWhenHidden: true, - }); + options, + ); } else { const res = await fetch(chatPath, chatPayload); clearTimeout(requestTimeoutId); From b0758cccde8709af7fa31aed8c019029c97be82b Mon Sep 17 00:00:00 2001 From: suruiqiang Date: Tue, 11 Feb 2025 16:08:30 +0800 Subject: [PATCH 7/9] optimization --- app/client/platforms/alibaba.ts | 10 ++++++---- app/client/platforms/bytedance.ts | 11 ++++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/app/client/platforms/alibaba.ts b/app/client/platforms/alibaba.ts index 13cb558f9..44dbd847a 100644 --- a/app/client/platforms/alibaba.ts +++ b/app/client/platforms/alibaba.ts @@ -171,6 +171,9 @@ export class QwenApi implements LLMApi { reasoning_content: string | null; }; }>; + + if (!choices?.length) return { isThinking: false, content: "" }; + const tool_calls = choices[0]?.message?.tool_calls; if (tool_calls?.length > 0) { const index = tool_calls[0]?.index; @@ -190,6 +193,7 @@ export class QwenApi implements LLMApi { runTools[index]["function"]["arguments"] += args; } } + const reasoning = choices[0]?.message?.reasoning_content; const content = choices[0]?.message?.content; @@ -227,10 +231,8 @@ export class QwenApi implements LLMApi { toolCallMessage: any, toolCallResult: any[], ) => { - // @ts-ignore - requestPayload?.messages?.splice( - // @ts-ignore - requestPayload?.messages?.length, + requestPayload?.input?.messages?.splice( + requestPayload?.input?.messages?.length, 0, toolCallMessage, ...toolCallResult, diff --git a/app/client/platforms/bytedance.ts b/app/client/platforms/bytedance.ts index 5d7ddebeb..5e2e63f58 100644 --- a/app/client/platforms/bytedance.ts +++ b/app/client/platforms/bytedance.ts @@ -37,7 +37,7 @@ export interface OpenAIListModelResponse { }>; } -interface RequestPayload { +interface RequestPayloadForByteDance { messages: { role: "system" | "user" | "assistant"; content: string | MultimodalContent[]; @@ -105,7 +105,7 @@ export class DoubaoApi implements LLMApi { }; const shouldStream = !!options.config.stream; - const requestPayload: RequestPayload = { + const requestPayload: RequestPayloadForByteDance = { messages, stream: shouldStream, model: modelConfig.model, @@ -157,6 +157,9 @@ export class DoubaoApi implements LLMApi { reasoning_content: string | null; }; }>; + + if (!choices?.length) return { isThinking: false, content: "" }; + const tool_calls = choices[0]?.delta?.tool_calls; if (tool_calls?.length > 0) { const index = tool_calls[0]?.index; @@ -209,13 +212,11 @@ export class DoubaoApi implements LLMApi { }, // processToolMessage, include tool_calls message and tool call results ( - requestPayload: RequestPayload, + requestPayload: RequestPayloadForByteDance, toolCallMessage: any, toolCallResult: any[], ) => { - // @ts-ignore requestPayload?.messages?.splice( - // @ts-ignore requestPayload?.messages?.length, 0, toolCallMessage, From 97142583224faa28e7cdd43eba75b77828f280af Mon Sep 17 00:00:00 2001 From: suruiqiang Date: Tue, 11 Feb 2025 18:57:16 +0800 Subject: [PATCH 8/9] support deepseek-r1@OpenAI's reasoning_content, parse from stream --- app/client/platforms/openai.ts | 40 +++++++++++++++++++++++++++++++--- app/utils/chat.ts | 18 +++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/app/client/platforms/openai.ts b/app/client/platforms/openai.ts index fbe533cad..9d43c8161 100644 --- a/app/client/platforms/openai.ts +++ b/app/client/platforms/openai.ts @@ -22,7 +22,7 @@ import { preProcessImageContent, uploadImage, base64Image2Blob, - stream, + streamWithThink, } from "@/app/utils/chat"; import { cloudflareAIGatewayUrl } from "@/app/utils/cloudflare"; import { ModelSize, DalleQuality, DalleStyle } from "@/app/typing"; @@ -294,7 +294,7 @@ export class ChatGPTApi implements LLMApi { useChatStore.getState().currentSession().mask?.plugin || [], ); // console.log("getAsTools", tools, funcs); - stream( + streamWithThink( chatPath, requestPayload, getHeaders(), @@ -309,8 +309,12 @@ export class ChatGPTApi implements LLMApi { delta: { content: string; tool_calls: ChatMessageTool[]; + reasoning_content: string | null; }; }>; + + if (!choices?.length) return { isThinking: false, content: "" }; + const tool_calls = choices[0]?.delta?.tool_calls; if (tool_calls?.length > 0) { const id = tool_calls[0]?.id; @@ -330,7 +334,37 @@ export class ChatGPTApi implements LLMApi { runTools[index]["function"]["arguments"] += args; } } - return choices[0]?.delta?.content; + + const reasoning = choices[0]?.delta?.reasoning_content; + const content = choices[0]?.delta?.content; + + // Skip if both content and reasoning_content are empty or null + if ( + (!reasoning || reasoning.trim().length === 0) && + (!content || content.trim().length === 0) + ) { + return { + isThinking: false, + content: "", + }; + } + + if (reasoning && reasoning.trim().length > 0) { + return { + isThinking: true, + content: reasoning, + }; + } else if (content && content.trim().length > 0) { + return { + isThinking: false, + content: content, + }; + } + + return { + isThinking: false, + content: "", + }; }, // processToolMessage, include tool_calls message and tool call results ( diff --git a/app/utils/chat.ts b/app/utils/chat.ts index b77955e6e..efc496f2c 100644 --- a/app/utils/chat.ts +++ b/app/utils/chat.ts @@ -400,6 +400,7 @@ export function streamWithThink( let responseRes: Response; let isInThinkingMode = false; let lastIsThinking = false; + let lastIsThinkingTagged = false; //between and tags // animate response to make it looks smooth function animateResponseText() { @@ -579,6 +580,23 @@ export function streamWithThink( if (!chunk?.content || chunk.content.length === 0) { return; } + + // deal with and tags start + if (!chunk.isThinking) { + if (chunk.content.startsWith("")) { + chunk.isThinking = true; + chunk.content = chunk.content.slice(7).trim(); + lastIsThinkingTagged = true; + } else if (chunk.content.endsWith("")) { + chunk.isThinking = false; + chunk.content = chunk.content.slice(0, -8).trim(); + lastIsThinkingTagged = false; + } else if (lastIsThinkingTagged) { + chunk.isThinking = true; + } + } + // deal with and tags start + // Check if thinking mode changed const isThinkingChanged = lastIsThinking !== chunk.isThinking; lastIsThinking = chunk.isThinking; From 476d946f961a551ffedc7734dcce28faa7dc30fe Mon Sep 17 00:00:00 2001 From: suruiqiang Date: Wed, 12 Feb 2025 17:49:54 +0800 Subject: [PATCH 9/9] fix bug (trim eats space or \n mistakenly), optimize timeout by model --- app/client/platforms/alibaba.ts | 18 +++++++----------- app/client/platforms/baidu.ts | 11 +++-------- app/client/platforms/bytedance.ts | 22 ++++++++++------------ app/client/platforms/deepseek.ts | 25 +++++++------------------ app/client/platforms/glm.ts | 15 +++++++-------- app/client/platforms/google.ts | 10 +++------- app/client/platforms/openai.ts | 14 ++++++-------- app/client/platforms/siliconflow.ts | 10 +++------- app/client/platforms/tencent.ts | 10 +++++++--- app/client/platforms/xai.ts | 5 +++-- app/utils.ts | 20 +++++++++++++++++++- 11 files changed, 75 insertions(+), 85 deletions(-) diff --git a/app/client/platforms/alibaba.ts b/app/client/platforms/alibaba.ts index 44dbd847a..88511768c 100644 --- a/app/client/platforms/alibaba.ts +++ b/app/client/platforms/alibaba.ts @@ -1,10 +1,5 @@ "use client"; -import { - ApiPath, - Alibaba, - ALIBABA_BASE_URL, - REQUEST_TIMEOUT_MS, -} from "@/app/constant"; +import { ApiPath, Alibaba, ALIBABA_BASE_URL } from "@/app/constant"; import { useAccessStore, useAppConfig, @@ -25,6 +20,7 @@ import { getClientConfig } from "@/app/config/client"; import { getMessageTextContent, getMessageTextContentWithoutThinking, + getTimeoutMSByModel, } from "@/app/utils"; import { fetch } from "@/app/utils/stream"; @@ -144,7 +140,7 @@ export class QwenApi implements LLMApi { // make a fetch request const requestTimeoutId = setTimeout( () => controller.abort(), - REQUEST_TIMEOUT_MS, + getTimeoutMSByModel(options.config.model), ); if (shouldStream) { @@ -199,8 +195,8 @@ export class QwenApi implements LLMApi { // Skip if both content and reasoning_content are empty or null if ( - (!reasoning || reasoning.trim().length === 0) && - (!content || content.trim().length === 0) + (!reasoning || reasoning.length === 0) && + (!content || content.length === 0) ) { return { isThinking: false, @@ -208,12 +204,12 @@ export class QwenApi implements LLMApi { }; } - if (reasoning && reasoning.trim().length > 0) { + if (reasoning && reasoning.length > 0) { return { isThinking: true, content: reasoning, }; - } else if (content && content.trim().length > 0) { + } else if (content && content.length > 0) { return { isThinking: false, content: content, diff --git a/app/client/platforms/baidu.ts b/app/client/platforms/baidu.ts index 9e8c2f139..dc990db41 100644 --- a/app/client/platforms/baidu.ts +++ b/app/client/platforms/baidu.ts @@ -1,10 +1,5 @@ "use client"; -import { - ApiPath, - Baidu, - BAIDU_BASE_URL, - REQUEST_TIMEOUT_MS, -} from "@/app/constant"; +import { ApiPath, Baidu, BAIDU_BASE_URL } from "@/app/constant"; import { useAccessStore, useAppConfig, useChatStore } from "@/app/store"; import { getAccessToken } from "@/app/utils/baidu"; @@ -23,7 +18,7 @@ import { } from "@fortaine/fetch-event-source"; import { prettyObject } from "@/app/utils/format"; import { getClientConfig } from "@/app/config/client"; -import { getMessageTextContent } from "@/app/utils"; +import { getMessageTextContent, getTimeoutMSByModel } from "@/app/utils"; import { fetch } from "@/app/utils/stream"; export interface OpenAIListModelResponse { @@ -155,7 +150,7 @@ export class ErnieApi implements LLMApi { // make a fetch request const requestTimeoutId = setTimeout( () => controller.abort(), - REQUEST_TIMEOUT_MS, + getTimeoutMSByModel(options.config.model), ); if (shouldStream) { diff --git a/app/client/platforms/bytedance.ts b/app/client/platforms/bytedance.ts index 5e2e63f58..f9524cba2 100644 --- a/app/client/platforms/bytedance.ts +++ b/app/client/platforms/bytedance.ts @@ -1,10 +1,5 @@ "use client"; -import { - ApiPath, - ByteDance, - BYTEDANCE_BASE_URL, - REQUEST_TIMEOUT_MS, -} from "@/app/constant"; +import { ApiPath, ByteDance, BYTEDANCE_BASE_URL } from "@/app/constant"; import { useAccessStore, useAppConfig, @@ -25,7 +20,10 @@ import { import { streamWithThink } from "@/app/utils/chat"; import { getClientConfig } from "@/app/config/client"; import { preProcessImageContent } from "@/app/utils/chat"; -import { getMessageTextContentWithoutThinking } from "@/app/utils"; +import { + getMessageTextContentWithoutThinking, + getTimeoutMSByModel, +} from "@/app/utils"; import { fetch } from "@/app/utils/stream"; export interface OpenAIListModelResponse { @@ -130,7 +128,7 @@ export class DoubaoApi implements LLMApi { // make a fetch request const requestTimeoutId = setTimeout( () => controller.abort(), - REQUEST_TIMEOUT_MS, + getTimeoutMSByModel(options.config.model), ); if (shouldStream) { @@ -184,8 +182,8 @@ export class DoubaoApi implements LLMApi { // Skip if both content and reasoning_content are empty or null if ( - (!reasoning || reasoning.trim().length === 0) && - (!content || content.trim().length === 0) + (!reasoning || reasoning.length === 0) && + (!content || content.length === 0) ) { return { isThinking: false, @@ -193,12 +191,12 @@ export class DoubaoApi implements LLMApi { }; } - if (reasoning && reasoning.trim().length > 0) { + if (reasoning && reasoning.length > 0) { return { isThinking: true, content: reasoning, }; - } else if (content && content.trim().length > 0) { + } else if (content && content.length > 0) { return { isThinking: false, content: content, diff --git a/app/client/platforms/deepseek.ts b/app/client/platforms/deepseek.ts index c436ae61d..b21d24cef 100644 --- a/app/client/platforms/deepseek.ts +++ b/app/client/platforms/deepseek.ts @@ -1,12 +1,6 @@ "use client"; // azure and openai, using same models. so using same LLMApi. -import { - ApiPath, - DEEPSEEK_BASE_URL, - DeepSeek, - REQUEST_TIMEOUT_MS, - REQUEST_TIMEOUT_MS_FOR_THINKING, -} from "@/app/constant"; +import { ApiPath, DEEPSEEK_BASE_URL, DeepSeek } from "@/app/constant"; import { useAccessStore, useAppConfig, @@ -26,6 +20,7 @@ import { getClientConfig } from "@/app/config/client"; import { getMessageTextContent, getMessageTextContentWithoutThinking, + getTimeoutMSByModel, } from "@/app/utils"; import { RequestPayload } from "./openai"; import { fetch } from "@/app/utils/stream"; @@ -116,16 +111,10 @@ export class DeepSeekApi implements LLMApi { headers: getHeaders(), }; - // console.log(chatPayload); - - const isR1 = - options.config.model.endsWith("-reasoner") || - options.config.model.endsWith("-r1"); - // make a fetch request const requestTimeoutId = setTimeout( () => controller.abort(), - isR1 ? REQUEST_TIMEOUT_MS_FOR_THINKING : REQUEST_TIMEOUT_MS, + getTimeoutMSByModel(options.config.model), ); if (shouldStream) { @@ -176,8 +165,8 @@ export class DeepSeekApi implements LLMApi { // Skip if both content and reasoning_content are empty or null if ( - (!reasoning || reasoning.trim().length === 0) && - (!content || content.trim().length === 0) + (!reasoning || reasoning.length === 0) && + (!content || content.length === 0) ) { return { isThinking: false, @@ -185,12 +174,12 @@ export class DeepSeekApi implements LLMApi { }; } - if (reasoning && reasoning.trim().length > 0) { + if (reasoning && reasoning.length > 0) { return { isThinking: true, content: reasoning, }; - } else if (content && content.trim().length > 0) { + } else if (content && content.length > 0) { return { isThinking: false, content: content, diff --git a/app/client/platforms/glm.ts b/app/client/platforms/glm.ts index a8d1869e3..98b10277d 100644 --- a/app/client/platforms/glm.ts +++ b/app/client/platforms/glm.ts @@ -1,10 +1,5 @@ "use client"; -import { - ApiPath, - CHATGLM_BASE_URL, - ChatGLM, - REQUEST_TIMEOUT_MS, -} from "@/app/constant"; +import { ApiPath, CHATGLM_BASE_URL, ChatGLM } from "@/app/constant"; import { useAccessStore, useAppConfig, @@ -21,7 +16,11 @@ import { SpeechOptions, } from "../api"; import { getClientConfig } from "@/app/config/client"; -import { getMessageTextContent, isVisionModel } from "@/app/utils"; +import { + getMessageTextContent, + isVisionModel, + getTimeoutMSByModel, +} from "@/app/utils"; import { RequestPayload } from "./openai"; import { fetch } from "@/app/utils/stream"; import { preProcessImageContent } from "@/app/utils/chat"; @@ -191,7 +190,7 @@ export class ChatGLMApi implements LLMApi { const requestTimeoutId = setTimeout( () => controller.abort(), - REQUEST_TIMEOUT_MS, + getTimeoutMSByModel(options.config.model), ); if (modelType === "image" || modelType === "video") { diff --git a/app/client/platforms/google.ts b/app/client/platforms/google.ts index 1e593dd42..654f0e3e4 100644 --- a/app/client/platforms/google.ts +++ b/app/client/platforms/google.ts @@ -1,9 +1,4 @@ -import { - ApiPath, - Google, - REQUEST_TIMEOUT_MS, - REQUEST_TIMEOUT_MS_FOR_THINKING, -} from "@/app/constant"; +import { ApiPath, Google } from "@/app/constant"; import { ChatOptions, getHeaders, @@ -27,6 +22,7 @@ import { getMessageTextContent, getMessageImages, isVisionModel, + getTimeoutMSByModel, } from "@/app/utils"; import { preProcessImageContent } from "@/app/utils/chat"; import { nanoid } from "nanoid"; @@ -206,7 +202,7 @@ export class GeminiProApi implements LLMApi { // make a fetch request const requestTimeoutId = setTimeout( () => controller.abort(), - isThinking ? REQUEST_TIMEOUT_MS_FOR_THINKING : REQUEST_TIMEOUT_MS, + getTimeoutMSByModel(options.config.model), ); if (shouldStream) { diff --git a/app/client/platforms/openai.ts b/app/client/platforms/openai.ts index 9d43c8161..c6f3fc425 100644 --- a/app/client/platforms/openai.ts +++ b/app/client/platforms/openai.ts @@ -8,7 +8,6 @@ import { Azure, REQUEST_TIMEOUT_MS, ServiceProvider, - REQUEST_TIMEOUT_MS_FOR_THINKING, } from "@/app/constant"; import { ChatMessageTool, @@ -42,6 +41,7 @@ import { getMessageTextContent, isVisionModel, isDalle3 as _isDalle3, + getTimeoutMSByModel, } from "@/app/utils"; import { fetch } from "@/app/utils/stream"; @@ -340,8 +340,8 @@ export class ChatGPTApi implements LLMApi { // Skip if both content and reasoning_content are empty or null if ( - (!reasoning || reasoning.trim().length === 0) && - (!content || content.trim().length === 0) + (!reasoning || reasoning.length === 0) && + (!content || content.length === 0) ) { return { isThinking: false, @@ -349,12 +349,12 @@ export class ChatGPTApi implements LLMApi { }; } - if (reasoning && reasoning.trim().length > 0) { + if (reasoning && reasoning.length > 0) { return { isThinking: true, content: reasoning, }; - } else if (content && content.trim().length > 0) { + } else if (content && content.length > 0) { return { isThinking: false, content: content, @@ -396,9 +396,7 @@ export class ChatGPTApi implements LLMApi { // make a fetch request const requestTimeoutId = setTimeout( () => controller.abort(), - isDalle3 || isO1OrO3 - ? REQUEST_TIMEOUT_MS_FOR_THINKING - : REQUEST_TIMEOUT_MS, // dalle3 using b64_json is slow. + getTimeoutMSByModel(options.config.model), ); const res = await fetch(chatPath, chatPayload); diff --git a/app/client/platforms/siliconflow.ts b/app/client/platforms/siliconflow.ts index 1ad316a61..92c0261c4 100644 --- a/app/client/platforms/siliconflow.ts +++ b/app/client/platforms/siliconflow.ts @@ -1,11 +1,6 @@ "use client"; // azure and openai, using same models. so using same LLMApi. -import { - ApiPath, - SILICONFLOW_BASE_URL, - SiliconFlow, - REQUEST_TIMEOUT_MS_FOR_THINKING, -} from "@/app/constant"; +import { ApiPath, SILICONFLOW_BASE_URL, SiliconFlow } from "@/app/constant"; import { useAccessStore, useAppConfig, @@ -25,6 +20,7 @@ import { getClientConfig } from "@/app/config/client"; import { getMessageTextContent, getMessageTextContentWithoutThinking, + getTimeoutMSByModel, } from "@/app/utils"; import { RequestPayload } from "./openai"; import { fetch } from "@/app/utils/stream"; @@ -123,7 +119,7 @@ export class SiliconflowApi implements LLMApi { // Use extended timeout for thinking models as they typically require more processing time const requestTimeoutId = setTimeout( () => controller.abort(), - REQUEST_TIMEOUT_MS_FOR_THINKING, + getTimeoutMSByModel(options.config.model), ); if (shouldStream) { diff --git a/app/client/platforms/tencent.ts b/app/client/platforms/tencent.ts index 580844a5b..8adeb1b3e 100644 --- a/app/client/platforms/tencent.ts +++ b/app/client/platforms/tencent.ts @@ -1,5 +1,5 @@ "use client"; -import { ApiPath, TENCENT_BASE_URL, REQUEST_TIMEOUT_MS } from "@/app/constant"; +import { ApiPath, TENCENT_BASE_URL } from "@/app/constant"; import { useAccessStore, useAppConfig, useChatStore } from "@/app/store"; import { @@ -17,7 +17,11 @@ import { } from "@fortaine/fetch-event-source"; import { prettyObject } from "@/app/utils/format"; import { getClientConfig } from "@/app/config/client"; -import { getMessageTextContent, isVisionModel } from "@/app/utils"; +import { + getMessageTextContent, + isVisionModel, + getTimeoutMSByModel, +} from "@/app/utils"; import mapKeys from "lodash-es/mapKeys"; import mapValues from "lodash-es/mapValues"; import isArray from "lodash-es/isArray"; @@ -135,7 +139,7 @@ export class HunyuanApi implements LLMApi { // make a fetch request const requestTimeoutId = setTimeout( () => controller.abort(), - REQUEST_TIMEOUT_MS, + getTimeoutMSByModel(options.config.model), ); if (shouldStream) { diff --git a/app/client/platforms/xai.ts b/app/client/platforms/xai.ts index 8c41c2d98..830ad4778 100644 --- a/app/client/platforms/xai.ts +++ b/app/client/platforms/xai.ts @@ -1,6 +1,6 @@ "use client"; // azure and openai, using same models. so using same LLMApi. -import { ApiPath, XAI_BASE_URL, XAI, REQUEST_TIMEOUT_MS } from "@/app/constant"; +import { ApiPath, XAI_BASE_URL, XAI } from "@/app/constant"; import { useAccessStore, useAppConfig, @@ -17,6 +17,7 @@ import { SpeechOptions, } from "../api"; import { getClientConfig } from "@/app/config/client"; +import { getTimeoutMSByModel } from "@/app/utils"; import { preProcessImageContent } from "@/app/utils/chat"; import { RequestPayload } from "./openai"; import { fetch } from "@/app/utils/stream"; @@ -103,7 +104,7 @@ export class XAIApi implements LLMApi { // make a fetch request const requestTimeoutId = setTimeout( () => controller.abort(), - REQUEST_TIMEOUT_MS, + getTimeoutMSByModel(options.config.model), ); if (shouldStream) { diff --git a/app/utils.ts b/app/utils.ts index f23378019..6183e03b0 100644 --- a/app/utils.ts +++ b/app/utils.ts @@ -2,7 +2,11 @@ import { useEffect, useState } from "react"; import { showToast } from "./components/ui-lib"; import Locale from "./locales"; import { RequestMessage } from "./client/api"; -import { ServiceProvider } from "./constant"; +import { + REQUEST_TIMEOUT_MS, + REQUEST_TIMEOUT_MS_FOR_THINKING, + ServiceProvider, +} from "./constant"; // import { fetch as tauriFetch, ResponseType } from "@tauri-apps/api/http"; import { fetch as tauriStreamFetch } from "./utils/stream"; import { VISION_MODEL_REGEXES, EXCLUDE_VISION_MODEL_REGEXES } from "./constant"; @@ -292,6 +296,20 @@ export function isDalle3(model: string) { return "dall-e-3" === model; } +export function getTimeoutMSByModel(model: string) { + model = model.toLowerCase(); + if ( + model.startsWith("dall-e") || + model.startsWith("dalle") || + model.startsWith("o1") || + model.startsWith("o3") || + model.includes("deepseek-r") || + model.includes("-thinking") + ) + return REQUEST_TIMEOUT_MS_FOR_THINKING; + return REQUEST_TIMEOUT_MS; +} + export function getModelSizes(model: string): ModelSize[] { if (isDalle3(model)) { return ["1024x1024", "1792x1024", "1024x1792"];