From b709ee3983ee410981302c2f35e02a89f34ce959 Mon Sep 17 00:00:00 2001 From: EvanWu <850123119@qq.com> Date: Mon, 24 Feb 2025 20:18:07 +0800 Subject: [PATCH 1/8] feat(alibaba): Added alibaba vision model and omni model support --- app/client/api.ts | 5 +++++ app/client/platforms/alibaba.ts | 38 ++++++++++++++++++++++----------- app/constant.ts | 10 ++++++++- app/utils/chat.ts | 22 +++++++++++++++++++ 4 files changed, 62 insertions(+), 13 deletions(-) diff --git a/app/client/api.ts b/app/client/api.ts index 64ac82b2a..f5288593d 100644 --- a/app/client/api.ts +++ b/app/client/api.ts @@ -40,6 +40,11 @@ export interface MultimodalContent { }; } +export interface MultimodalContentForAlibaba { + text?: string; + image?: string; +} + export interface RequestMessage { role: MessageRole; content: string | MultimodalContent[]; diff --git a/app/client/platforms/alibaba.ts b/app/client/platforms/alibaba.ts index 88511768c..4875e5c02 100644 --- a/app/client/platforms/alibaba.ts +++ b/app/client/platforms/alibaba.ts @@ -7,7 +7,10 @@ import { ChatMessageTool, usePluginStore, } from "@/app/store"; -import { streamWithThink } from "@/app/utils/chat"; +import { + preProcessImageContentForAlibabaDashScope, + streamWithThink, +} from "@/app/utils/chat"; import { ChatOptions, getHeaders, @@ -15,12 +18,14 @@ import { LLMModel, SpeechOptions, MultimodalContent, + MultimodalContentForAlibaba, } from "../api"; import { getClientConfig } from "@/app/config/client"; import { getMessageTextContent, getMessageTextContentWithoutThinking, getTimeoutMSByModel, + isVisionModel, } from "@/app/utils"; import { fetch } from "@/app/utils/stream"; @@ -89,14 +94,6 @@ export class QwenApi implements LLMApi { } async chat(options: ChatOptions) { - const messages = options.messages.map((v) => ({ - role: v.role, - content: - v.role === "assistant" - ? getMessageTextContentWithoutThinking(v) - : getMessageTextContent(v), - })); - const modelConfig = { ...useAppConfig.getState().modelConfig, ...useChatStore.getState().currentSession().mask.modelConfig, @@ -105,6 +102,21 @@ export class QwenApi implements LLMApi { }, }; + const visionModel = isVisionModel(options.config.model); + + const messages: ChatOptions["messages"] = []; + for (const v of options.messages) { + const content = ( + visionModel + ? await preProcessImageContentForAlibabaDashScope(v.content) + : v.role === "assistant" + ? getMessageTextContentWithoutThinking(v) + : getMessageTextContent(v) + ) as any; + + messages.push({ role: v.role, content }); + } + const shouldStream = !!options.config.stream; const requestPayload: RequestPayload = { model: modelConfig.model, @@ -129,7 +141,7 @@ export class QwenApi implements LLMApi { "X-DashScope-SSE": shouldStream ? "enable" : "disable", }; - const chatPath = this.path(Alibaba.ChatPath); + const chatPath = this.path(Alibaba.ChatPath(modelConfig.model)); const chatPayload = { method: "POST", body: JSON.stringify(requestPayload), @@ -162,7 +174,7 @@ export class QwenApi implements LLMApi { const json = JSON.parse(text); const choices = json.output.choices as Array<{ message: { - content: string | null; + content: string | null | MultimodalContentForAlibaba[]; tool_calls: ChatMessageTool[]; reasoning_content: string | null; }; @@ -212,7 +224,9 @@ export class QwenApi implements LLMApi { } else if (content && content.length > 0) { return { isThinking: false, - content: content, + content: Array.isArray(content) + ? content.map((item) => item.text).join(",") + : content, }; } diff --git a/app/constant.ts b/app/constant.ts index 50aaf7921..358467c63 100644 --- a/app/constant.ts +++ b/app/constant.ts @@ -221,7 +221,12 @@ export const ByteDance = { export const Alibaba = { ExampleEndpoint: ALIBABA_BASE_URL, - ChatPath: "v1/services/aigc/text-generation/generation", + ChatPath: (modelName: string) => { + if (modelName.includes("vl") || modelName.includes("omni")) { + return "v1/services/aigc/multimodal-generation/generation"; + } + return `v1/services/aigc/text-generation/generation`; + }, }; export const Tencent = { @@ -568,6 +573,9 @@ const alibabaModes = [ "qwen-max-0403", "qwen-max-0107", "qwen-max-longcontext", + "qwen-omni-turbo", + "qwen-vl-plus", + "qwen-vl-max", ]; const tencentModels = [ diff --git a/app/utils/chat.ts b/app/utils/chat.ts index efc496f2c..ecb2fa468 100644 --- a/app/utils/chat.ts +++ b/app/utils/chat.ts @@ -92,6 +92,28 @@ export async function preProcessImageContent( return result; } +export async function preProcessImageContentForAlibabaDashScope( + content: RequestMessage["content"], +) { + if (typeof content === "string") { + return content; + } + const result = []; + for (const part of content) { + if (part?.type == "image_url" && part?.image_url?.url) { + try { + const url = await cacheImageToBase64Image(part?.image_url?.url); + result.push({ image: url }); + } catch (error) { + console.error("Error processing image URL:", error); + } + } else { + result.push({ ...part }); + } + } + return result; +} + const imageCaches: Record = {}; export function cacheImageToBase64Image(imageUrl: string) { if (imageUrl.includes(CACHE_URL_PREFIX)) { From f3154b20a559e0d5b3d0025b13827adb66e1fae0 Mon Sep 17 00:00:00 2001 From: hyiip Date: Tue, 25 Feb 2025 03:55:24 +0800 Subject: [PATCH 2/8] claude 3.7 support --- app/constant.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/constant.ts b/app/constant.ts index 50aaf7921..02ba8dc81 100644 --- a/app/constant.ts +++ b/app/constant.ts @@ -535,6 +535,8 @@ const anthropicModels = [ "claude-3-5-sonnet-20240620", "claude-3-5-sonnet-20241022", "claude-3-5-sonnet-latest", + "claude-3-7-sonnet-20250219", + "claude-3-7-sonnet-latest", ]; const baiduModels = [ From 0a25a1a8cbfde5ba8536afda5624195ab1708cbc Mon Sep 17 00:00:00 2001 From: EvanWu <850123119@qq.com> Date: Tue, 25 Feb 2025 09:22:47 +0800 Subject: [PATCH 3/8] refacto(app/utils/chat.ts)r: optimize function preProcessImageContentBase --- app/utils/chat.ts | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-) diff --git a/app/utils/chat.ts b/app/utils/chat.ts index ecb2fa468..879d3d198 100644 --- a/app/utils/chat.ts +++ b/app/utils/chat.ts @@ -70,8 +70,9 @@ export function compressImage(file: Blob, maxSize: number): Promise { }); } -export async function preProcessImageContent( +export async function preProcessImageContentBase( content: RequestMessage["content"], + transformImageUrl: (url: string) => Promise<{ [key: string]: any }>, ) { if (typeof content === "string") { return content; @@ -81,7 +82,7 @@ export async function preProcessImageContent( if (part?.type == "image_url" && part?.image_url?.url) { try { const url = await cacheImageToBase64Image(part?.image_url?.url); - result.push({ type: part.type, image_url: { url } }); + result.push(await transformImageUrl(url)); } catch (error) { console.error("Error processing image URL:", error); } @@ -92,26 +93,21 @@ export async function preProcessImageContent( return result; } +export async function preProcessImageContent( + content: RequestMessage["content"], +) { + return preProcessImageContentBase(content, async (url) => ({ + type: "image_url", + image_url: { url }, + })); +} + export async function preProcessImageContentForAlibabaDashScope( content: RequestMessage["content"], ) { - if (typeof content === "string") { - return content; - } - const result = []; - for (const part of content) { - if (part?.type == "image_url" && part?.image_url?.url) { - try { - const url = await cacheImageToBase64Image(part?.image_url?.url); - result.push({ image: url }); - } catch (error) { - console.error("Error processing image URL:", error); - } - } else { - result.push({ ...part }); - } - } - return result; + return preProcessImageContentBase(content, async (url) => ({ + image: url, + })); } const imageCaches: Record = {}; From ebcb4db245d3b7b4d34f807c5c7aaa5975ac5330 Mon Sep 17 00:00:00 2001 From: Rex Ng Date: Tue, 25 Feb 2025 14:30:18 +0800 Subject: [PATCH 4/8] Fix: Improve Mistral icon detection and remove redundant code. - Added "codestral" to the list of acceptable names for the Mistral icon, ensuring proper detection. - Removed duplicate `toLowerCase()` calls. --- app/components/emoji.tsx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/components/emoji.tsx b/app/components/emoji.tsx index 1bf39ac1d..31d7f0ac6 100644 --- a/app/components/emoji.tsx +++ b/app/components/emoji.tsx @@ -66,11 +66,11 @@ export function Avatar(props: { model?: ModelType; avatar?: string }) { LlmIcon = BotIconGemma; } else if (modelName.startsWith("claude")) { LlmIcon = BotIconClaude; - } else if (modelName.toLowerCase().includes("llama")) { + } else if (modelName.includes("llama")) { LlmIcon = BotIconMeta; - } else if (modelName.startsWith("mixtral")) { + } else if (modelName.startsWith("mixtral") || modelName.startsWith("codestral")) { LlmIcon = BotIconMistral; - } else if (modelName.toLowerCase().includes("deepseek")) { + } else if (modelName.includes("deepseek")) { LlmIcon = BotIconDeepseek; } else if (modelName.startsWith("moonshot")) { LlmIcon = BotIconMoonshot; @@ -85,7 +85,7 @@ export function Avatar(props: { model?: ModelType; avatar?: string }) { } else if (modelName.startsWith("doubao") || modelName.startsWith("ep-")) { LlmIcon = BotIconDoubao; } else if ( - modelName.toLowerCase().includes("glm") || + modelName.includes("glm") || modelName.startsWith("cogview-") || modelName.startsWith("cogvideox-") ) { From a2c4e468a08cfe7108d30ac0e63fe43c63fb4bef Mon Sep 17 00:00:00 2001 From: EvanWu <850123119@qq.com> Date: Wed, 26 Feb 2025 19:58:32 +0800 Subject: [PATCH 5/8] fix(app/utils/chat.ts): fix type error --- app/utils/chat.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/utils/chat.ts b/app/utils/chat.ts index 879d3d198..cae775512 100644 --- a/app/utils/chat.ts +++ b/app/utils/chat.ts @@ -3,7 +3,7 @@ import { UPLOAD_URL, REQUEST_TIMEOUT_MS, } from "@/app/constant"; -import { RequestMessage } from "@/app/client/api"; +import { MultimodalContent, RequestMessage } from "@/app/client/api"; import Locale from "@/app/locales"; import { EventStreamContentType, @@ -99,7 +99,7 @@ export async function preProcessImageContent( return preProcessImageContentBase(content, async (url) => ({ type: "image_url", image_url: { url }, - })); + })) as Promise; } export async function preProcessImageContentForAlibabaDashScope( From ad6666eeafb38c1faa00ced357187138d7e09bcb Mon Sep 17 00:00:00 2001 From: "Mr. AGI" <102142660+agi-dude@users.noreply.github.com> Date: Fri, 28 Feb 2025 10:47:52 +0500 Subject: [PATCH 6/8] Update README.md --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index fbc087697..93a5289bd 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,6 @@ English / [简体中文](./README_CN.md) [![MacOS][MacOS-image]][download-url] [![Linux][Linux-image]][download-url] -[NextChatAI](https://nextchat.dev/chat?utm_source=readme) / [Web App Demo](https://app.nextchat.dev) / [Desktop App](https://github.com/Yidadaa/ChatGPT-Next-Web/releases) [NextChatAI](https://nextchat.club?utm_source=readme) / [Web App Demo](https://app.nextchat.dev) / [Desktop App](https://github.com/Yidadaa/ChatGPT-Next-Web/releases) / [Discord](https://discord.gg/YCkeafCafC) / [Enterprise Edition](#enterprise-edition) / [Twitter](https://twitter.com/NextChatDev) From 9f0182b55efac275094a36fc6a8487f2f619be91 Mon Sep 17 00:00:00 2001 From: Kadxy <2230318258@qq.com> Date: Fri, 28 Feb 2025 13:52:26 +0800 Subject: [PATCH 7/8] fix: enforce that the first message (excluding system messages) is a user message in the Deepseek API --- app/client/platforms/deepseek.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/app/client/platforms/deepseek.ts b/app/client/platforms/deepseek.ts index b21d24cef..db67a92f0 100644 --- a/app/client/platforms/deepseek.ts +++ b/app/client/platforms/deepseek.ts @@ -75,6 +75,25 @@ export class DeepSeekApi implements LLMApi { } } + // 检测并修复消息顺序，确保除system外的第一个消息是user + const filteredMessages: ChatOptions["messages"] = []; + let hasFoundFirstUser = false; + + for (const msg of messages) { + if (msg.role === "system") { + // Keep all system messages + filteredMessages.push(msg); + } else if (msg.role === "user") { + // User message directly added + filteredMessages.push(msg); + hasFoundFirstUser = true; + } else if (hasFoundFirstUser) { + // After finding the first user message, all subsequent non-system messages are retained. + filteredMessages.push(msg); + } + // If hasFoundFirstUser is false and it is not a system message, it will be skipped. + } + const modelConfig = { ...useAppConfig.getState().modelConfig, ...useChatStore.getState().currentSession().mask.modelConfig, From 2d4180f5be9639fc7a7e050834b1706ac2ee47ee Mon Sep 17 00:00:00 2001 From: Kadxy <2230318258@qq.com> Date: Fri, 28 Feb 2025 13:59:30 +0800 Subject: [PATCH 8/8] fix: update request payload to use filtered messages in Deepseek API --- app/client/platforms/deepseek.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/client/platforms/deepseek.ts b/app/client/platforms/deepseek.ts index db67a92f0..1b38b40cc 100644 --- a/app/client/platforms/deepseek.ts +++ b/app/client/platforms/deepseek.ts @@ -104,7 +104,7 @@ export class DeepSeekApi implements LLMApi { }; const requestPayload: RequestPayload = { - messages, + messages: filteredMessages, stream: options.config.stream, model: modelConfig.model, temperature: modelConfig.temperature,