mirror of
https://github.com/ChatGPTNextWeb/ChatGPT-Next-Web.git
synced 2025-05-19 20:20:16 +09:00
Merge 121b0954b0 into 3809375694
This commit is contained in:
glay
GitHub
commit
767657a7f7
@ -76,6 +76,12 @@ ANTHROPIC_URL=
|
||||
### (optional)
|
||||
WHITE_WEBDAV_ENDPOINTS=
|
||||
|
||||
|
||||
### bedrock (optional)
|
||||
AWS_REGION=
|
||||
AWS_ACCESS_KEY=AKIA
|
||||
AWS_SECRET_KEY=
|
||||
|
||||
### siliconflow Api key (optional)
|
||||
SILICONFLOW_API_KEY=
|
||||
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { ApiPath } from "@/app/constant";
|
||||
import { NextRequest } from "next/server";
|
||||
import { handle as openaiHandler } from "../../openai";
|
||||
import { handle as bedrockHandler } from "../../bedrock";
|
||||
import { handle as azureHandler } from "../../azure";
|
||||
import { handle as googleHandler } from "../../google";
|
||||
import { handle as anthropicHandler } from "../../anthropic";
|
||||
@ -23,12 +24,15 @@ async function handle(
|
||||
const apiPath = `/api/${params.provider}`;
|
||||
console.log(`[${params.provider} Route] params `, params);
|
||||
switch (apiPath) {
|
||||
case ApiPath.Bedrock:
|
||||
return bedrockHandler(req, { params });
|
||||
case ApiPath.Azure:
|
||||
return azureHandler(req, { params });
|
||||
case ApiPath.Google:
|
||||
return googleHandler(req, { params });
|
||||
case ApiPath.Anthropic:
|
||||
return anthropicHandler(req, { params });
|
||||
|
||||
case ApiPath.Baidu:
|
||||
return baiduHandler(req, { params });
|
||||
case ApiPath.ByteDance:
|
||||
|
||||
@ -52,7 +52,6 @@ export function auth(req: NextRequest, modelProvider: ModelProvider) {
|
||||
msg: "you are not allowed to access with your own api key",
|
||||
};
|
||||
}
|
||||
|
||||
// if user does not provide an api key, inject system api key
|
||||
if (!apiKey) {
|
||||
const serverConfig = getServerSideConfig();
|
||||
@ -101,6 +100,14 @@ export function auth(req: NextRequest, modelProvider: ModelProvider) {
|
||||
case ModelProvider.ChatGLM:
|
||||
systemApiKey = serverConfig.chatglmApiKey;
|
||||
break;
|
||||
case ModelProvider.Bedrock:
|
||||
systemApiKey =
|
||||
serverConfig.awsRegion +
|
||||
":" +
|
||||
serverConfig.awsAccessKey +
|
||||
":" +
|
||||
serverConfig.awsSecretKey;
|
||||
break;
|
||||
case ModelProvider.SiliconFlow:
|
||||
systemApiKey = serverConfig.siliconFlowApiKey;
|
||||
break;
|
||||
|
||||
177
app/api/bedrock.ts
Normal file
177
app/api/bedrock.ts
Normal file
@ -0,0 +1,177 @@
|
||||
import { NextRequest, NextResponse } from "next/server";
|
||||
import { auth } from "./auth";
|
||||
import {
|
||||
sign,
|
||||
decrypt,
|
||||
getBedrockEndpoint,
|
||||
BedrockCredentials,
|
||||
} from "../utils/aws";
|
||||
import { getServerSideConfig } from "../config/server";
|
||||
import { ModelProvider } from "../constant";
|
||||
import { prettyObject } from "../utils/format";
|
||||
|
||||
const ALLOWED_PATH = new Set(["chat", "models"]);
|
||||
|
||||
async function getBedrockCredentials(
|
||||
req: NextRequest,
|
||||
): Promise<BedrockCredentials> {
|
||||
// Get AWS credentials from server config first
|
||||
const config = getServerSideConfig();
|
||||
let awsRegion = config.awsRegion;
|
||||
let awsAccessKey = config.awsAccessKey;
|
||||
let awsSecretKey = config.awsSecretKey;
|
||||
|
||||
// If server-side credentials are not available, parse from Authorization header
|
||||
if (!awsRegion || !awsAccessKey || !awsSecretKey) {
|
||||
const authHeader = req.headers.get("Authorization");
|
||||
if (!authHeader || !authHeader.startsWith("Bearer ")) {
|
||||
throw new Error("Missing or invalid Authorization header");
|
||||
}
|
||||
|
||||
const [_, credentials] = authHeader.split("Bearer ");
|
||||
const [encryptedRegion, encryptedAccessKey, encryptedSecretKey] =
|
||||
credentials.split(":");
|
||||
|
||||
if (!encryptedRegion || !encryptedAccessKey || !encryptedSecretKey) {
|
||||
throw new Error("Invalid Authorization header format");
|
||||
}
|
||||
const encryptionKey = req.headers.get("XEncryptionKey") || "";
|
||||
// Decrypt the credentials
|
||||
[awsRegion, awsAccessKey, awsSecretKey] = await Promise.all([
|
||||
decrypt(encryptedRegion, encryptionKey),
|
||||
decrypt(encryptedAccessKey, encryptionKey),
|
||||
decrypt(encryptedSecretKey, encryptionKey),
|
||||
]);
|
||||
|
||||
if (!awsRegion || !awsAccessKey || !awsSecretKey) {
|
||||
throw new Error(
|
||||
"Failed to decrypt AWS credentials. Please ensure ENCRYPTION_KEY is set correctly.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
region: awsRegion,
|
||||
accessKeyId: awsAccessKey,
|
||||
secretAccessKey: awsSecretKey,
|
||||
};
|
||||
}
|
||||
|
||||
async function requestBedrock(req: NextRequest) {
|
||||
const controller = new AbortController();
|
||||
const timeoutId = setTimeout(() => controller.abort(), 10 * 60 * 1000);
|
||||
|
||||
try {
|
||||
// Get credentials and model info
|
||||
const credentials = await getBedrockCredentials(req);
|
||||
const modelId = req.headers.get("XModelID");
|
||||
const shouldStream = req.headers.get("ShouldStream") !== "false";
|
||||
|
||||
if (!modelId) {
|
||||
throw new Error("Missing model ID");
|
||||
}
|
||||
|
||||
// Parse and validate request body
|
||||
const bodyText = await req.clone().text();
|
||||
if (!bodyText) {
|
||||
throw new Error("Request body is empty");
|
||||
}
|
||||
let bodyJson;
|
||||
try {
|
||||
bodyJson = JSON.parse(bodyText);
|
||||
} catch (e) {
|
||||
throw new Error(`Invalid JSON in request body: ${e}`);
|
||||
}
|
||||
console.log("[Bedrock Request] Initiating request");
|
||||
// Get endpoint and prepare request
|
||||
const endpoint = getBedrockEndpoint(
|
||||
credentials.region,
|
||||
modelId,
|
||||
shouldStream,
|
||||
);
|
||||
const requestBody: any = {
|
||||
...bodyJson,
|
||||
};
|
||||
// Sign request
|
||||
const headers = await sign({
|
||||
method: "POST",
|
||||
url: endpoint,
|
||||
region: credentials.region,
|
||||
accessKeyId: credentials.accessKeyId,
|
||||
secretAccessKey: credentials.secretAccessKey,
|
||||
body: JSON.stringify(requestBody),
|
||||
service: "bedrock",
|
||||
isStreaming: shouldStream,
|
||||
});
|
||||
|
||||
// Make request to AWS Bedrock
|
||||
// console.log(
|
||||
// "[Bedrock Request] Final Body:",
|
||||
// JSON.stringify(requestBody, null, 2),
|
||||
// );
|
||||
const res = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: JSON.stringify(requestBody),
|
||||
redirect: "manual",
|
||||
// @ts-ignore
|
||||
duplex: "half",
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
const error = await res.text();
|
||||
console.error("[Bedrock Error] Request failed with status:", res.status);
|
||||
try {
|
||||
const errorJson = JSON.parse(error);
|
||||
throw new Error(errorJson.message || error);
|
||||
} catch {
|
||||
throw new Error(
|
||||
`Bedrock request failed with status ${res.status}: ${
|
||||
error || "No error message"
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (!res.body) {
|
||||
console.error("[Bedrock Error] Empty response body");
|
||||
throw new Error(
|
||||
"Empty response from Bedrock. Please check AWS credentials and permissions.",
|
||||
);
|
||||
}
|
||||
return res;
|
||||
} catch (e) {
|
||||
console.error("[Bedrock Request Error]:", e);
|
||||
throw e;
|
||||
} finally {
|
||||
clearTimeout(timeoutId);
|
||||
}
|
||||
}
|
||||
|
||||
export async function handle(
|
||||
req: NextRequest,
|
||||
{ params }: { params: { path: string[] } },
|
||||
) {
|
||||
const subpath = params.path.join("/");
|
||||
if (!ALLOWED_PATH.has(subpath)) {
|
||||
return NextResponse.json(
|
||||
{ error: true, msg: "you are not allowed to request " + subpath },
|
||||
{ status: 403 },
|
||||
);
|
||||
}
|
||||
|
||||
const authResult = auth(req, ModelProvider.Bedrock);
|
||||
if (authResult.error) {
|
||||
return NextResponse.json(authResult, {
|
||||
status: 401,
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
return await requestBedrock(req);
|
||||
} catch (e) {
|
||||
console.error("Handler error:", e);
|
||||
return NextResponse.json(prettyObject(e));
|
||||
}
|
||||
}
|
||||
@ -23,8 +23,10 @@ import { SparkApi } from "./platforms/iflytek";
|
||||
import { DeepSeekApi } from "./platforms/deepseek";
|
||||
import { XAIApi } from "./platforms/xai";
|
||||
import { ChatGLMApi } from "./platforms/glm";
|
||||
import { BedrockApi } from "./platforms/bedrock";
|
||||
import { SiliconflowApi } from "./platforms/siliconflow";
|
||||
|
||||
|
||||
export const ROLES = ["system", "user", "assistant"] as const;
|
||||
export type MessageRole = (typeof ROLES)[number];
|
||||
|
||||
@ -137,6 +139,9 @@ export class ClientApi {
|
||||
|
||||
constructor(provider: ModelProvider = ModelProvider.GPT) {
|
||||
switch (provider) {
|
||||
case ModelProvider.Bedrock:
|
||||
this.llm = new BedrockApi();
|
||||
break;
|
||||
case ModelProvider.GeminiPro:
|
||||
this.llm = new GeminiProApi();
|
||||
break;
|
||||
@ -252,6 +257,7 @@ export function getHeaders(ignoreHeaders: boolean = false) {
|
||||
|
||||
function getConfig() {
|
||||
const modelConfig = chatStore.currentSession().mask.modelConfig;
|
||||
const isBedrock = modelConfig.providerName === ServiceProvider.Bedrock;
|
||||
const isGoogle = modelConfig.providerName === ServiceProvider.Google;
|
||||
const isAzure = modelConfig.providerName === ServiceProvider.Azure;
|
||||
const isAnthropic = modelConfig.providerName === ServiceProvider.Anthropic;
|
||||
@ -292,6 +298,7 @@ export function getHeaders(ignoreHeaders: boolean = false) {
|
||||
: ""
|
||||
: accessStore.openaiApiKey;
|
||||
return {
|
||||
isBedrock,
|
||||
isGoogle,
|
||||
isAzure,
|
||||
isAnthropic,
|
||||
@ -320,6 +327,7 @@ export function getHeaders(ignoreHeaders: boolean = false) {
|
||||
}
|
||||
|
||||
const {
|
||||
isBedrock,
|
||||
isGoogle,
|
||||
isAzure,
|
||||
isAnthropic,
|
||||
@ -340,17 +348,23 @@ export function getHeaders(ignoreHeaders: boolean = false) {
|
||||
|
||||
const authHeader = getAuthHeader();
|
||||
|
||||
const bearerToken = getBearerToken(
|
||||
apiKey,
|
||||
isAzure || isAnthropic || isGoogle,
|
||||
);
|
||||
|
||||
if (bearerToken) {
|
||||
headers[authHeader] = bearerToken;
|
||||
} else if (isEnabledAccessControl && validString(accessStore.accessCode)) {
|
||||
headers["Authorization"] = getBearerToken(
|
||||
ACCESS_CODE_PREFIX + accessStore.accessCode,
|
||||
if (isBedrock) {
|
||||
if (apiKey) {
|
||||
headers[authHeader] = getBearerToken(apiKey);
|
||||
}
|
||||
} else {
|
||||
const bearerToken = getBearerToken(
|
||||
apiKey,
|
||||
isAzure || isAnthropic || isGoogle,
|
||||
);
|
||||
|
||||
if (bearerToken) {
|
||||
headers[authHeader] = bearerToken;
|
||||
} else if (isEnabledAccessControl && validString(accessStore.accessCode)) {
|
||||
headers["Authorization"] = getBearerToken(
|
||||
ACCESS_CODE_PREFIX + accessStore.accessCode,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return headers;
|
||||
@ -358,6 +372,8 @@ export function getHeaders(ignoreHeaders: boolean = false) {
|
||||
|
||||
export function getClientApi(provider: ServiceProvider): ClientApi {
|
||||
switch (provider) {
|
||||
case ServiceProvider.Bedrock:
|
||||
return new ClientApi(ModelProvider.Bedrock);
|
||||
case ServiceProvider.Google:
|
||||
return new ClientApi(ModelProvider.GeminiPro);
|
||||
case ServiceProvider.Anthropic:
|
||||
|
||||
859
app/client/platforms/bedrock.ts
Normal file
859
app/client/platforms/bedrock.ts
Normal file
@ -0,0 +1,859 @@
|
||||
"use client";
|
||||
import { ChatOptions, getHeaders, LLMApi, SpeechOptions } from "../api";
|
||||
import {
|
||||
useAppConfig,
|
||||
usePluginStore,
|
||||
useChatStore,
|
||||
useAccessStore,
|
||||
ChatMessageTool,
|
||||
} from "@/app/store";
|
||||
import { preProcessImageContent } from "@/app/utils/chat";
|
||||
import { getMessageTextContent, isVisionModel } from "@/app/utils";
|
||||
import { ApiPath, BEDROCK_BASE_URL, REQUEST_TIMEOUT_MS } from "@/app/constant";
|
||||
import { getClientConfig } from "@/app/config/client";
|
||||
import {
|
||||
extractMessage,
|
||||
processMessage,
|
||||
processChunks,
|
||||
parseEventData,
|
||||
sign,
|
||||
} from "@/app/utils/aws";
|
||||
import { prettyObject } from "@/app/utils/format";
|
||||
import Locale from "@/app/locales";
|
||||
import { encrypt } from "@/app/utils/aws";
|
||||
|
||||
const ClaudeMapper = {
|
||||
assistant: "assistant",
|
||||
user: "user",
|
||||
system: "user",
|
||||
} as const;
|
||||
|
||||
const MistralMapper = {
|
||||
system: "system",
|
||||
user: "user",
|
||||
assistant: "assistant",
|
||||
} as const;
|
||||
type MistralRole = keyof typeof MistralMapper;
|
||||
|
||||
interface Tool {
|
||||
function?: {
|
||||
name?: string;
|
||||
description?: string;
|
||||
parameters?: any;
|
||||
};
|
||||
}
|
||||
const isApp = !!getClientConfig()?.isApp;
|
||||
// const isApp = true;
|
||||
async function getBedrockHeaders(
|
||||
modelId: string,
|
||||
chatPath: string,
|
||||
finalRequestBody: any,
|
||||
shouldStream: boolean,
|
||||
): Promise<Record<string, string>> {
|
||||
const accessStore = useAccessStore.getState();
|
||||
const bedrockHeaders = isApp
|
||||
? await sign({
|
||||
method: "POST",
|
||||
url: chatPath,
|
||||
region: accessStore.awsRegion,
|
||||
accessKeyId: accessStore.awsAccessKey,
|
||||
secretAccessKey: accessStore.awsSecretKey,
|
||||
body: finalRequestBody,
|
||||
service: "bedrock",
|
||||
headers: {},
|
||||
isStreaming: shouldStream,
|
||||
})
|
||||
: getHeaders();
|
||||
|
||||
if (!isApp) {
|
||||
const { awsRegion, awsAccessKey, awsSecretKey, encryptionKey } =
|
||||
accessStore;
|
||||
|
||||
const bedrockHeadersConfig = {
|
||||
XModelID: modelId,
|
||||
XEncryptionKey: encryptionKey,
|
||||
ShouldStream: String(shouldStream),
|
||||
Authorization: await createAuthHeader(
|
||||
awsRegion,
|
||||
awsAccessKey,
|
||||
awsSecretKey,
|
||||
encryptionKey,
|
||||
),
|
||||
};
|
||||
|
||||
Object.assign(bedrockHeaders, bedrockHeadersConfig);
|
||||
}
|
||||
|
||||
return bedrockHeaders;
|
||||
}
|
||||
|
||||
// Helper function to create Authorization header
|
||||
async function createAuthHeader(
|
||||
region: string,
|
||||
accessKey: string,
|
||||
secretKey: string,
|
||||
encryptionKey: string,
|
||||
): Promise<string> {
|
||||
const encryptedValues = await Promise.all([
|
||||
encrypt(region, encryptionKey),
|
||||
encrypt(accessKey, encryptionKey),
|
||||
encrypt(secretKey, encryptionKey),
|
||||
]);
|
||||
|
||||
return `Bearer ${encryptedValues.join(":")}`;
|
||||
}
|
||||
|
||||
export class BedrockApi implements LLMApi {
|
||||
speech(options: SpeechOptions): Promise<ArrayBuffer> {
|
||||
throw new Error("Speech not implemented for Bedrock.");
|
||||
}
|
||||
|
||||
formatRequestBody(messages: ChatOptions["messages"], modelConfig: any) {
|
||||
const model = modelConfig.model;
|
||||
const visionModel = isVisionModel(modelConfig.model);
|
||||
|
||||
// Get tools if available
|
||||
const [tools] = usePluginStore
|
||||
.getState()
|
||||
.getAsTools(useChatStore.getState().currentSession().mask?.plugin || []);
|
||||
|
||||
const toolsArray = (tools as Tool[]) || [];
|
||||
|
||||
// Handle Nova models
|
||||
if (model.includes("amazon.nova")) {
|
||||
// Extract system message if present
|
||||
const systemMessage = messages.find((m) => m.role === "system");
|
||||
const conversationMessages = messages.filter((m) => m.role !== "system");
|
||||
|
||||
const requestBody: any = {
|
||||
schemaVersion: "messages-v1",
|
||||
messages: conversationMessages.map((message) => {
|
||||
const content = Array.isArray(message.content)
|
||||
? message.content
|
||||
: [{ text: getMessageTextContent(message) }];
|
||||
|
||||
return {
|
||||
role: message.role,
|
||||
content: content.map((item: any) => {
|
||||
// Handle text content
|
||||
if (item.text || typeof item === "string") {
|
||||
return { text: item.text || item };
|
||||
}
|
||||
// Handle image content
|
||||
if (item.image_url?.url) {
|
||||
const { url = "" } = item.image_url;
|
||||
const colonIndex = url.indexOf(":");
|
||||
const semicolonIndex = url.indexOf(";");
|
||||
const comma = url.indexOf(",");
|
||||
|
||||
// Extract format from mime type
|
||||
const mimeType = url.slice(colonIndex + 1, semicolonIndex);
|
||||
const format = mimeType.split("/")[1];
|
||||
const data = url.slice(comma + 1);
|
||||
|
||||
return {
|
||||
image: {
|
||||
format,
|
||||
source: {
|
||||
bytes: data,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
return item;
|
||||
}),
|
||||
};
|
||||
}),
|
||||
inferenceConfig: {
|
||||
temperature: modelConfig.temperature || 0.7,
|
||||
top_p: modelConfig.top_p || 0.9,
|
||||
top_k: modelConfig.top_k || 50,
|
||||
max_new_tokens: modelConfig.max_tokens || 1000,
|
||||
stopSequences: modelConfig.stop || [],
|
||||
},
|
||||
};
|
||||
|
||||
// Add system message if present
|
||||
if (systemMessage) {
|
||||
requestBody.system = [
|
||||
{
|
||||
text: getMessageTextContent(systemMessage),
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
// Add tools if available - exact Nova format
|
||||
if (toolsArray.length > 0) {
|
||||
requestBody.toolConfig = {
|
||||
tools: toolsArray.map((tool) => ({
|
||||
toolSpec: {
|
||||
name: tool?.function?.name || "",
|
||||
description: tool?.function?.description || "",
|
||||
inputSchema: {
|
||||
json: {
|
||||
type: "object",
|
||||
properties: tool?.function?.parameters?.properties || {},
|
||||
required: tool?.function?.parameters?.required || [],
|
||||
},
|
||||
},
|
||||
},
|
||||
})),
|
||||
toolChoice: { auto: {} },
|
||||
};
|
||||
}
|
||||
|
||||
return requestBody;
|
||||
}
|
||||
|
||||
// Handle Titan models
|
||||
if (model.startsWith("amazon.titan")) {
|
||||
const inputText = messages
|
||||
.map((message) => {
|
||||
return `${message.role}: ${getMessageTextContent(message)}`;
|
||||
})
|
||||
.join("\n\n");
|
||||
|
||||
return {
|
||||
inputText,
|
||||
textGenerationConfig: {
|
||||
maxTokenCount: modelConfig.max_tokens,
|
||||
temperature: modelConfig.temperature,
|
||||
stopSequences: [],
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// Handle LLaMA models
|
||||
if (model.includes("meta.llama")) {
|
||||
let prompt = "<|begin_of_text|>";
|
||||
|
||||
// Extract system message if present
|
||||
const systemMessage = messages.find((m) => m.role === "system");
|
||||
if (systemMessage) {
|
||||
prompt += `<|start_header_id|>system<|end_header_id|>\n${getMessageTextContent(
|
||||
systemMessage,
|
||||
)}<|eot_id|>`;
|
||||
}
|
||||
|
||||
// Format the conversation
|
||||
const conversationMessages = messages.filter((m) => m.role !== "system");
|
||||
for (const message of conversationMessages) {
|
||||
const role = message.role === "assistant" ? "assistant" : "user";
|
||||
const content = getMessageTextContent(message);
|
||||
prompt += `<|start_header_id|>${role}<|end_header_id|>\n${content}<|eot_id|>`;
|
||||
}
|
||||
|
||||
// Add the final assistant header to prompt completion
|
||||
prompt += "<|start_header_id|>assistant<|end_header_id|>";
|
||||
|
||||
return {
|
||||
prompt,
|
||||
max_gen_len: modelConfig.max_tokens || 512,
|
||||
temperature: modelConfig.temperature || 0.7,
|
||||
top_p: modelConfig.top_p || 0.9,
|
||||
};
|
||||
}
|
||||
|
||||
// Handle Mistral models
|
||||
if (model.includes("mistral.mistral")) {
|
||||
const formattedMessages = messages.map((message) => ({
|
||||
role: MistralMapper[message.role as MistralRole] || "user",
|
||||
content: getMessageTextContent(message),
|
||||
}));
|
||||
|
||||
const requestBody: any = {
|
||||
messages: formattedMessages,
|
||||
max_tokens: modelConfig.max_tokens || 4096,
|
||||
temperature: modelConfig.temperature || 0.7,
|
||||
top_p: modelConfig.top_p || 0.9,
|
||||
};
|
||||
|
||||
// Add tools if available
|
||||
if (toolsArray.length > 0) {
|
||||
requestBody.tool_choice = "auto";
|
||||
requestBody.tools = toolsArray.map((tool) => ({
|
||||
type: "function",
|
||||
function: {
|
||||
name: tool?.function?.name,
|
||||
description: tool?.function?.description,
|
||||
parameters: tool?.function?.parameters,
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
return requestBody;
|
||||
}
|
||||
|
||||
// Handle Claude models
|
||||
const keys = ["system", "user"];
|
||||
// roles must alternate between "user" and "assistant" in claude, so add a fake assistant message between two user messages
|
||||
for (let i = 0; i < messages.length - 1; i++) {
|
||||
const message = messages[i];
|
||||
const nextMessage = messages[i + 1];
|
||||
|
||||
if (keys.includes(message.role) && keys.includes(nextMessage.role)) {
|
||||
messages[i] = [
|
||||
message,
|
||||
{
|
||||
role: "assistant",
|
||||
content: ";",
|
||||
},
|
||||
] as any;
|
||||
}
|
||||
}
|
||||
const prompt = messages
|
||||
.flat()
|
||||
.filter((v) => {
|
||||
if (!v.content) return false;
|
||||
if (typeof v.content === "string" && !v.content.trim()) return false;
|
||||
return true;
|
||||
})
|
||||
.map((v) => {
|
||||
const { role, content } = v;
|
||||
const insideRole = ClaudeMapper[role] ?? "user";
|
||||
|
||||
if (!visionModel || typeof content === "string") {
|
||||
return {
|
||||
role: insideRole,
|
||||
content: getMessageTextContent(v),
|
||||
};
|
||||
}
|
||||
return {
|
||||
role: insideRole,
|
||||
content: content
|
||||
.filter((v) => v.image_url || v.text)
|
||||
.map(({ type, text, image_url }) => {
|
||||
if (type === "text") {
|
||||
return {
|
||||
type,
|
||||
text: text!,
|
||||
};
|
||||
}
|
||||
const { url = "" } = image_url || {};
|
||||
const colonIndex = url.indexOf(":");
|
||||
const semicolonIndex = url.indexOf(";");
|
||||
const comma = url.indexOf(",");
|
||||
|
||||
const mimeType = url.slice(colonIndex + 1, semicolonIndex);
|
||||
const encodeType = url.slice(semicolonIndex + 1, comma);
|
||||
const data = url.slice(comma + 1);
|
||||
|
||||
return {
|
||||
type: "image" as const,
|
||||
source: {
|
||||
type: encodeType,
|
||||
media_type: mimeType,
|
||||
data,
|
||||
},
|
||||
};
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
if (prompt[0]?.role === "assistant") {
|
||||
prompt.unshift({
|
||||
role: "user",
|
||||
content: ";",
|
||||
});
|
||||
}
|
||||
const requestBody: any = {
|
||||
anthropic_version: useAccessStore.getState().bedrockAnthropicVersion,
|
||||
max_tokens: modelConfig.max_tokens,
|
||||
messages: prompt,
|
||||
temperature: modelConfig.temperature,
|
||||
top_p: modelConfig.top_p || 0.9,
|
||||
top_k: modelConfig.top_k || 5,
|
||||
};
|
||||
|
||||
// Add tools if available for Claude models
|
||||
if (toolsArray.length > 0 && model.includes("anthropic.claude")) {
|
||||
requestBody.tools = toolsArray.map((tool) => ({
|
||||
name: tool?.function?.name || "",
|
||||
description: tool?.function?.description || "",
|
||||
input_schema: tool?.function?.parameters || {},
|
||||
}));
|
||||
}
|
||||
|
||||
return requestBody;
|
||||
}
|
||||
|
||||
async chat(options: ChatOptions) {
|
||||
const accessStore = useAccessStore.getState();
|
||||
|
||||
const shouldStream = !!options.config.stream;
|
||||
|
||||
const modelConfig = {
|
||||
...useAppConfig.getState().modelConfig,
|
||||
...useChatStore.getState().currentSession().mask.modelConfig,
|
||||
...{
|
||||
model: options.config.model,
|
||||
},
|
||||
};
|
||||
|
||||
// try get base64image from local cache image_url
|
||||
const messages: ChatOptions["messages"] = [];
|
||||
for (const v of options.messages) {
|
||||
const content = await preProcessImageContent(v.content);
|
||||
messages.push({ role: v.role, content });
|
||||
}
|
||||
|
||||
const controller = new AbortController();
|
||||
options.onController?.(controller);
|
||||
|
||||
let finalRequestBody = this.formatRequestBody(messages, modelConfig);
|
||||
|
||||
try {
|
||||
const bedrockAPIPath = `${BEDROCK_BASE_URL}/model/${
|
||||
modelConfig.model
|
||||
}/invoke${shouldStream ? "-with-response-stream" : ""}`;
|
||||
const chatPath = isApp ? bedrockAPIPath : ApiPath.Bedrock + "/chat";
|
||||
|
||||
if (process.env.NODE_ENV !== "production") {
|
||||
console.debug("[Bedrock Client] Request:", {
|
||||
path: chatPath,
|
||||
model: modelConfig.model,
|
||||
messages: messages.length,
|
||||
stream: shouldStream,
|
||||
});
|
||||
}
|
||||
|
||||
if (shouldStream) {
|
||||
const [tools, funcs] = usePluginStore
|
||||
.getState()
|
||||
.getAsTools(
|
||||
useChatStore.getState().currentSession().mask?.plugin || [],
|
||||
);
|
||||
return bedrockStream(
|
||||
modelConfig.model,
|
||||
chatPath,
|
||||
finalRequestBody,
|
||||
funcs,
|
||||
controller,
|
||||
// processToolMessage, include tool_calls message and tool call results
|
||||
(
|
||||
requestPayload: any[],
|
||||
toolCallMessage: any,
|
||||
toolCallResult: any[],
|
||||
) => {
|
||||
const modelId = modelConfig.model;
|
||||
const isMistral = modelId.includes("mistral.mistral");
|
||||
const isClaude = modelId.includes("anthropic.claude");
|
||||
const isNova = modelId.includes("amazon.nova");
|
||||
|
||||
if (isClaude) {
|
||||
// Format for Claude
|
||||
// @ts-ignore
|
||||
requestPayload?.messages?.splice(
|
||||
// @ts-ignore
|
||||
requestPayload?.messages?.length,
|
||||
0,
|
||||
{
|
||||
role: "assistant",
|
||||
content: toolCallMessage.tool_calls.map(
|
||||
(tool: ChatMessageTool) => ({
|
||||
type: "tool_use",
|
||||
id: tool.id,
|
||||
name: tool?.function?.name,
|
||||
input: tool?.function?.arguments
|
||||
? JSON.parse(tool?.function?.arguments)
|
||||
: {},
|
||||
}),
|
||||
),
|
||||
},
|
||||
// @ts-ignore
|
||||
...toolCallResult.map((result) => ({
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
type: "tool_result",
|
||||
tool_use_id: result.tool_call_id,
|
||||
content: result.content,
|
||||
},
|
||||
],
|
||||
})),
|
||||
);
|
||||
} else if (isMistral) {
|
||||
// Format for Mistral
|
||||
// @ts-ignore
|
||||
requestPayload?.messages?.splice(
|
||||
// @ts-ignore
|
||||
requestPayload?.messages?.length,
|
||||
0,
|
||||
{
|
||||
role: "assistant",
|
||||
content: "",
|
||||
// @ts-ignore
|
||||
tool_calls: toolCallMessage.tool_calls.map(
|
||||
(tool: ChatMessageTool) => ({
|
||||
id: tool.id,
|
||||
function: {
|
||||
name: tool?.function?.name,
|
||||
arguments: tool?.function?.arguments || "{}",
|
||||
},
|
||||
}),
|
||||
),
|
||||
},
|
||||
...toolCallResult.map((result) => ({
|
||||
role: "tool",
|
||||
tool_call_id: result.tool_call_id,
|
||||
content: result.content,
|
||||
})),
|
||||
);
|
||||
} else if (isNova) {
|
||||
// Format for Nova - Updated format
|
||||
// @ts-ignore
|
||||
requestPayload?.messages?.splice(
|
||||
// @ts-ignore
|
||||
requestPayload?.messages?.length,
|
||||
0,
|
||||
{
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
toolUse: {
|
||||
toolUseId: toolCallMessage.tool_calls[0].id,
|
||||
name: toolCallMessage.tool_calls[0]?.function?.name,
|
||||
input:
|
||||
typeof toolCallMessage.tool_calls[0]?.function
|
||||
?.arguments === "string"
|
||||
? JSON.parse(
|
||||
toolCallMessage.tool_calls[0]?.function
|
||||
?.arguments,
|
||||
)
|
||||
: toolCallMessage.tool_calls[0]?.function
|
||||
?.arguments || {},
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: [
|
||||
{
|
||||
toolResult: {
|
||||
toolUseId: toolCallResult[0].tool_call_id,
|
||||
content: [
|
||||
{
|
||||
json: {
|
||||
content: toolCallResult[0].content,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
} else {
|
||||
console.warn(
|
||||
`[Bedrock Client] Unhandled model type for tool calls: ${modelId}`,
|
||||
);
|
||||
}
|
||||
},
|
||||
options,
|
||||
);
|
||||
} else {
|
||||
try {
|
||||
controller.signal.onabort = () =>
|
||||
options.onFinish("", new Response(null, { status: 400 }));
|
||||
const newHeaders = await getBedrockHeaders(
|
||||
modelConfig.model,
|
||||
chatPath,
|
||||
JSON.stringify(finalRequestBody),
|
||||
shouldStream,
|
||||
);
|
||||
const res = await fetch(chatPath, {
|
||||
method: "POST",
|
||||
headers: newHeaders,
|
||||
body: JSON.stringify(finalRequestBody),
|
||||
});
|
||||
const contentType = res.headers.get("content-type");
|
||||
console.log(
|
||||
"[Bedrock Not Stream Request] response content type: ",
|
||||
contentType,
|
||||
);
|
||||
const resJson = await res.json();
|
||||
const message = extractMessage(resJson);
|
||||
options.onFinish(message, res);
|
||||
} catch (e) {
|
||||
const error =
|
||||
e instanceof Error ? e : new Error("Unknown error occurred");
|
||||
console.error("[Bedrock Client] Chat failed:", error.message);
|
||||
options.onError?.(error);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error("[Bedrock Client] Chat error:", e);
|
||||
options.onError?.(e as Error);
|
||||
}
|
||||
}
|
||||
|
||||
async usage() {
|
||||
return { used: 0, total: 0 };
|
||||
}
|
||||
|
||||
async models() {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function bedrockStream(
|
||||
modelId: string,
|
||||
chatPath: string,
|
||||
requestPayload: any,
|
||||
funcs: Record<string, Function>,
|
||||
controller: AbortController,
|
||||
processToolMessage: (
|
||||
requestPayload: any,
|
||||
toolCallMessage: any,
|
||||
toolCallResult: any[],
|
||||
) => void,
|
||||
options: any,
|
||||
) {
|
||||
let responseText = "";
|
||||
let remainText = "";
|
||||
let finished = false;
|
||||
let running = false;
|
||||
let runTools: any[] = [];
|
||||
let responseRes: Response;
|
||||
let index = -1;
|
||||
let chunks: Uint8Array[] = [];
|
||||
let pendingChunk: Uint8Array | null = null;
|
||||
|
||||
function animateResponseText() {
|
||||
if (finished || controller.signal.aborted) {
|
||||
responseText += remainText;
|
||||
console.log("[Response Animation] finished");
|
||||
if (responseText?.length === 0) {
|
||||
options.onError?.(new Error("empty response from server"));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (remainText.length > 0) {
|
||||
const fetchCount = Math.max(1, Math.round(remainText.length / 60));
|
||||
const fetchText = remainText.slice(0, fetchCount);
|
||||
responseText += fetchText;
|
||||
remainText = remainText.slice(fetchCount);
|
||||
options.onUpdate?.(responseText, fetchText);
|
||||
}
|
||||
|
||||
requestAnimationFrame(animateResponseText);
|
||||
}
|
||||
|
||||
animateResponseText();
|
||||
|
||||
const finish = () => {
|
||||
if (!finished) {
|
||||
if (!running && runTools.length > 0) {
|
||||
const toolCallMessage = {
|
||||
role: "assistant",
|
||||
tool_calls: [...runTools],
|
||||
};
|
||||
running = true;
|
||||
runTools.splice(0, runTools.length);
|
||||
return Promise.all(
|
||||
toolCallMessage.tool_calls.map((tool) => {
|
||||
options?.onBeforeTool?.(tool);
|
||||
const funcName = tool?.function?.name || tool?.name;
|
||||
if (!funcName || !funcs[funcName]) {
|
||||
console.error(`Function ${funcName} not found in funcs:`, funcs);
|
||||
return Promise.reject(`Function ${funcName} not found`);
|
||||
}
|
||||
return Promise.resolve(
|
||||
funcs[funcName](
|
||||
tool?.function?.arguments
|
||||
? JSON.parse(tool?.function?.arguments)
|
||||
: {},
|
||||
),
|
||||
)
|
||||
.then((res) => {
|
||||
let content = res.data || res?.statusText;
|
||||
content =
|
||||
typeof content === "string"
|
||||
? content
|
||||
: JSON.stringify(content);
|
||||
if (res.status >= 300) {
|
||||
return Promise.reject(content);
|
||||
}
|
||||
return content;
|
||||
})
|
||||
.then((content) => {
|
||||
options?.onAfterTool?.({
|
||||
...tool,
|
||||
content,
|
||||
isError: false,
|
||||
});
|
||||
return content;
|
||||
})
|
||||
.catch((e) => {
|
||||
options?.onAfterTool?.({
|
||||
...tool,
|
||||
isError: true,
|
||||
errorMsg: e.toString(),
|
||||
});
|
||||
return e.toString();
|
||||
})
|
||||
.then((content) => ({
|
||||
name: funcName,
|
||||
role: "tool",
|
||||
content,
|
||||
tool_call_id: tool.id,
|
||||
}));
|
||||
}),
|
||||
).then((toolCallResult) => {
|
||||
processToolMessage(requestPayload, toolCallMessage, toolCallResult);
|
||||
setTimeout(() => {
|
||||
console.debug("[BedrockAPI for toolCallResult] restart");
|
||||
running = false;
|
||||
bedrockChatApi(modelId, chatPath, requestPayload, true);
|
||||
}, 60);
|
||||
});
|
||||
}
|
||||
if (running) {
|
||||
return;
|
||||
}
|
||||
console.debug("[BedrockAPI] end");
|
||||
finished = true;
|
||||
options.onFinish(responseText + remainText, responseRes);
|
||||
}
|
||||
};
|
||||
|
||||
controller.signal.onabort = finish;
|
||||
|
||||
async function bedrockChatApi(
|
||||
modelId: string,
|
||||
chatPath: string,
|
||||
requestPayload: any,
|
||||
shouldStream: boolean,
|
||||
) {
|
||||
const requestTimeoutId = setTimeout(
|
||||
() => controller.abort(),
|
||||
REQUEST_TIMEOUT_MS,
|
||||
);
|
||||
|
||||
const newHeaders = await getBedrockHeaders(
|
||||
modelId,
|
||||
chatPath,
|
||||
JSON.stringify(requestPayload),
|
||||
shouldStream,
|
||||
);
|
||||
try {
|
||||
const res = await fetch(chatPath, {
|
||||
method: "POST",
|
||||
headers: newHeaders,
|
||||
body: JSON.stringify(requestPayload),
|
||||
redirect: "manual",
|
||||
// @ts-ignore
|
||||
duplex: "half",
|
||||
signal: controller.signal,
|
||||
});
|
||||
|
||||
clearTimeout(requestTimeoutId);
|
||||
responseRes = res;
|
||||
|
||||
const contentType = res.headers.get("content-type");
|
||||
// console.log(
|
||||
// "[Bedrock Stream Request] response content type: ",
|
||||
// contentType,
|
||||
// );
|
||||
|
||||
if (contentType?.startsWith("text/plain")) {
|
||||
responseText = await res.text();
|
||||
return finish();
|
||||
}
|
||||
|
||||
if (
|
||||
!res.ok ||
|
||||
res.status !== 200 ||
|
||||
!contentType?.startsWith("application/vnd.amazon.eventstream")
|
||||
) {
|
||||
const responseTexts = [responseText];
|
||||
let extraInfo = await res.text();
|
||||
try {
|
||||
const resJson = await res.clone().json();
|
||||
extraInfo = prettyObject(resJson);
|
||||
} catch {}
|
||||
|
||||
if (res.status === 401) {
|
||||
responseTexts.push(Locale.Error.Unauthorized);
|
||||
}
|
||||
|
||||
if (extraInfo) {
|
||||
responseTexts.push(extraInfo);
|
||||
}
|
||||
|
||||
responseText = responseTexts.join("\n\n");
|
||||
return finish();
|
||||
}
|
||||
|
||||
const reader = res.body?.getReader();
|
||||
if (!reader) {
|
||||
throw new Error("No response body reader available");
|
||||
}
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) {
|
||||
if (pendingChunk) {
|
||||
try {
|
||||
const parsed = parseEventData(pendingChunk);
|
||||
if (parsed) {
|
||||
const result = processMessage(
|
||||
parsed,
|
||||
remainText,
|
||||
runTools,
|
||||
index,
|
||||
);
|
||||
remainText = result.remainText;
|
||||
index = result.index;
|
||||
}
|
||||
} catch (e) {
|
||||
console.error("[Final Chunk Process Error]:", e);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
chunks.push(value);
|
||||
|
||||
const result = processChunks(
|
||||
chunks,
|
||||
pendingChunk,
|
||||
remainText,
|
||||
runTools,
|
||||
index,
|
||||
);
|
||||
chunks = result.chunks;
|
||||
pendingChunk = result.pendingChunk;
|
||||
remainText = result.remainText;
|
||||
index = result.index;
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(
|
||||
"[Bedrock Stream]:",
|
||||
err instanceof Error ? err.message : "Stream processing failed",
|
||||
);
|
||||
throw new Error("Failed to process stream response");
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
finish();
|
||||
}
|
||||
} catch (e) {
|
||||
if (e instanceof Error && e.name === "AbortError") {
|
||||
console.log("[Bedrock Client] Aborted by user");
|
||||
return;
|
||||
}
|
||||
console.error(
|
||||
"[Bedrock Request] Failed:",
|
||||
e instanceof Error ? e.message : "Request failed",
|
||||
);
|
||||
options.onError?.(e);
|
||||
throw new Error("Request processing failed");
|
||||
}
|
||||
}
|
||||
|
||||
console.debug("[BedrockAPI] start");
|
||||
bedrockChatApi(modelId, chatPath, requestPayload, true);
|
||||
}
|
||||
@ -750,4 +750,4 @@
|
||||
transform: translateX(0);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -21,6 +21,7 @@ import BotIconGrok from "../icons/llm-icons/grok.svg";
|
||||
import BotIconHunyuan from "../icons/llm-icons/hunyuan.svg";
|
||||
import BotIconDoubao from "../icons/llm-icons/doubao.svg";
|
||||
import BotIconChatglm from "../icons/llm-icons/chatglm.svg";
|
||||
import BotIconBedrock from "../icons/llm-icons/bedrock-color.svg";
|
||||
|
||||
export function getEmojiUrl(unified: string, style: EmojiStyle) {
|
||||
// Whoever owns this Content Delivery Network (CDN), I am using your CDN to serve emojis
|
||||
@ -68,7 +69,10 @@ export function Avatar(props: { model?: ModelType; avatar?: string }) {
|
||||
LlmIcon = BotIconClaude;
|
||||
} else if (modelName.includes("llama")) {
|
||||
LlmIcon = BotIconMeta;
|
||||
} else if (modelName.startsWith("mixtral") || modelName.startsWith("codestral")) {
|
||||
} else if (
|
||||
modelName.startsWith("mixtral") ||
|
||||
modelName.startsWith("codestral")
|
||||
) {
|
||||
LlmIcon = BotIconMistral;
|
||||
} else if (modelName.includes("deepseek")) {
|
||||
LlmIcon = BotIconDeepseek;
|
||||
@ -90,6 +94,8 @@ export function Avatar(props: { model?: ModelType; avatar?: string }) {
|
||||
modelName.startsWith("cogvideox-")
|
||||
) {
|
||||
LlmIcon = BotIconChatglm;
|
||||
} else if (modelName.includes("nova")) {
|
||||
LlmIcon = BotIconBedrock;
|
||||
}
|
||||
|
||||
return (
|
||||
|
||||
@ -1,5 +1,4 @@
|
||||
import { useState, useEffect, useMemo } from "react";
|
||||
|
||||
import styles from "./settings.module.scss";
|
||||
|
||||
import ResetIcon from "../icons/reload.svg";
|
||||
@ -967,7 +966,89 @@ export function Settings() {
|
||||
</ListItem>
|
||||
</>
|
||||
);
|
||||
|
||||
const bedrockConfigComponent = accessStore.provider ===
|
||||
ServiceProvider.Bedrock && (
|
||||
<>
|
||||
<ListItem
|
||||
title={Locale.Settings.Access.Bedrock.Region.Title}
|
||||
subTitle={Locale.Settings.Access.Bedrock.Region.SubTitle}
|
||||
>
|
||||
<input
|
||||
aria-label={Locale.Settings.Access.Bedrock.Region.Title}
|
||||
type="text"
|
||||
value={accessStore.awsRegion}
|
||||
placeholder="us-west-2"
|
||||
onChange={(e) =>
|
||||
accessStore.update((access) => {
|
||||
const region = e.currentTarget.value;
|
||||
access.awsRegion = region;
|
||||
})
|
||||
}
|
||||
/>
|
||||
</ListItem>
|
||||
<ListItem
|
||||
title={Locale.Settings.Access.Bedrock.AccessKey.Title}
|
||||
subTitle={Locale.Settings.Access.Bedrock.AccessKey.SubTitle}
|
||||
>
|
||||
<PasswordInput
|
||||
aria-label={Locale.Settings.Access.Bedrock.AccessKey.Title}
|
||||
value={accessStore.awsAccessKey}
|
||||
type="text"
|
||||
placeholder={Locale.Settings.Access.Bedrock.AccessKey.Placeholder}
|
||||
onChange={(e) => {
|
||||
accessStore.update((access) => {
|
||||
const accessKey = e.currentTarget.value;
|
||||
access.awsAccessKey = accessKey;
|
||||
});
|
||||
}}
|
||||
maskWhenShow={true}
|
||||
/>
|
||||
</ListItem>
|
||||
<ListItem
|
||||
title={Locale.Settings.Access.Bedrock.SecretKey.Title}
|
||||
subTitle={Locale.Settings.Access.Bedrock.SecretKey.SubTitle}
|
||||
>
|
||||
<PasswordInput
|
||||
aria-label={Locale.Settings.Access.Bedrock.SecretKey.Title}
|
||||
value={accessStore.awsSecretKey}
|
||||
type="text"
|
||||
placeholder={Locale.Settings.Access.Bedrock.SecretKey.Placeholder}
|
||||
onChange={(e) => {
|
||||
accessStore.update((access) => {
|
||||
const secretKey = e.currentTarget.value;
|
||||
access.awsSecretKey = secretKey;
|
||||
});
|
||||
}}
|
||||
maskWhenShow={true}
|
||||
/>
|
||||
</ListItem>
|
||||
<ListItem
|
||||
title={Locale.Settings.Access.Bedrock.EncryptionKey.Title}
|
||||
subTitle={Locale.Settings.Access.Bedrock.EncryptionKey.SubTitle}
|
||||
>
|
||||
<PasswordInput
|
||||
aria-label={Locale.Settings.Access.Bedrock.EncryptionKey.Title}
|
||||
value={accessStore.encryptionKey}
|
||||
type="text"
|
||||
placeholder={Locale.Settings.Access.Bedrock.EncryptionKey.Placeholder}
|
||||
onChange={(e) => {
|
||||
accessStore.update(
|
||||
(access) => (access.encryptionKey = e.currentTarget.value),
|
||||
);
|
||||
}}
|
||||
onBlur={(e) => {
|
||||
const value = e.currentTarget.value;
|
||||
if (!value || value.length < 8) {
|
||||
showToast(Locale.Settings.Access.Bedrock.EncryptionKey.Invalid);
|
||||
accessStore.update((access) => (access.encryptionKey = ""));
|
||||
return;
|
||||
}
|
||||
}}
|
||||
maskWhenShow={true}
|
||||
/>
|
||||
</ListItem>
|
||||
</>
|
||||
);
|
||||
const baiduConfigComponent = accessStore.provider ===
|
||||
ServiceProvider.Baidu && (
|
||||
<>
|
||||
@ -1808,6 +1889,7 @@ export function Settings() {
|
||||
</ListItem>
|
||||
|
||||
{openAIConfigComponent}
|
||||
{bedrockConfigComponent}
|
||||
{azureConfigComponent}
|
||||
{googleConfigComponent}
|
||||
{anthropicConfigComponent}
|
||||
|
||||
@ -11,6 +11,7 @@ import MaxIcon from "../icons/max.svg";
|
||||
import MinIcon from "../icons/min.svg";
|
||||
|
||||
import Locale from "../locales";
|
||||
import { maskSensitiveValue } from "../utils/aws";
|
||||
|
||||
import { createRoot } from "react-dom/client";
|
||||
import React, {
|
||||
@ -270,13 +271,25 @@ export function Input(props: InputProps) {
|
||||
}
|
||||
|
||||
export function PasswordInput(
|
||||
props: HTMLProps<HTMLInputElement> & { aria?: string },
|
||||
props: HTMLProps<HTMLInputElement> & {
|
||||
aria?: string;
|
||||
maskWhenShow?: boolean;
|
||||
},
|
||||
) {
|
||||
const [visible, setVisible] = useState(false);
|
||||
const [isEditing, setIsEditing] = useState(false);
|
||||
const { maskWhenShow, onChange, value, ...inputProps } = props;
|
||||
|
||||
function changeVisibility() {
|
||||
setVisible(!visible);
|
||||
}
|
||||
|
||||
// Get display value - use masked value only when showing and maskWhenShow is true and not editing
|
||||
const displayValue =
|
||||
maskWhenShow && visible && value && !isEditing
|
||||
? maskSensitiveValue(value as string)
|
||||
: value;
|
||||
|
||||
return (
|
||||
<div className={"password-input-container"}>
|
||||
<IconButton
|
||||
@ -286,7 +299,11 @@ export function PasswordInput(
|
||||
className={"password-eye"}
|
||||
/>
|
||||
<input
|
||||
{...props}
|
||||
{...inputProps}
|
||||
value={displayValue}
|
||||
onChange={onChange}
|
||||
onFocus={() => setIsEditing(true)}
|
||||
onBlur={() => setIsEditing(false)}
|
||||
type={visible ? "text" : "password"}
|
||||
className={"password-input"}
|
||||
/>
|
||||
@ -552,6 +569,7 @@ export function Selector<T>(props: {
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export function FullScreen(props: any) {
|
||||
const { children, right = 10, top = 10, ...rest } = props;
|
||||
const ref = useRef<HTMLDivElement>();
|
||||
|
||||
@ -13,6 +13,12 @@ declare global {
|
||||
BASE_URL?: string;
|
||||
OPENAI_ORG_ID?: string; // openai only
|
||||
|
||||
// bedrock only
|
||||
AWS_REGION?: string;
|
||||
AWS_ACCESS_KEY?: string;
|
||||
AWS_SECRET_KEY?: string;
|
||||
ENCRYPTION_KEY?: string;
|
||||
|
||||
VERCEL?: string;
|
||||
BUILD_MODE?: "standalone" | "export";
|
||||
BUILD_APP?: string; // is building desktop app
|
||||
@ -148,7 +154,10 @@ export const getServerSideConfig = () => {
|
||||
}
|
||||
|
||||
const isStability = !!process.env.STABILITY_API_KEY;
|
||||
|
||||
const isBedrock =
|
||||
!!process.env.AWS_REGION &&
|
||||
!!process.env.AWS_ACCESS_KEY &&
|
||||
!!process.env.AWS_SECRET_KEY;
|
||||
const isAzure = !!process.env.AZURE_URL;
|
||||
const isGoogle = !!process.env.GOOGLE_API_KEY;
|
||||
const isAnthropic = !!process.env.ANTHROPIC_API_KEY;
|
||||
@ -180,6 +189,12 @@ export const getServerSideConfig = () => {
|
||||
apiKey: getApiKey(process.env.OPENAI_API_KEY),
|
||||
openaiOrgId: process.env.OPENAI_ORG_ID,
|
||||
|
||||
isBedrock,
|
||||
awsRegion: process.env.AWS_REGION,
|
||||
awsAccessKey: process.env.AWS_ACCESS_KEY,
|
||||
awsSecretKey: process.env.AWS_SECRET_KEY,
|
||||
encryptionKey: process.env.ENCRYPTION_KEY,
|
||||
|
||||
isStability,
|
||||
stabilityUrl: process.env.STABILITY_URL,
|
||||
stabilityApiKey: getApiKey(process.env.STABILITY_API_KEY),
|
||||
|
||||
@ -56,6 +56,7 @@ export enum Path {
|
||||
|
||||
export enum ApiPath {
|
||||
Cors = "",
|
||||
Bedrock = "/api/bedrock",
|
||||
Azure = "/api/azure",
|
||||
OpenAI = "/api/openai",
|
||||
Anthropic = "/api/anthropic",
|
||||
@ -129,6 +130,7 @@ export enum ServiceProvider {
|
||||
XAI = "XAI",
|
||||
ChatGLM = "ChatGLM",
|
||||
DeepSeek = "DeepSeek",
|
||||
Bedrock = "Bedrock",
|
||||
SiliconFlow = "SiliconFlow",
|
||||
}
|
||||
|
||||
@ -155,6 +157,7 @@ export enum ModelProvider {
|
||||
XAI = "XAI",
|
||||
ChatGLM = "ChatGLM",
|
||||
DeepSeek = "DeepSeek",
|
||||
Bedrock = "Bedrock",
|
||||
SiliconFlow = "SiliconFlow",
|
||||
}
|
||||
|
||||
@ -260,6 +263,15 @@ export const ChatGLM = {
|
||||
VideoPath: "api/paas/v4/videos/generations",
|
||||
};
|
||||
|
||||
export const Bedrock = {
|
||||
ChatPath: "model", // Simplified path since we'll append the full path in bedrock.ts
|
||||
ApiVersion: "2023-11-01",
|
||||
getEndpoint: (region: string = "us-west-2") =>
|
||||
`https://bedrock-runtime.${region}.amazonaws.com`,
|
||||
};
|
||||
// Get the region from access store for BEDROCK_BASE_URL
|
||||
export const BEDROCK_BASE_URL = Bedrock.getEndpoint();
|
||||
|
||||
export const SiliconFlow = {
|
||||
ExampleEndpoint: SILICONFLOW_BASE_URL,
|
||||
ChatPath: "v1/chat/completions",
|
||||
@ -477,6 +489,8 @@ export const VISION_MODEL_REGEXES = [
|
||||
/gpt-4-turbo(?!.*preview)/, // Matches "gpt-4-turbo" but not "gpt-4-turbo-preview"
|
||||
/^dall-e-3$/, // Matches exactly "dall-e-3"
|
||||
/glm-4v/,
|
||||
/nova-lite/,
|
||||
/nova-pro/,
|
||||
/vl/i,
|
||||
/o3/,
|
||||
/o4-mini/,
|
||||
@ -522,6 +536,29 @@ const openaiModels = [
|
||||
"o4-mini",
|
||||
];
|
||||
|
||||
const bedrockModels = [
|
||||
// Amazon nova Models
|
||||
"us.amazon.nova-micro-v1:0",
|
||||
"us.amazon.nova-lite-v1:0",
|
||||
"us.amazon.nova-pro-v1:0",
|
||||
// Claude Models
|
||||
"anthropic.claude-3-haiku-20240307-v1:0",
|
||||
"anthropic.claude-3-5-haiku-20241022-v1:0",
|
||||
"anthropic.claude-3-sonnet-20240229-v1:0",
|
||||
"anthropic.claude-3-5-sonnet-20241022-v2:0",
|
||||
"anthropic.claude-3-opus-20240229-v1:0",
|
||||
"us.anthropic.claude-3-7-sonnet-20250219-v1:0",
|
||||
// Meta Llama Models
|
||||
"us.meta.llama3-1-8b-instruct-v1:0",
|
||||
"us.meta.llama3-1-70b-instruct-v1:0",
|
||||
"us.meta.llama3-2-11b-instruct-v1:0",
|
||||
"us.meta.llama3-2-90b-instruct-v1:0",
|
||||
"us.meta.llama3-3-70b-instruct-v1:0",
|
||||
// Mistral Models
|
||||
"mistral.mistral-large-2402-v1:0",
|
||||
"mistral.mistral-large-2407-v1:0",
|
||||
];
|
||||
|
||||
const googleModels = [
|
||||
"gemini-1.0-pro", // Deprecated on 2/15/2025
|
||||
"gemini-1.5-pro-latest",
|
||||
@ -794,6 +831,7 @@ export const DEFAULT_MODELS = [
|
||||
sorted: 11,
|
||||
},
|
||||
})),
|
||||
|
||||
...chatglmModels.map((name) => ({
|
||||
name,
|
||||
available: true,
|
||||
@ -827,6 +865,17 @@ export const DEFAULT_MODELS = [
|
||||
sorted: 14,
|
||||
},
|
||||
})),
|
||||
...bedrockModels.map((name) => ({
|
||||
name,
|
||||
available: true,
|
||||
sorted: seq++,
|
||||
provider: {
|
||||
id: "bedrock",
|
||||
providerName: "Bedrock",
|
||||
providerType: "bedrock",
|
||||
sorted: 15,
|
||||
},
|
||||
})),
|
||||
] as const;
|
||||
|
||||
export const CHAT_PAGE_SIZE = 15;
|
||||
|
||||
6
app/icons/llm-icons/bedrock-color.svg
Normal file
6
app/icons/llm-icons/bedrock-color.svg
Normal file
@ -0,0 +1,6 @@
|
||||
<svg width="32" height="32" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<rect width="24" height="24" rx="12" fill="#055F4E"/>
|
||||
<g transform="translate(3.500000, 3.500000)" fill="#FFFFFF">
|
||||
<path d="M8.5,14.3722387 L6.37725,15.0801387 L5.54950001,14.5279637 L6.45075,14.2269637 L6.17425,13.3974637 L4.62725,13.9128637 L4.125,13.5786387 L4.125,11.1872287 C4.125,11.0218637 4.03137501,10.8704887 3.8835,10.7961137 L2.375,10.0418637 L2.375,7.95761374 L3.6875,7.30136374 L5,7.95761374 L5,9.4372287 C5,9.6034887 5.09362501,9.7548637 5.2415,9.8292387 L6.9915,10.7042387 L7.3835,9.9211137 L5.875,9.1668637 L5.875,7.95761374 L7.3835,7.20423874 C7.53137501,7.12986374 7.625,6.97848874 7.625,6.81222874 L7.625,5.49972874 L6.75,5.49972874 L6.75,6.54186374 L5.4375,7.19811374 L4.125,6.54186374 L4.125,4.42173874 L5,3.83811374 L5,5.49972874 L5.875,5.49972874 L5.875,3.25536374 L6.37725,2.92023624 L8.5,3.62811374 L8.5,14.3722387 Z M13.3125,13.3747387 C13.5531251,13.3747387 13.75,13.5707387 13.75,13.8122387 C13.75,14.0537387 13.5531251,14.2497387 13.3125,14.2497387 C13.071875,14.2497387 12.875,14.0537387 12.875,13.8122387 C12.875,13.5707387 13.071875,13.3747387 13.3125,13.3747387 L13.3125,13.3747387 Z M12.4375,3.74972874 C12.678125,3.74972874 12.875,3.94572874 12.875,4.18722874 C12.875,4.42873874 12.678125,4.62472874 12.4375,4.62472874 C12.196875,4.62472874 12,4.42873874 12,4.18722874 C12,3.94572874 12.196875,3.74972874 12.4375,3.74972874 L12.4375,3.74972874 Z M14.1875,8.99972874 C14.428125,8.99972874 14.625,9.1957287 14.625,9.4372287 C14.625,9.6787387 14.428125,9.8747287 14.1875,9.8747287 C13.946875,9.8747287 13.75,9.6787387 13.75,9.4372287 C13.75,9.1957287 13.946875,8.99972874 14.1875,8.99972874 L14.1875,8.99972874 Z M12.9555,9.8747287 C13.136625,10.3831137 13.617875,10.7497287 14.1875,10.7497287 C14.911375,10.7497287 15.5,10.1617387 15.5,9.4372287 C15.5,8.71361374 14.911375,8.12472874 14.1875,8.12472874 C13.617875,8.12472874 13.136625,8.49223874 12.9555,8.99972874 L9.375,8.99972874 L9.375,7.24972874 L12.4375,7.24972874 C12.679,7.24972874 12.875,7.05461374 12.875,6.81222874 L12.875,5.41923874 C13.383375,5.23811374 13.75,4.75686374 13.75,4.18722874 C13.75,3.46361374 13.161375,2.87472874 12.4375,2.87472874 C11.713625,2.87472874 11.125,3.46361374 11.125,4.18722874 C11.125,4.75686374 11.491625,5.23811374 12,5.41923874 L12,6.37472874 L9.375,6.37472874 L9.375,3.31222874 C9.375,3.12411374 9.25425,2.95703624 9.07575,2.89748624 L6.45075,2.02248624 C6.322125,1.97451366 6.182125,1.99886621 6.070125,2.07323866 L3.445125,3.82323874 C3.3235,3.90461374 3.25,4.04111374 3.25,4.18722874 L3.25,6.54186374 L1.7415,7.29611374 C1.593625,7.37048874 1.5,7.52186374 1.5,7.68722874 L1.5,10.3122287 C1.5,10.4784887 1.593625,10.6298637 1.7415,10.7042387 L3.25,11.4584887 L3.25,13.8122387 C3.25,13.9583387 3.3235,14.0957387 3.445125,14.1762387 L6.070125,15.9262387 C6.142751,15.9752387 6.22675,15.9997387 6.3125,15.9997387 C6.359375,15.9997387 6.40525,15.9927387 6.45075,15.9769637 L9.07575,15.1019637 C9.25425,15.0433387 9.375,14.8762387 9.375,14.6872387 L9.375,12.4997387 L11.381375,12.4997387 L12.127751,13.2469637 L12.139125,13.2356387 C12.053375,13.4106387 12,13.6048387 12,13.8122387 C12,14.5358387 12.588625,15.1247387 13.3125,15.1247387 C14.036375,15.1247387 14.625,14.5358387 14.625,13.8122387 C14.625,13.0886387 14.036375,12.4997387 13.3125,12.4997387 C13.104251,12.4997387 12.91,12.5531387 12.735875,12.6397387 L12.747251,12.6283387 L11.872251,11.7533387 C11.79,11.6711137 11.679125,11.6247287 11.5625,11.6247287 L9.375,11.6247287 L9.375,9.8747287 L12.9555,9.8747287 Z"/>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 3.5 KiB |
@ -343,6 +343,32 @@ const cn = {
|
||||
SubTitle: "除默认地址外，必须包含 http(s)://",
|
||||
},
|
||||
},
|
||||
Bedrock: {
|
||||
Region: {
|
||||
Title: "AWS 区域",
|
||||
SubTitle: "Bedrock 服务所在的 AWS 区域",
|
||||
Placeholder: "us-west-2",
|
||||
Invalid: "无效的 AWS 区域格式。示例：us-west-2",
|
||||
},
|
||||
AccessKey: {
|
||||
Title: "AWS 访问密钥 ID",
|
||||
SubTitle: "用于 Bedrock 服务的 AWS 访问密钥 ID",
|
||||
Placeholder: "AKIA...",
|
||||
Invalid: "无效的 AWS Access Key 格式。必须为20个字符。",
|
||||
},
|
||||
SecretKey: {
|
||||
Title: "AWS 私有访问密钥",
|
||||
SubTitle: "用于 Bedrock 服务的 AWS 私有访问密钥",
|
||||
Placeholder: "****",
|
||||
Invalid: "无效的 AWS Secret Key 格式。必须为40个字符。",
|
||||
},
|
||||
EncryptionKey: {
|
||||
Title: "加密密钥",
|
||||
SubTitle: "用于配置数据的加密密钥",
|
||||
Placeholder: "输入加密密钥",
|
||||
Invalid: "无效的加密密钥。必须至少包含8个字符！",
|
||||
},
|
||||
},
|
||||
Azure: {
|
||||
ApiKey: {
|
||||
Title: "接口密钥",
|
||||
|
||||
@ -347,6 +347,33 @@ const en: LocaleType = {
|
||||
SubTitle: "Must start with http(s):// or use /api/openai as default",
|
||||
},
|
||||
},
|
||||
Bedrock: {
|
||||
Region: {
|
||||
Title: "AWS Region",
|
||||
SubTitle: "The AWS region where Bedrock service is located",
|
||||
Placeholder: "us-west-2",
|
||||
Invalid: "Invalid AWS region format. Example: us-west-2",
|
||||
},
|
||||
AccessKey: {
|
||||
Title: "AWS Access Key ID",
|
||||
SubTitle: "Your AWS access key ID for Bedrock service",
|
||||
Placeholder: "AKIA...",
|
||||
Invalid: "Invalid AWS access key format. Must be 20 characters long.",
|
||||
},
|
||||
SecretKey: {
|
||||
Title: "AWS Secret Access Key",
|
||||
SubTitle: "Your AWS secret access key for Bedrock service",
|
||||
Placeholder: "****",
|
||||
Invalid: "Invalid AWS secret key format. Must be 40 characters long.",
|
||||
},
|
||||
EncryptionKey: {
|
||||
Title: "Encryption Key",
|
||||
SubTitle: "Your encryption key for configuration data",
|
||||
Placeholder: "Enter encryption key",
|
||||
Invalid:
|
||||
"Invalid encryption key format. Must no less than 8 characters long!",
|
||||
},
|
||||
},
|
||||
Azure: {
|
||||
ApiKey: {
|
||||
Title: "Azure Api Key",
|
||||
|
||||
@ -16,6 +16,7 @@ import {
|
||||
DEEPSEEK_BASE_URL,
|
||||
XAI_BASE_URL,
|
||||
CHATGLM_BASE_URL,
|
||||
BEDROCK_BASE_URL,
|
||||
SILICONFLOW_BASE_URL,
|
||||
} from "../constant";
|
||||
import { getHeaders } from "../client/api";
|
||||
@ -24,36 +25,26 @@ import { createPersistStore } from "../utils/store";
|
||||
import { ensure } from "../utils/clone";
|
||||
import { DEFAULT_CONFIG } from "./config";
|
||||
import { getModelProvider } from "../utils/model";
|
||||
import { encrypt, decrypt } from "../utils/aws";
|
||||
|
||||
let fetchState = 0; // 0 not fetch, 1 fetching, 2 done
|
||||
|
||||
const isApp = getClientConfig()?.buildMode === "export";
|
||||
|
||||
const DEFAULT_OPENAI_URL = isApp ? OPENAI_BASE_URL : ApiPath.OpenAI;
|
||||
|
||||
const DEFAULT_GOOGLE_URL = isApp ? GEMINI_BASE_URL : ApiPath.Google;
|
||||
|
||||
const DEFAULT_ANTHROPIC_URL = isApp ? ANTHROPIC_BASE_URL : ApiPath.Anthropic;
|
||||
|
||||
const DEFAULT_BAIDU_URL = isApp ? BAIDU_BASE_URL : ApiPath.Baidu;
|
||||
|
||||
const DEFAULT_BYTEDANCE_URL = isApp ? BYTEDANCE_BASE_URL : ApiPath.ByteDance;
|
||||
|
||||
const DEFAULT_ALIBABA_URL = isApp ? ALIBABA_BASE_URL : ApiPath.Alibaba;
|
||||
|
||||
const DEFAULT_TENCENT_URL = isApp ? TENCENT_BASE_URL : ApiPath.Tencent;
|
||||
|
||||
const DEFAULT_MOONSHOT_URL = isApp ? MOONSHOT_BASE_URL : ApiPath.Moonshot;
|
||||
|
||||
const DEFAULT_STABILITY_URL = isApp ? STABILITY_BASE_URL : ApiPath.Stability;
|
||||
|
||||
const DEFAULT_IFLYTEK_URL = isApp ? IFLYTEK_BASE_URL : ApiPath.Iflytek;
|
||||
|
||||
const DEFAULT_DEEPSEEK_URL = isApp ? DEEPSEEK_BASE_URL : ApiPath.DeepSeek;
|
||||
|
||||
const DEFAULT_XAI_URL = isApp ? XAI_BASE_URL : ApiPath.XAI;
|
||||
|
||||
const DEFAULT_CHATGLM_URL = isApp ? CHATGLM_BASE_URL : ApiPath.ChatGLM;
|
||||
const DEFAULT_BEDROCK_URL = isApp ? BEDROCK_BASE_URL : ApiPath.Bedrock;
|
||||
|
||||
const DEFAULT_SILICONFLOW_URL = isApp
|
||||
? SILICONFLOW_BASE_URL
|
||||
@ -128,10 +119,19 @@ const DEFAULT_ACCESS_STATE = {
|
||||
chatglmUrl: DEFAULT_CHATGLM_URL,
|
||||
chatglmApiKey: "",
|
||||
|
||||
// aws bedrock
|
||||
bedrockUrl: DEFAULT_BEDROCK_URL,
|
||||
awsRegion: "",
|
||||
awsAccessKey: "",
|
||||
awsSecretKey: "",
|
||||
encryptionKey: "",
|
||||
bedrockAnthropicVersion: "bedrock-2023-05-31",
|
||||
|
||||
// siliconflow
|
||||
siliconflowUrl: DEFAULT_SILICONFLOW_URL,
|
||||
siliconflowApiKey: "",
|
||||
|
||||
|
||||
// server config
|
||||
needCode: true,
|
||||
hideUserApiKey: false,
|
||||
@ -148,11 +148,9 @@ const DEFAULT_ACCESS_STATE = {
|
||||
|
||||
export const useAccessStore = createPersistStore(
|
||||
{ ...DEFAULT_ACCESS_STATE },
|
||||
|
||||
(set, get) => ({
|
||||
enabledAccessControl() {
|
||||
this.fetch();
|
||||
|
||||
return get().needCode;
|
||||
},
|
||||
getVisionModels() {
|
||||
@ -161,7 +159,6 @@ export const useAccessStore = createPersistStore(
|
||||
},
|
||||
edgeVoiceName() {
|
||||
this.fetch();
|
||||
|
||||
return get().edgeTTSVoiceName;
|
||||
},
|
||||
|
||||
@ -200,6 +197,7 @@ export const useAccessStore = createPersistStore(
|
||||
isValidMoonshot() {
|
||||
return ensure(get(), ["moonshotApiKey"]);
|
||||
},
|
||||
|
||||
isValidIflytek() {
|
||||
return ensure(get(), ["iflytekApiKey"]);
|
||||
},
|
||||
@ -215,8 +213,19 @@ export const useAccessStore = createPersistStore(
|
||||
return ensure(get(), ["chatglmApiKey"]);
|
||||
},
|
||||
|
||||
|
||||
isValidBedrock() {
|
||||
return ensure(get(), [
|
||||
"awsRegion",
|
||||
"awsAccessKey",
|
||||
"awsSecretKey",
|
||||
"encryptionKey",
|
||||
]);
|
||||
},
|
||||
|
||||
isValidSiliconFlow() {
|
||||
return ensure(get(), ["siliconflowApiKey"]);
|
||||
|
||||
},
|
||||
|
||||
isAuthorized() {
|
||||
@ -237,11 +246,13 @@ export const useAccessStore = createPersistStore(
|
||||
this.isValidDeepSeek() ||
|
||||
this.isValidXAI() ||
|
||||
this.isValidChatGLM() ||
|
||||
this.isValidBedrock() ||
|
||||
this.isValidSiliconFlow() ||
|
||||
!this.enabledAccessControl() ||
|
||||
(this.enabledAccessControl() && ensure(get(), ["accessCode"]))
|
||||
);
|
||||
},
|
||||
|
||||
fetch() {
|
||||
if (fetchState > 0 || getClientConfig()?.buildMode === "export") return;
|
||||
fetchState = 1;
|
||||
@ -260,7 +271,6 @@ export const useAccessStore = createPersistStore(
|
||||
DEFAULT_CONFIG.modelConfig.model = model;
|
||||
DEFAULT_CONFIG.modelConfig.providerName = providerName as any;
|
||||
}
|
||||
|
||||
return res;
|
||||
})
|
||||
.then((res: DangerConfig) => {
|
||||
@ -274,6 +284,43 @@ export const useAccessStore = createPersistStore(
|
||||
fetchState = 2;
|
||||
});
|
||||
},
|
||||
|
||||
// Override the set method to encrypt AWS credentials before storage
|
||||
set: (partial: { [key: string]: any }) => {
|
||||
if (partial.awsAccessKey) {
|
||||
partial.awsAccessKey = encrypt(
|
||||
partial.awsAccessKey,
|
||||
partial.encryptionKey,
|
||||
);
|
||||
}
|
||||
if (partial.awsSecretKey) {
|
||||
partial.awsSecretKey = encrypt(
|
||||
partial.awsSecretKey,
|
||||
partial.encryptionKey,
|
||||
);
|
||||
}
|
||||
if (partial.awsRegion) {
|
||||
partial.awsRegion = encrypt(partial.awsRegion, partial.encryptionKey);
|
||||
}
|
||||
set(partial);
|
||||
},
|
||||
|
||||
// Add getter to decrypt AWS credentials when needed
|
||||
get: () => {
|
||||
const state = get();
|
||||
return {
|
||||
...state,
|
||||
awsRegion: state.awsRegion
|
||||
? decrypt(state.awsRegion, state.encryptionKey)
|
||||
: "",
|
||||
awsAccessKey: state.awsAccessKey
|
||||
? decrypt(state.awsAccessKey, state.encryptionKey)
|
||||
: "",
|
||||
awsSecretKey: state.awsSecretKey
|
||||
? decrypt(state.awsSecretKey, state.encryptionKey)
|
||||
: "",
|
||||
};
|
||||
},
|
||||
}),
|
||||
{
|
||||
name: StoreKey.Access,
|
||||
|
||||
@ -344,6 +344,13 @@ export function showPlugins(provider: ServiceProvider, model: string) {
|
||||
if (provider == ServiceProvider.Anthropic && !model.includes("claude-2")) {
|
||||
return true;
|
||||
}
|
||||
if (
|
||||
(provider == ServiceProvider.Bedrock && model.includes("claude-3")) ||
|
||||
model.includes("mistral-large") ||
|
||||
model.includes("amazon.nova")
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
if (provider == ServiceProvider.Google && !model.includes("vision")) {
|
||||
return true;
|
||||
}
|
||||
|
||||
696
app/utils/aws.ts
Normal file
696
app/utils/aws.ts
Normal file
@ -0,0 +1,696 @@
|
||||
// Types and Interfaces
|
||||
export interface BedrockCredentials {
|
||||
region: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
}
|
||||
|
||||
// Type definitions for better type safety
|
||||
type ParsedEvent = Record<string, any>;
|
||||
type EventResult = ParsedEvent[];
|
||||
|
||||
// Using a dot as separator since it's not used in Base64
|
||||
const SEPARATOR = "~";
|
||||
|
||||
// Unified crypto utilities for both frontend and backend
|
||||
async function generateKey(
|
||||
password: string,
|
||||
salt: Uint8Array,
|
||||
): Promise<CryptoKey> {
|
||||
const enc = new TextEncoder();
|
||||
const keyMaterial = await crypto.subtle.importKey(
|
||||
"raw",
|
||||
enc.encode(password),
|
||||
{ name: "PBKDF2" },
|
||||
false,
|
||||
["deriveBits", "deriveKey"],
|
||||
);
|
||||
|
||||
return crypto.subtle.deriveKey(
|
||||
{
|
||||
name: "PBKDF2",
|
||||
salt,
|
||||
iterations: 100000,
|
||||
hash: "SHA-256",
|
||||
},
|
||||
keyMaterial,
|
||||
{ name: "AES-GCM", length: 256 },
|
||||
false,
|
||||
["encrypt", "decrypt"],
|
||||
);
|
||||
}
|
||||
|
||||
function arrayBufferToBase64(buffer: ArrayBuffer | Uint8Array): string {
|
||||
const bytes = buffer instanceof Uint8Array ? buffer : new Uint8Array(buffer);
|
||||
return btoa(String.fromCharCode(...bytes));
|
||||
}
|
||||
|
||||
function base64ToArrayBuffer(base64: string): Uint8Array {
|
||||
const binaryString = atob(base64);
|
||||
const bytes = new Uint8Array(binaryString.length);
|
||||
for (let i = 0; i < binaryString.length; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
export async function encrypt(
|
||||
data: string,
|
||||
encryptionKey: string,
|
||||
): Promise<string> {
|
||||
if (!data) return "";
|
||||
if (!encryptionKey) {
|
||||
throw new Error("Encryption key is required for AWS credential encryption");
|
||||
}
|
||||
|
||||
try {
|
||||
const enc = new TextEncoder();
|
||||
const salt = crypto.getRandomValues(new Uint8Array(16));
|
||||
const iv = crypto.getRandomValues(new Uint8Array(12));
|
||||
const key = await generateKey(encryptionKey, salt);
|
||||
|
||||
const encrypted = await crypto.subtle.encrypt(
|
||||
{
|
||||
name: "AES-GCM",
|
||||
iv,
|
||||
},
|
||||
key,
|
||||
enc.encode(data),
|
||||
);
|
||||
|
||||
// Convert to base64 strings
|
||||
const encryptedBase64 = arrayBufferToBase64(encrypted);
|
||||
const saltBase64 = arrayBufferToBase64(salt);
|
||||
const ivBase64 = arrayBufferToBase64(iv);
|
||||
|
||||
return [saltBase64, ivBase64, encryptedBase64].join(SEPARATOR);
|
||||
} catch (error) {
|
||||
// console.error("[Encryption Error]:", error);
|
||||
throw new Error("Failed to encrypt AWS credentials");
|
||||
}
|
||||
}
|
||||
|
||||
export async function decrypt(
|
||||
encryptedData: string,
|
||||
encryptionKey: string,
|
||||
): Promise<string> {
|
||||
if (!encryptedData) return "";
|
||||
if (!encryptionKey) {
|
||||
throw new Error("Encryption key is required for AWS credential decryption");
|
||||
}
|
||||
|
||||
try {
|
||||
const [saltBase64, ivBase64, cipherBase64] = encryptedData.split(SEPARATOR);
|
||||
|
||||
// Convert base64 strings back to Uint8Arrays
|
||||
const salt = base64ToArrayBuffer(saltBase64);
|
||||
const iv = base64ToArrayBuffer(ivBase64);
|
||||
const cipherData = base64ToArrayBuffer(cipherBase64);
|
||||
|
||||
const key = await generateKey(encryptionKey, salt);
|
||||
|
||||
const decrypted = await crypto.subtle.decrypt(
|
||||
{
|
||||
name: "AES-GCM",
|
||||
iv,
|
||||
},
|
||||
key,
|
||||
cipherData,
|
||||
);
|
||||
|
||||
const dec = new TextDecoder();
|
||||
return dec.decode(decrypted);
|
||||
} catch (error) {
|
||||
throw new Error("Failed to decrypt AWS credentials");
|
||||
}
|
||||
}
|
||||
|
||||
export function maskSensitiveValue(value: string): string {
|
||||
if (!value) return "";
|
||||
if (value.length <= 6) return value;
|
||||
const masked = "*".repeat(value.length - 6);
|
||||
return value.slice(0, 3) + masked + value.slice(-3);
|
||||
}
|
||||
|
||||
// AWS Signing
|
||||
export interface SignParams {
|
||||
method: string;
|
||||
url: string;
|
||||
region: string;
|
||||
accessKeyId: string;
|
||||
secretAccessKey: string;
|
||||
body: string | object;
|
||||
service: string;
|
||||
headers?: Record<string, string>;
|
||||
isStreaming?: boolean;
|
||||
}
|
||||
|
||||
async function createHmac(
|
||||
key: ArrayBuffer | Uint8Array,
|
||||
data: string,
|
||||
): Promise<ArrayBuffer> {
|
||||
const encoder = new TextEncoder();
|
||||
const keyData = key instanceof Uint8Array ? key : new Uint8Array(key);
|
||||
const keyObject = await crypto.subtle.importKey(
|
||||
"raw",
|
||||
keyData,
|
||||
{ name: "HMAC", hash: "SHA-256" },
|
||||
false,
|
||||
["sign"],
|
||||
);
|
||||
return crypto.subtle.sign("HMAC", keyObject, encoder.encode(data));
|
||||
}
|
||||
|
||||
async function getSigningKey(
|
||||
secretKey: string,
|
||||
dateStamp: string,
|
||||
region: string,
|
||||
service: string,
|
||||
): Promise<ArrayBuffer> {
|
||||
const encoder = new TextEncoder();
|
||||
const kDate = await createHmac(encoder.encode("AWS4" + secretKey), dateStamp);
|
||||
const kRegion = await createHmac(kDate, region);
|
||||
const kService = await createHmac(kRegion, service);
|
||||
const kSigning = await createHmac(kService, "aws4_request");
|
||||
return kSigning;
|
||||
}
|
||||
|
||||
function normalizeHeaderValue(value: string): string {
|
||||
return value.replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function encodeRFC3986(str: string): string {
|
||||
return encodeURIComponent(str)
|
||||
.replace(
|
||||
/[!'()*]/g,
|
||||
(c) => "%" + c.charCodeAt(0).toString(16).toUpperCase(),
|
||||
)
|
||||
.replace(/[-_.~]/g, (c) => c);
|
||||
}
|
||||
|
||||
function getCanonicalUri(path: string): string {
|
||||
if (!path || path === "/") return "/";
|
||||
|
||||
return (
|
||||
"/" +
|
||||
path
|
||||
.split("/")
|
||||
.map((segment) => {
|
||||
if (!segment) return "";
|
||||
if (segment === "invoke-with-response-stream") return segment;
|
||||
|
||||
if (segment.includes("model/")) {
|
||||
return segment
|
||||
.split(/(model\/)/)
|
||||
.map((part) => {
|
||||
if (part === "model/") return part;
|
||||
return part
|
||||
.split(/([.:])/g)
|
||||
.map((subpart, i) =>
|
||||
i % 2 === 1 ? subpart : encodeRFC3986(subpart),
|
||||
)
|
||||
.join("");
|
||||
})
|
||||
.join("");
|
||||
}
|
||||
|
||||
return encodeRFC3986(segment);
|
||||
})
|
||||
.join("/")
|
||||
);
|
||||
}
|
||||
|
||||
export async function sign({
|
||||
method,
|
||||
url,
|
||||
region,
|
||||
accessKeyId,
|
||||
secretAccessKey,
|
||||
body,
|
||||
service,
|
||||
headers: customHeaders = {},
|
||||
isStreaming = true,
|
||||
}: SignParams): Promise<Record<string, string>> {
|
||||
try {
|
||||
const endpoint = new URL(url);
|
||||
const canonicalUri = getCanonicalUri(endpoint.pathname.slice(1));
|
||||
const canonicalQueryString = endpoint.search.slice(1);
|
||||
|
||||
const now = new Date();
|
||||
const amzDate = now.toISOString().replace(/[:-]|\.\d{3}/g, "");
|
||||
const dateStamp = amzDate.slice(0, 8);
|
||||
|
||||
const bodyString = typeof body === "string" ? body : JSON.stringify(body);
|
||||
const encoder = new TextEncoder();
|
||||
const payloadBuffer = await crypto.subtle.digest(
|
||||
"SHA-256",
|
||||
encoder.encode(bodyString),
|
||||
);
|
||||
const payloadHash = Array.from(new Uint8Array(payloadBuffer))
|
||||
.map((b) => b.toString(16).padStart(2, "0"))
|
||||
.join("");
|
||||
|
||||
const headers: Record<string, string> = {
|
||||
accept: isStreaming
|
||||
? "application/vnd.amazon.eventstream"
|
||||
: "application/json",
|
||||
"content-type": "application/json",
|
||||
host: endpoint.host,
|
||||
"x-amz-content-sha256": payloadHash,
|
||||
"x-amz-date": amzDate,
|
||||
...customHeaders,
|
||||
};
|
||||
|
||||
// Add x-amzn-bedrock-accept header for streaming requests
|
||||
if (isStreaming) {
|
||||
headers["x-amzn-bedrock-accept"] = "*/*";
|
||||
}
|
||||
|
||||
const sortedHeaderKeys = Object.keys(headers).sort((a, b) =>
|
||||
a.toLowerCase().localeCompare(b.toLowerCase()),
|
||||
);
|
||||
|
||||
const canonicalHeaders = sortedHeaderKeys
|
||||
.map(
|
||||
(key) => `${key.toLowerCase()}:${normalizeHeaderValue(headers[key])}\n`,
|
||||
)
|
||||
.join("");
|
||||
|
||||
const signedHeaders = sortedHeaderKeys
|
||||
.map((key) => key.toLowerCase())
|
||||
.join(";");
|
||||
|
||||
const canonicalRequest = [
|
||||
method.toUpperCase(),
|
||||
canonicalUri,
|
||||
canonicalQueryString,
|
||||
canonicalHeaders,
|
||||
signedHeaders,
|
||||
payloadHash,
|
||||
].join("\n");
|
||||
|
||||
const algorithm = "AWS4-HMAC-SHA256";
|
||||
const credentialScope = `${dateStamp}/${region}/${service}/aws4_request`;
|
||||
|
||||
const canonicalRequestHash = Array.from(
|
||||
new Uint8Array(
|
||||
await crypto.subtle.digest("SHA-256", encoder.encode(canonicalRequest)),
|
||||
),
|
||||
)
|
||||
.map((b) => b.toString(16).padStart(2, "0"))
|
||||
.join("");
|
||||
|
||||
const stringToSign = [
|
||||
algorithm,
|
||||
amzDate,
|
||||
credentialScope,
|
||||
canonicalRequestHash,
|
||||
].join("\n");
|
||||
|
||||
const signingKey = await getSigningKey(
|
||||
secretAccessKey,
|
||||
dateStamp,
|
||||
region,
|
||||
service,
|
||||
);
|
||||
|
||||
const signature = Array.from(
|
||||
new Uint8Array(await createHmac(signingKey, stringToSign)),
|
||||
)
|
||||
.map((b) => b.toString(16).padStart(2, "0"))
|
||||
.join("");
|
||||
|
||||
const authorization = [
|
||||
`${algorithm} Credential=${accessKeyId}/${credentialScope}`,
|
||||
`SignedHeaders=${signedHeaders}`,
|
||||
`Signature=${signature}`,
|
||||
].join(", ");
|
||||
|
||||
return {
|
||||
...headers,
|
||||
Authorization: authorization,
|
||||
};
|
||||
} catch (error) {
|
||||
console.error("[AWS Signing Error]: Failed to sign request");
|
||||
throw new Error("Failed to sign AWS request");
|
||||
}
|
||||
}
|
||||
|
||||
// Bedrock utilities
|
||||
function decodeBase64(base64String: string): string {
|
||||
try {
|
||||
const bytes = Buffer.from(base64String, "base64");
|
||||
const decoder = new TextDecoder("utf-8");
|
||||
return decoder.decode(bytes);
|
||||
} catch (e) {
|
||||
console.error("[Base64 Decode Error]:", e);
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
export function parseEventData(chunk: Uint8Array): EventResult {
|
||||
const decoder = new TextDecoder("utf-8");
|
||||
const text = decoder.decode(chunk);
|
||||
const results: EventResult = [];
|
||||
|
||||
try {
|
||||
// First try to parse as regular JSON
|
||||
const parsed = JSON.parse(text);
|
||||
if (parsed.bytes) {
|
||||
const decoded = decodeBase64(parsed.bytes);
|
||||
try {
|
||||
const decodedJson = JSON.parse(decoded);
|
||||
results.push(decodedJson);
|
||||
} catch (e) {
|
||||
results.push({ output: decoded });
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
if (typeof parsed.body === "string") {
|
||||
try {
|
||||
const parsedBody = JSON.parse(parsed.body);
|
||||
results.push(parsedBody);
|
||||
} catch (e) {
|
||||
results.push({ output: parsed.body });
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
results.push(parsed.body || parsed);
|
||||
return results;
|
||||
} catch (e) {
|
||||
// If regular JSON parse fails, try to extract event content
|
||||
const eventRegex = /:event-type[^\{]+(\{[^\}]+\})/g;
|
||||
let match;
|
||||
|
||||
while ((match = eventRegex.exec(text)) !== null) {
|
||||
try {
|
||||
const eventData = match[1];
|
||||
const parsed = JSON.parse(eventData);
|
||||
|
||||
if (parsed.bytes) {
|
||||
const decoded = decodeBase64(parsed.bytes);
|
||||
try {
|
||||
const decodedJson = JSON.parse(decoded);
|
||||
if (decodedJson.choices?.[0]?.message?.content) {
|
||||
results.push({ output: decodedJson.choices[0].message.content });
|
||||
} else {
|
||||
results.push(decodedJson);
|
||||
}
|
||||
} catch (e) {
|
||||
results.push({ output: decoded });
|
||||
}
|
||||
} else {
|
||||
results.push(parsed);
|
||||
}
|
||||
} catch (e) {
|
||||
console.debug("[Event Parse Warning]:", e);
|
||||
}
|
||||
}
|
||||
|
||||
// If no events were found, try to extract clean text
|
||||
if (results.length === 0) {
|
||||
// Remove event metadata markers and clean the text
|
||||
const cleanText = text
|
||||
.replace(/\{KG[^:]+:event-type[^}]+\}/g, "") // Remove event markers
|
||||
.replace(/[\x00-\x1F\x7F-\x9F\uFEFF]/g, "") // Remove control characters
|
||||
.trim();
|
||||
|
||||
if (cleanText) {
|
||||
results.push({ output: cleanText });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
export function processMessage(
|
||||
data: ParsedEvent,
|
||||
remainText: string,
|
||||
runTools: any[],
|
||||
index: number,
|
||||
): { remainText: string; index: number } {
|
||||
if (!data) return { remainText, index };
|
||||
|
||||
try {
|
||||
// Handle Nova's tool calls with exact schema match
|
||||
// console.log("processMessage data=========================",data);
|
||||
if (data.contentBlockStart?.start?.toolUse) {
|
||||
const toolUse = data.contentBlockStart.start.toolUse;
|
||||
index += 1;
|
||||
runTools.push({
|
||||
id: toolUse.toolUseId,
|
||||
type: "function",
|
||||
function: {
|
||||
name: toolUse.name || "", // Ensure name is always present
|
||||
arguments: "{}", // Initialize empty arguments
|
||||
},
|
||||
});
|
||||
return { remainText, index };
|
||||
}
|
||||
|
||||
// Handle Nova's tool input in contentBlockDelta
|
||||
if (data.contentBlockDelta?.delta?.toolUse?.input) {
|
||||
if (runTools[index]) {
|
||||
runTools[index].function.arguments =
|
||||
data.contentBlockDelta.delta.toolUse.input;
|
||||
}
|
||||
return { remainText, index };
|
||||
}
|
||||
|
||||
// Handle Nova's text content
|
||||
if (data.output?.message?.content?.[0]?.text) {
|
||||
remainText += data.output.message.content[0].text;
|
||||
return { remainText, index };
|
||||
}
|
||||
|
||||
// Handle Nova's messageStart event
|
||||
if (data.messageStart) {
|
||||
return { remainText, index };
|
||||
}
|
||||
|
||||
// Handle Nova's text delta
|
||||
if (data.contentBlockDelta?.delta?.text) {
|
||||
remainText += data.contentBlockDelta.delta.text;
|
||||
return { remainText, index };
|
||||
}
|
||||
|
||||
// Handle Nova's contentBlockStop event
|
||||
if (data.contentBlockStop) {
|
||||
return { remainText, index };
|
||||
}
|
||||
|
||||
// Handle Nova's messageStop event
|
||||
if (data.messageStop) {
|
||||
return { remainText, index };
|
||||
}
|
||||
|
||||
// Handle message_start event (for other models)
|
||||
if (data.type === "message_start") {
|
||||
return { remainText, index };
|
||||
}
|
||||
|
||||
// Handle content_block_start event (for other models)
|
||||
if (data.type === "content_block_start") {
|
||||
if (data.content_block?.type === "tool_use") {
|
||||
index += 1;
|
||||
runTools.push({
|
||||
id: data.content_block.id,
|
||||
type: "function",
|
||||
function: {
|
||||
name: data.content_block.name || "", // Ensure name is always present
|
||||
arguments: "",
|
||||
},
|
||||
});
|
||||
}
|
||||
return { remainText, index };
|
||||
}
|
||||
|
||||
// Handle content_block_delta event (for other models)
|
||||
if (data.type === "content_block_delta") {
|
||||
if (data.delta?.type === "input_json_delta" && runTools[index]) {
|
||||
runTools[index].function.arguments += data.delta.partial_json;
|
||||
} else if (data.delta?.type === "text_delta") {
|
||||
const newText = data.delta.text || "";
|
||||
remainText += newText;
|
||||
}
|
||||
return { remainText, index };
|
||||
}
|
||||
|
||||
// Handle tool calls for other models
|
||||
if (data.choices?.[0]?.message?.tool_calls) {
|
||||
for (const toolCall of data.choices[0].message.tool_calls) {
|
||||
index += 1;
|
||||
runTools.push({
|
||||
id: toolCall.id || `tool-${Date.now()}`,
|
||||
type: "function",
|
||||
function: {
|
||||
name: toolCall.function?.name || "", // Ensure name is always present
|
||||
arguments: toolCall.function?.arguments || "",
|
||||
},
|
||||
});
|
||||
}
|
||||
return { remainText, index };
|
||||
}
|
||||
|
||||
// Handle various response formats
|
||||
let newText = "";
|
||||
if (data.delta?.text) {
|
||||
newText = data.delta.text;
|
||||
} else if (data.choices?.[0]?.message?.content) {
|
||||
newText = data.choices[0].message.content;
|
||||
} else if (data.content?.[0]?.text) {
|
||||
newText = data.content[0].text;
|
||||
} else if (data.generation) {
|
||||
newText = data.generation;
|
||||
} else if (data.outputText) {
|
||||
newText = data.outputText;
|
||||
} else if (data.response) {
|
||||
newText = data.response;
|
||||
} else if (data.output) {
|
||||
newText = data.output;
|
||||
}
|
||||
|
||||
// Only append if we have new text
|
||||
if (newText) {
|
||||
remainText += newText;
|
||||
}
|
||||
} catch (e) {
|
||||
console.warn("Failed to process Bedrock message:");
|
||||
}
|
||||
|
||||
return { remainText, index };
|
||||
}
|
||||
|
||||
export function processChunks(
|
||||
chunks: Uint8Array[],
|
||||
pendingChunk: Uint8Array | null,
|
||||
remainText: string,
|
||||
runTools: any[],
|
||||
index: number,
|
||||
): {
|
||||
chunks: Uint8Array[];
|
||||
pendingChunk: Uint8Array | null;
|
||||
remainText: string;
|
||||
index: number;
|
||||
} {
|
||||
let currentText = remainText;
|
||||
let currentIndex = index;
|
||||
|
||||
while (chunks.length > 0) {
|
||||
const chunk = chunks[0];
|
||||
try {
|
||||
// If there's a pending chunk, try to merge it with the current chunk
|
||||
let chunkToProcess = chunk;
|
||||
if (pendingChunk) {
|
||||
const mergedChunk = new Uint8Array(pendingChunk.length + chunk.length);
|
||||
mergedChunk.set(pendingChunk);
|
||||
mergedChunk.set(chunk, pendingChunk.length);
|
||||
chunkToProcess = mergedChunk;
|
||||
pendingChunk = null;
|
||||
}
|
||||
|
||||
// Try to process the chunk
|
||||
const parsedEvents = parseEventData(chunkToProcess);
|
||||
if (parsedEvents.length > 0) {
|
||||
// Process each event in the chunk
|
||||
for (const parsed of parsedEvents) {
|
||||
const result = processMessage(
|
||||
parsed,
|
||||
currentText,
|
||||
runTools,
|
||||
currentIndex,
|
||||
);
|
||||
currentText = result.remainText;
|
||||
currentIndex = result.index;
|
||||
}
|
||||
chunks.shift(); // Remove processed chunk
|
||||
} else {
|
||||
// If parsing fails, it might be an incomplete chunk
|
||||
pendingChunk = chunkToProcess;
|
||||
chunks.shift();
|
||||
}
|
||||
} catch (e) {
|
||||
// console.error("[Chunk Process Error]:", e);
|
||||
// chunks.shift(); // Remove error chunk
|
||||
// pendingChunk = null; // Reset pending chunk on error
|
||||
console.warn("Failed to process chunk, attempting recovery");
|
||||
// Attempt to recover by processing the next chunk
|
||||
if (chunks.length > 1) {
|
||||
chunks.shift();
|
||||
pendingChunk = null;
|
||||
} else {
|
||||
// If this is the last chunk, throw to prevent data loss
|
||||
throw new Error("Failed to process final chunk");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
chunks,
|
||||
pendingChunk,
|
||||
remainText: currentText,
|
||||
index: currentIndex,
|
||||
};
|
||||
}
|
||||
|
||||
export function getBedrockEndpoint(
|
||||
region: string,
|
||||
modelId: string,
|
||||
shouldStream: boolean,
|
||||
): string {
|
||||
if (!region || !modelId) {
|
||||
throw new Error("Region and model ID are required for Bedrock endpoint");
|
||||
}
|
||||
const baseEndpoint = `https://bedrock-runtime.${region}.amazonaws.com`;
|
||||
const endpoint =
|
||||
shouldStream === false
|
||||
? `${baseEndpoint}/model/${modelId}/invoke`
|
||||
: `${baseEndpoint}/model/${modelId}/invoke-with-response-stream`;
|
||||
return endpoint;
|
||||
}
|
||||
|
||||
export function extractMessage(res: any, modelId: string = ""): string {
|
||||
if (!res) {
|
||||
throw new Error("Empty response received");
|
||||
}
|
||||
|
||||
let message = "";
|
||||
|
||||
// Handle Nova model response format
|
||||
if (modelId.toLowerCase().includes("nova")) {
|
||||
if (res.output?.message?.content?.[0]?.text) {
|
||||
message = res.output.message.content[0].text;
|
||||
} else {
|
||||
message = res.output || "";
|
||||
}
|
||||
}
|
||||
// Handle Mistral model response format
|
||||
else if (modelId.toLowerCase().includes("mistral")) {
|
||||
if (res.choices?.[0]?.message?.content) {
|
||||
message = res.choices[0].message.content;
|
||||
} else {
|
||||
message = res.output || "";
|
||||
}
|
||||
}
|
||||
// Handle Llama model response format
|
||||
else if (modelId.toLowerCase().includes("llama")) {
|
||||
message = res?.generation || "";
|
||||
}
|
||||
// Handle Titan model response format
|
||||
else if (modelId.toLowerCase().includes("titan")) {
|
||||
message = res?.outputText || "";
|
||||
}
|
||||
// Handle Claude and other models
|
||||
else if (res.content?.[0]?.text) {
|
||||
message = res.content[0].text;
|
||||
}
|
||||
// Handle other response formats
|
||||
else {
|
||||
message = res.output || res.response || res.message || "";
|
||||
}
|
||||
|
||||
return message;
|
||||
}
|
||||
258
docs/bedrock-response-format.md
Normal file
258
docs/bedrock-response-format.md
Normal file
@ -0,0 +1,258 @@
|
||||
# Understanding Bedrock Response Format
|
||||
|
||||
The AWS Bedrock streaming response format consists of multiple Server-Sent Events (SSE) chunks. Each chunk follows this structure:
|
||||
|
||||
```
|
||||
:event-type chunk
|
||||
:content-type application/json
|
||||
:message-type event
|
||||
{"bytes":"base64_encoded_data","p":"signature"}
|
||||
```
|
||||
|
||||
## Model-Specific Response Formats
|
||||
|
||||
### Claude 3 Format
|
||||
|
||||
When using Claude 3 models (e.g., claude-3-haiku-20240307), the decoded messages include:
|
||||
|
||||
1. **message_start**
|
||||
```json
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_bdrk_01A6sahWac4XVTR9sX3rgvsZ",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-3-haiku-20240307",
|
||||
"content": [],
|
||||
"stop_reason": null,
|
||||
"stop_sequence": null,
|
||||
"usage": {
|
||||
"input_tokens": 8,
|
||||
"output_tokens": 1
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
2. **content_block_start**
|
||||
```json
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 0,
|
||||
"content_block": {
|
||||
"type": "text",
|
||||
"text": ""
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
3. **content_block_delta**
|
||||
```json
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 0,
|
||||
"delta": {
|
||||
"type": "text_delta",
|
||||
"text": "Hello"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Mistral Format
|
||||
|
||||
When using Mistral models (e.g., mistral-large-2407), the decoded messages have a different structure:
|
||||
|
||||
```json
|
||||
{
|
||||
"id": "b0098812-0ad9-42da-9f17-a5e2f554eb6b",
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1732582566,
|
||||
"model": "mistral-large-2407",
|
||||
"choices": [{
|
||||
"index": 0,
|
||||
"logprobs": null,
|
||||
"context_logits": null,
|
||||
"generation_logits": null,
|
||||
"message": {
|
||||
"role": null,
|
||||
"content": "Hello",
|
||||
"tool_calls": null,
|
||||
"index": null,
|
||||
"tool_call_id": null
|
||||
},
|
||||
"stop_reason": null
|
||||
}],
|
||||
"usage": null,
|
||||
"p": null
|
||||
}
|
||||
```
|
||||
|
||||
### Llama Format
|
||||
|
||||
When using Llama models (3.1 or 3.2), the decoded messages use a simpler structure focused on generation tokens:
|
||||
|
||||
```json
|
||||
{
|
||||
"generation": "Hello",
|
||||
"prompt_token_count": null,
|
||||
"generation_token_count": 2,
|
||||
"stop_reason": null
|
||||
}
|
||||
```
|
||||
|
||||
Each chunk contains:
|
||||
- generation: The generated text piece
|
||||
- prompt_token_count: Token count of the input (only present in first chunk)
|
||||
- generation_token_count: Running count of generated tokens
|
||||
- stop_reason: Indicates completion (null until final chunk)
|
||||
|
||||
First chunk example (includes prompt_token_count):
|
||||
```json
|
||||
{
|
||||
"generation": "\n\n",
|
||||
"prompt_token_count": 10,
|
||||
"generation_token_count": 1,
|
||||
"stop_reason": null
|
||||
}
|
||||
```
|
||||
|
||||
### Titan Text Format
|
||||
|
||||
When using Amazon's Titan models (text or TG1), the response comes as a single chunk with complete text and metrics:
|
||||
|
||||
```json
|
||||
{
|
||||
"outputText": "\nBot: Hello! How can I help you today?",
|
||||
"index": 0,
|
||||
"totalOutputTextTokenCount": 13,
|
||||
"completionReason": "FINISH",
|
||||
"inputTextTokenCount": 3,
|
||||
"amazon-bedrock-invocationMetrics": {
|
||||
"inputTokenCount": 3,
|
||||
"outputTokenCount": 13,
|
||||
"invocationLatency": 833,
|
||||
"firstByteLatency": 833
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Both Titan text and Titan TG1 use the same response format, with only minor differences in token counts and latency values. For example, here's a TG1 response:
|
||||
|
||||
```json
|
||||
{
|
||||
"outputText": "\nBot: Hello! How can I help you?",
|
||||
"index": 0,
|
||||
"totalOutputTextTokenCount": 12,
|
||||
"completionReason": "FINISH",
|
||||
"inputTextTokenCount": 3,
|
||||
"amazon-bedrock-invocationMetrics": {
|
||||
"inputTokenCount": 3,
|
||||
"outputTokenCount": 12,
|
||||
"invocationLatency": 845,
|
||||
"firstByteLatency": 845
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Key fields:
|
||||
- outputText: The complete generated response
|
||||
- totalOutputTextTokenCount: Total tokens in the response
|
||||
- completionReason: Reason for completion (e.g., "FINISH")
|
||||
- inputTextTokenCount: Number of input tokens
|
||||
- amazon-bedrock-invocationMetrics: Detailed performance metrics
|
||||
|
||||
## Model-Specific Completion Metrics
|
||||
|
||||
### Mistral
|
||||
```json
|
||||
{
|
||||
"usage": {
|
||||
"prompt_tokens": 5,
|
||||
"total_tokens": 29,
|
||||
"completion_tokens": 24
|
||||
},
|
||||
"amazon-bedrock-invocationMetrics": {
|
||||
"inputTokenCount": 5,
|
||||
"outputTokenCount": 24,
|
||||
"invocationLatency": 719,
|
||||
"firstByteLatency": 148
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Claude 3
|
||||
Included in the message_delta with stop_reason.
|
||||
|
||||
### Llama
|
||||
Included in the final chunk with stop_reason "stop":
|
||||
```json
|
||||
{
|
||||
"amazon-bedrock-invocationMetrics": {
|
||||
"inputTokenCount": 10,
|
||||
"outputTokenCount": 11,
|
||||
"invocationLatency": 873,
|
||||
"firstByteLatency": 550
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Titan
|
||||
Both Titan text and TG1 include metrics in the single response chunk:
|
||||
```json
|
||||
{
|
||||
"amazon-bedrock-invocationMetrics": {
|
||||
"inputTokenCount": 3,
|
||||
"outputTokenCount": 12,
|
||||
"invocationLatency": 845,
|
||||
"firstByteLatency": 845
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## How the Response is Processed
|
||||
|
||||
1. The raw response is first split into chunks based on SSE format
|
||||
2. For each chunk:
|
||||
- The base64 encoded data is decoded
|
||||
- The JSON is parsed to extract the message content
|
||||
- Based on the model type and message type, different processing is applied:
|
||||
|
||||
### Claude 3 Processing
|
||||
- message_start: Initializes a new message with model info and usage stats
|
||||
- content_block_start: Starts a new content block (text, tool use, etc.)
|
||||
- content_block_delta: Adds incremental content to the current block
|
||||
- message_delta: Updates message metadata
|
||||
|
||||
### Mistral Processing
|
||||
- Each chunk contains a complete message object with choices array
|
||||
- The content is streamed through the message.content field
|
||||
- Final chunk includes token usage and invocation metrics
|
||||
|
||||
### Llama Processing
|
||||
- Each chunk contains a generation field with the text piece
|
||||
- First chunk includes prompt_token_count
|
||||
- Tracks generation progress through generation_token_count
|
||||
- Simple streaming format focused on text generation
|
||||
- Final chunk includes complete metrics
|
||||
|
||||
### Titan Processing
|
||||
- Single chunk response with complete text
|
||||
- No streaming - returns full response at once
|
||||
- Includes comprehensive metrics in the same chunk
|
||||
|
||||
## Handling in Code
|
||||
|
||||
The response is processed by the `transformBedrockStream` function in `app/utils/aws.ts`, which:
|
||||
|
||||
1. Reads the stream chunks
|
||||
2. Parses each chunk using `parseEventData`
|
||||
3. Handles model-specific formats:
|
||||
- For Claude: Processes message_start, content_block_start, content_block_delta
|
||||
- For Mistral: Extracts content from choices[0].message.content
|
||||
- For Llama: Uses the generation field directly
|
||||
- For Titan: Uses the outputText field from the single response
|
||||
4. Transforms the parsed data into a consistent format for the client
|
||||
5. Yields the transformed data as SSE events
|
||||
|
||||
This allows for real-time streaming of the model's response while maintaining a consistent format for the client application, regardless of which model is being used.
|
||||
@ -94,4 +94,4 @@
|
||||
"lint-staged/yaml": "^2.2.2"
|
||||
},
|
||||
"packageManager": "yarn@1.22.19"
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user