From 600a7d2197de67f70e6b1668f4ce05f951aad88f Mon Sep 17 00:00:00 2001 From: Hk-Gosuto Date: Sun, 10 Dec 2023 21:23:40 +0800 Subject: [PATCH] feat: images using object service storage --- app/api/file/upload/route.ts | 57 ++++++++++++++++++++++++++++++++++ app/client/api.ts | 29 +++++++++++++++++ app/client/platforms/openai.ts | 31 +++++++++++------- app/client/platforms/utils.ts | 19 ++++++++++++ app/components/chat.tsx | 30 +++++++++--------- docs/s3-oss.md | 4 ++- 6 files changed, 142 insertions(+), 28 deletions(-) create mode 100644 app/api/file/upload/route.ts create mode 100644 app/client/platforms/utils.ts diff --git a/app/api/file/upload/route.ts b/app/api/file/upload/route.ts new file mode 100644 index 000000000..92c9cee89 --- /dev/null +++ b/app/api/file/upload/route.ts @@ -0,0 +1,57 @@ +import { NextRequest, NextResponse } from "next/server"; +import { auth } from "../../auth"; +import S3FileStorage from "../../../utils/s3_file_storage"; + +async function handle(req: NextRequest) { + if (req.method === "OPTIONS") { + return NextResponse.json({ body: "OK" }, { status: 200 }); + } + + const authResult = auth(req); + if (authResult.error) { + return NextResponse.json(authResult, { + status: 401, + }); + } + + try { + const formData = await req.formData(); + const image = formData.get("file") as File; + + const imageReader = image.stream().getReader(); + const imageData: number[] = []; + + while (true) { + const { done, value } = await imageReader.read(); + if (done) break; + imageData.push(...value); + } + + const buffer = Buffer.from(imageData); + + var fileName = `${Date.now()}.png`; + await S3FileStorage.put(fileName, buffer); + return NextResponse.json( + { + fileName: fileName, + }, + { + status: 200, + }, + ); + } catch (e) { + return NextResponse.json( + { + error: true, + msg: (e as Error).message, + }, + { + status: 500, + }, + ); + } +} + +export const POST = handle; + +export const runtime = "edge"; diff --git a/app/client/api.ts b/app/client/api.ts index ccd876e90..704212d6b 100644 --- a/app/client/api.ts +++ b/app/client/api.ts @@ -2,6 +2,7 @@ import { getClientConfig } from "../config/client"; import { ACCESS_CODE_PREFIX, Azure, ServiceProvider } from "../constant"; import { ChatMessage, ModelType, useAccessStore } from "../store"; import { ChatGPTApi } from "./platforms/openai"; +import { FileApi } from "./platforms/utils"; export const ROLES = ["system", "user", "assistant"] as const; export type MessageRole = (typeof ROLES)[number]; @@ -97,9 +98,11 @@ export abstract class ToolApi { export class ClientApi { public llm: LLMApi; + public file: FileApi; constructor() { this.llm = new ChatGPTApi(); + this.file = new FileApi(); } config() {} @@ -150,6 +153,32 @@ export class ClientApi { export const api = new ClientApi(); +export function getAuthHeaders() { + const accessStore = useAccessStore.getState(); + const headers: Record = {}; + + const isAzure = accessStore.provider === ServiceProvider.Azure; + const authHeader = isAzure ? "api-key" : "Authorization"; + const apiKey = isAzure ? accessStore.azureApiKey : accessStore.openaiApiKey; + + const makeBearer = (s: string) => `${isAzure ? "" : "Bearer "}${s.trim()}`; + const validString = (x: string) => x && x.length > 0; + + // use user's api key first + if (validString(apiKey)) { + headers[authHeader] = makeBearer(apiKey); + } else if ( + accessStore.enabledAccessControl() && + validString(accessStore.accessCode) + ) { + headers[authHeader] = makeBearer( + ACCESS_CODE_PREFIX + accessStore.accessCode, + ); + } + + return headers; +} + export function getHeaders() { const accessStore = useAccessStore.getState(); const headers: Record = { diff --git a/app/client/platforms/openai.ts b/app/client/platforms/openai.ts index 5f348f079..8898e8de1 100644 --- a/app/client/platforms/openai.ts +++ b/app/client/platforms/openai.ts @@ -74,7 +74,8 @@ export class ChatGPTApi implements LLMApi { } async chat(options: ChatOptions) { - const messages = options.messages.map((v) => { + const messages: any[] = []; + for (const v of options.messages) { let message: { role: string; content: { type: string; text?: string; image_url?: { url: string } }[]; @@ -87,15 +88,25 @@ export class ChatGPTApi implements LLMApi { text: v.content, }); if (v.image_url) { - message.content.push({ - type: "image_url", - image_url: { - url: v.image_url, - }, - }); + await fetch(v.image_url) + .then((response) => response.arrayBuffer()) + .then((buffer) => { + const base64Data = btoa( + String.fromCharCode(...new Uint8Array(buffer)), + ); + message.content.push({ + type: "image_url", + image_url: { + url: `data:image/jpeg;base64,${base64Data}`, + }, + }); + }) + .catch((error) => { + console.error(error); + }); } - return message; - }); + messages.push(message); + } const modelConfig = { ...useAppConfig.getState().modelConfig, @@ -104,7 +115,6 @@ export class ChatGPTApi implements LLMApi { model: options.config.model, }, }; - const requestPayload = { messages, stream: options.config.stream, @@ -177,7 +187,6 @@ export class ChatGPTApi implements LLMApi { }; controller.signal.onabort = finish; - fetchEventSource(chatPath, { ...chatPayload, async onopen(res) { diff --git a/app/client/platforms/utils.ts b/app/client/platforms/utils.ts new file mode 100644 index 000000000..d796166ab --- /dev/null +++ b/app/client/platforms/utils.ts @@ -0,0 +1,19 @@ +import { getAuthHeaders } from "../api"; + +export class FileApi { + async upload(file: any): Promise { + const formData = new FormData(); + formData.append("file", file); + var headers = getAuthHeaders(); + var res = await fetch("/api/file/upload", { + method: "POST", + body: formData, + headers: { + ...headers, + }, + }); + const resJson = await res.json(); + console.log(resJson); + return resJson.fileName; + } +} diff --git a/app/components/chat.tsx b/app/components/chat.tsx index ef42971e0..f06ec095b 100644 --- a/app/components/chat.tsx +++ b/app/components/chat.tsx @@ -96,6 +96,7 @@ import { ExportMessageModal } from "./exporter"; import { getClientConfig } from "../config/client"; import { useAllModels } from "../utils/hooks"; import Image from "next/image"; +import { api } from "../client/api"; const Markdown = dynamic(async () => (await import("./markdown")).Markdown, { loading: () => , @@ -457,18 +458,13 @@ export function ChatActions(props: { document.getElementById("chat-image-file-select-upload")?.click(); } - const onImageSelected = (e: any) => { + const onImageSelected = async (e: any) => { const file = e.target.files[0]; - const filename = file.name; - const reader = new FileReader(); - reader.readAsDataURL(file); - reader.onload = () => { - const base64 = reader.result; - props.imageSelected({ - filename, - base64, - }); - }; + const fileName = await api.file.upload(file); + props.imageSelected({ + fileName, + fileUrl: `/api/file/${fileName}`, + }); e.target.value = null; }; @@ -783,7 +779,7 @@ function _Chat() { } setIsLoading(true); chatStore - .onUserInput(userInput, userImage?.base64) + .onUserInput(userInput, userImage?.fileUrl) .then(() => setIsLoading(false)); localStorage.setItem(LAST_INPUT_KEY, userInput); localStorage.setItem(LAST_INPUT_IMAGE_KEY, userImage); @@ -935,7 +931,9 @@ function _Chat() { // resend the message setIsLoading(true); - chatStore.onUserInput(userMessage.content).then(() => setIsLoading(false)); + chatStore + .onUserInput(userMessage.content, userMessage.image_url) + .then(() => setIsLoading(false)); inputRef.current?.focus(); }; @@ -992,7 +990,7 @@ function _Chat() { ...createMessage({ role: "user", content: userInput, - image_url: userImage?.base64, + image_url: userImage?.fileUrl, }), preview: true, }, @@ -1005,7 +1003,7 @@ function _Chat() { isLoading, session.messages, userInput, - userImage?.base64, + userImage?.fileUrl, ]); const [msgRenderIndex, _setMsgRenderIndex] = useState( @@ -1427,7 +1425,7 @@ function _Chat() { style={{ position: "relative", width: "48px", height: "48px" }} >