feat: images using object service storage

app/api/file/upload/route.ts

@@ -0,0 +1,57 @@
+import { NextRequest, NextResponse } from "next/server";
+import { auth } from "../../auth";
+import S3FileStorage from "../../../utils/s3_file_storage";
+
+async function handle(req: NextRequest) {
+  if (req.method === "OPTIONS") {
+    return NextResponse.json({ body: "OK" }, { status: 200 });
+  }
+
+  const authResult = auth(req);
+  if (authResult.error) {
+    return NextResponse.json(authResult, {
+      status: 401,
+    });
+  }
+
+  try {
+    const formData = await req.formData();
+    const image = formData.get("file") as File;
+
+    const imageReader = image.stream().getReader();
+    const imageData: number[] = [];
+
+    while (true) {
+      const { done, value } = await imageReader.read();
+      if (done) break;
+      imageData.push(...value);
+    }
+
+    const buffer = Buffer.from(imageData);
+
+    var fileName = `${Date.now()}.png`;
+    await S3FileStorage.put(fileName, buffer);
+    return NextResponse.json(
+      {
+        fileName: fileName,
+      },
+      {
+        status: 200,
+      },
+    );
+  } catch (e) {
+    return NextResponse.json(
+      {
+        error: true,
+        msg: (e as Error).message,
+      },
+      {
+        status: 500,
+      },
+    );
+  }
+}
+
+export const POST = handle;
+
+export const runtime = "edge";

app/client/api.ts

@@ -2,6 +2,7 @@ import { getClientConfig } from "../config/client";
 import { ACCESS_CODE_PREFIX, Azure, ServiceProvider } from "../constant";
 import { ChatMessage, ModelType, useAccessStore } from "../store";
 import { ChatGPTApi } from "./platforms/openai";
+import { FileApi } from "./platforms/utils";
 
 export const ROLES = ["system", "user", "assistant"] as const;
 export type MessageRole = (typeof ROLES)[number];
@@ -97,9 +98,11 @@ export abstract class ToolApi {
 
 export class ClientApi {
   public llm: LLMApi;
+  public file: FileApi;
 
   constructor() {
     this.llm = new ChatGPTApi();
+    this.file = new FileApi();
   }
 
   config() {}
@@ -150,6 +153,32 @@ export class ClientApi {
 
 export const api = new ClientApi();
 
+export function getAuthHeaders() {
+  const accessStore = useAccessStore.getState();
+  const headers: Record<string, string> = {};
+
+  const isAzure = accessStore.provider === ServiceProvider.Azure;
+  const authHeader = isAzure ? "api-key" : "Authorization";
+  const apiKey = isAzure ? accessStore.azureApiKey : accessStore.openaiApiKey;
+
+  const makeBearer = (s: string) => `${isAzure ? "" : "Bearer "}${s.trim()}`;
+  const validString = (x: string) => x && x.length > 0;
+
+  // use user's api key first
+  if (validString(apiKey)) {
+    headers[authHeader] = makeBearer(apiKey);
+  } else if (
+    accessStore.enabledAccessControl() &&
+    validString(accessStore.accessCode)
+  ) {
+    headers[authHeader] = makeBearer(
+      ACCESS_CODE_PREFIX + accessStore.accessCode,
+    );
+  }
+
+  return headers;
+}
+
 export function getHeaders() {
   const accessStore = useAccessStore.getState();
   const headers: Record<string, string> = {

app/client/platforms/openai.ts

@@ -74,7 +74,8 @@ export class ChatGPTApi implements LLMApi {
   }
 
   async chat(options: ChatOptions) {
-    const messages = options.messages.map((v) => {
+    const messages: any[] = [];
+    for (const v of options.messages) {
       let message: {
         role: string;
         content: { type: string; text?: string; image_url?: { url: string } }[];
@@ -87,15 +88,25 @@ export class ChatGPTApi implements LLMApi {
         text: v.content,
       });
       if (v.image_url) {
+        await fetch(v.image_url)
+          .then((response) => response.arrayBuffer())
+          .then((buffer) => {
+            const base64Data = btoa(
+              String.fromCharCode(...new Uint8Array(buffer)),
+            );
             message.content.push({
               type: "image_url",
               image_url: {
-            url: v.image_url,
+                url: `data:image/jpeg;base64,${base64Data}`,
               },
             });
-      }
+          })
-      return message;
+          .catch((error) => {
+            console.error(error);
           });
+      }
+      messages.push(message);
+    }
 
     const modelConfig = {
       ...useAppConfig.getState().modelConfig,
@@ -104,7 +115,6 @@ export class ChatGPTApi implements LLMApi {
         model: options.config.model,
       },
     };
-
     const requestPayload = {
       messages,
       stream: options.config.stream,
@@ -177,7 +187,6 @@ export class ChatGPTApi implements LLMApi {
         };
 
         controller.signal.onabort = finish;
-
         fetchEventSource(chatPath, {
           ...chatPayload,
           async onopen(res) {

app/client/platforms/utils.ts

@@ -0,0 +1,19 @@
+import { getAuthHeaders } from "../api";
+
+export class FileApi {
+  async upload(file: any): Promise<void> {
+    const formData = new FormData();
+    formData.append("file", file);
+    var headers = getAuthHeaders();
+    var res = await fetch("/api/file/upload", {
+      method: "POST",
+      body: formData,
+      headers: {
+        ...headers,
+      },
+    });
+    const resJson = await res.json();
+    console.log(resJson);
+    return resJson.fileName;
+  }
+}

app/components/chat.tsx

@@ -96,6 +96,7 @@ import { ExportMessageModal } from "./exporter";
 import { getClientConfig } from "../config/client";
 import { useAllModels } from "../utils/hooks";
 import Image from "next/image";
+import { api } from "../client/api";
 
 const Markdown = dynamic(async () => (await import("./markdown")).Markdown, {
   loading: () => <LoadingIcon />,
@@ -457,18 +458,13 @@ export function ChatActions(props: {
     document.getElementById("chat-image-file-select-upload")?.click();
   }
 
-  const onImageSelected = (e: any) => {
+  const onImageSelected = async (e: any) => {
     const file = e.target.files[0];
-    const filename = file.name;
+    const fileName = await api.file.upload(file);
-    const reader = new FileReader();
-    reader.readAsDataURL(file);
-    reader.onload = () => {
-      const base64 = reader.result;
     props.imageSelected({
-        filename,
+      fileName,
-        base64,
+      fileUrl: `/api/file/${fileName}`,
     });
-    };
     e.target.value = null;
   };
 
@@ -783,7 +779,7 @@ function _Chat() {
     }
     setIsLoading(true);
     chatStore
-      .onUserInput(userInput, userImage?.base64)
+      .onUserInput(userInput, userImage?.fileUrl)
       .then(() => setIsLoading(false));
     localStorage.setItem(LAST_INPUT_KEY, userInput);
     localStorage.setItem(LAST_INPUT_IMAGE_KEY, userImage);
@@ -935,7 +931,9 @@ function _Chat() {
 
     // resend the message
     setIsLoading(true);
-    chatStore.onUserInput(userMessage.content).then(() => setIsLoading(false));
+    chatStore
+      .onUserInput(userMessage.content, userMessage.image_url)
+      .then(() => setIsLoading(false));
     inputRef.current?.focus();
   };
 
@@ -992,7 +990,7 @@ function _Chat() {
                 ...createMessage({
                   role: "user",
                   content: userInput,
-                  image_url: userImage?.base64,
+                  image_url: userImage?.fileUrl,
                 }),
                 preview: true,
               },
@@ -1005,7 +1003,7 @@ function _Chat() {
     isLoading,
     session.messages,
     userInput,
-    userImage?.base64,
+    userImage?.fileUrl,
   ]);
 
   const [msgRenderIndex, _setMsgRenderIndex] = useState(
@@ -1427,7 +1425,7 @@ function _Chat() {
                 style={{ position: "relative", width: "48px", height: "48px" }}
               >
                 <Image
-                  src={userImage.base64}
+                  src={userImage.fileUrl}
                   alt={userImage.filename}
                   title={userImage.filename}
                   layout="fill"

docs/s3-oss.md

@@ -8,9 +8,11 @@
 
 这边以 `又拍云` 做为演示，其它运营商请查询对应文档。
 
+参考: https://help.upyun.com/knowledge-base/aws-s3%E5%85%BC%E5%AE%B9/#e585bce5aeb9e5b7a5e585b7e7a4bae4be8b
+
 1. 登录 [又拍云 - 加速在线业务 - CDN加速 - 云存储 (upyun.com)](https://www.upyun.com/)
 2. 注册账户
-3. 进入”云存储“控制台[又拍云控制台 (upyun.com)](https://console.upyun.com/services/file/)
+3. 进入"云存储"控制台[又拍云控制台 (upyun.com)](https://console.upyun.com/services/file/)
 4. 创建一个服务，记录你的服务名
 5. 进入"用户管理"，"操作员"创建一个"操作员"并赋予相应权限
 6. 编辑"操作员"复制"AccessKey"和"SecretAccessKey"