From 666e62d75a374ae1b12de6c5ddd6e5c9479e8b51 Mon Sep 17 00:00:00 2001
From: Hk-Gosuto <yominogosuto@gmail.com>
Date: Wed, 20 Dec 2023 13:29:00 +0800
Subject: [PATCH] feat: support paste upload image

---
 .gitignore                     |  3 ++-
 app/client/platforms/openai.ts | 30 ++++++++++++++----------------
 app/components/chat.tsx        | 29 ++++++++++++++++++++++++++++-
 3 files changed, 44 insertions(+), 18 deletions(-)

diff --git a/.gitignore b/.gitignore
index a769884d0..69f4360c1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -45,4 +45,5 @@ dev
 *.key
 *.key.pub
 
-/public/uploads
\ No newline at end of file
+/public/uploads
+.vercel
diff --git a/app/client/platforms/openai.ts b/app/client/platforms/openai.ts
index 8898e8de1..9fa882f0c 100644
--- a/app/client/platforms/openai.ts
+++ b/app/client/platforms/openai.ts
@@ -24,6 +24,7 @@ import {
 import { prettyObject } from "@/app/utils/format";
 import { getClientConfig } from "@/app/config/client";
 import { makeAzurePath } from "@/app/azure";
+import axios from "axios";
 
 export interface OpenAIListModelResponse {
   object: string;
@@ -75,6 +76,12 @@ export class ChatGPTApi implements LLMApi {
 
   async chat(options: ChatOptions) {
     const messages: any[] = [];
+
+    const getImageBase64Data = async (url: string) => {
+      const response = await axios.get(url, { responseType: "arraybuffer" });
+      const base64 = Buffer.from(response.data, "binary").toString("base64");
+      return base64;
+    };
     for (const v of options.messages) {
       let message: {
         role: string;
@@ -88,22 +95,13 @@ export class ChatGPTApi implements LLMApi {
         text: v.content,
       });
       if (v.image_url) {
-        await fetch(v.image_url)
-          .then((response) => response.arrayBuffer())
-          .then((buffer) => {
-            const base64Data = btoa(
-              String.fromCharCode(...new Uint8Array(buffer)),
-            );
-            message.content.push({
-              type: "image_url",
-              image_url: {
-                url: `data:image/jpeg;base64,${base64Data}`,
-              },
-            });
-          })
-          .catch((error) => {
-            console.error(error);
-          });
+        var base64Data = await getImageBase64Data(v.image_url);
+        message.content.push({
+          type: "image_url",
+          image_url: {
+            url: `data:image/jpeg;base64,${base64Data}`,
+          },
+        });
       }
       messages.push(message);
     }
diff --git a/app/components/chat.tsx b/app/components/chat.tsx
index 03141a5be..de1287ed2 100644
--- a/app/components/chat.tsx
+++ b/app/components/chat.tsx
@@ -458,6 +458,10 @@ export function ChatActions(props: {
     document.getElementById("chat-image-file-select-upload")?.click();
   }
 
+  function closeImageButton() {
+    document.getElementById("chat-input-image-close")?.click();
+  }
+
   const onImageSelected = async (e: any) => {
     const file = e.target.files[0];
     const fileName = await api.file.upload(file);
@@ -488,7 +492,28 @@ export function ChatActions(props: {
       );
       showToast(nextModel);
     }
-  }, [chatStore, currentModel, models]);
+    const onPaste = (event: ClipboardEvent) => {
+      const items = event.clipboardData?.items || [];
+      for (let i = 0; i < items.length; i++) {
+        if (items[i].type.indexOf("image") === -1) continue;
+        const file = items[i].getAsFile();
+        if (file !== null) {
+          api.file.upload(file).then((fileName) => {
+            props.imageSelected({
+              fileName,
+              fileUrl: `/api/file/${fileName}`,
+            });
+          });
+        }
+      }
+    };
+    if (currentModel === "gpt-4-vision-preview") {
+      window.addEventListener("paste", onPaste);
+      return () => {
+        window.removeEventListener("paste", onPaste);
+      };
+    }
+  }, [chatStore, currentModel, models, props]);
 
   return (
     <div className={styles["chat-input-actions"]}>
@@ -594,6 +619,7 @@ export function ChatActions(props: {
               chatStore.updateCurrentSession((session) => {
                 session.mask.modelConfig.model = s[0] as ModelType;
                 session.mask.syncGlobalConfig = false;
+                closeImageButton();
               });
               showToast(s[0]);
             }}
@@ -1436,6 +1462,7 @@ function _Chat() {
               </div>
               <button
                 className={styles["chat-input-image-close"]}
+                id="chat-input-image-close"
                 onClick={() => {
                   setUserImage(null);
                 }}