fix unicode detector

app/src/components/FileProvider.tsx

@@ -1,5 +1,5 @@
-import React, {useEffect, useRef, useState} from "react";
+import React, { useEffect, useRef, useState } from "react";
-import {AlertCircle, File, FileCheck, Plus, X} from "lucide-react";
+import { AlertCircle, File, FileCheck, Plus, X } from "lucide-react";
 import "../assets/file.less";
 import {
   Dialog,
@@ -12,12 +12,12 @@ import {
 import { useTranslation } from "react-i18next";
 import { Alert, AlertTitle } from "./ui/alert.tsx";
 import { useToast } from "./ui/use-toast.ts";
-import {useDraggableInput} from "../utils.ts";
+import { useDraggableInput } from "../utils.ts";
 
 export type FileObject = {
   name: string;
   content: string;
-}
+};
 
 type FileProviderProps = {
   id: string;
@@ -34,6 +34,22 @@ type FileObjectProps = {
   onChange?: (filename?: string, data?: string) => void;
 };
 
+function isValidUnicode(str: string): boolean {
+  if (!Array.from(str).every(c => {
+    const code = c.codePointAt(0);
+    return c.length === 1 ? code <= 0xFFFF : code >= 0x010000 && code <= 0x10FFFF;
+  })) return false;
+  if (str.includes('\0')) {
+    return false;
+  }
+  
+  const binaryRegex = /[\0-\x1F\x7F-\xFF]/;
+  if (binaryRegex.test(str)) {
+    return false;
+  }
+  return true;
+}
+
 function FileProvider({
   id,
   className,
@@ -51,7 +67,7 @@ function FileProvider({
 
     return () => {
       setClearEvent && setClearEvent(() => {});
-    }
+    };
   }, [setClearEvent]);
 
   function clear() {
@@ -86,11 +102,11 @@ function FileProvider({
       <Dialog>
         <DialogTrigger asChild>
           <div className={`file-action`}>
-            {
+            {active ? (
-              active ?
+              <FileCheck className={`h-3.5 w-3.5`} />
-                <FileCheck className={`h-3.5 w-3.5`} /> :
+            ) : (
-                <Plus className={`h-3.5 w-3.5`} />
+              <Plus className={`h-3.5 w-3.5`} />
-            }
+            )}
           </div>
         </DialogTrigger>
         <DialogContent className={`file-dialog flex-dialog`}>
@@ -117,12 +133,7 @@ function FileProvider({
   );
 }
 
-function FileObject({
+function FileObject({ id, filename, className, onChange }: FileObjectProps) {
-  id,
-  filename,
-  className,
-  onChange,
-}: FileObjectProps) {
   const { t } = useTranslation();
   const { toast } = useToast();
   const ref = useRef(null);
@@ -134,7 +145,7 @@ function FileObject({
     return () => {
       target.removeEventListener("dragover", () => {});
       target.removeEventListener("drop", () => {});
-    }
+    };
   }, [ref]);
 
   const handleChange = (e?: React.ChangeEvent<HTMLInputElement>) => {
@@ -143,7 +154,7 @@ function FileObject({
       const reader = new FileReader();
       reader.onload = (e) => {
         const data = e.target?.result as string;
-        if (!/^[\x00-\x7F]*$/.test(data)) {
+        if (!isValidUnicode(data)) {
           toast({
             title: t("file.parse-error"),
             description: t("file.parse-error-prompt"),

app/src/components/Markdown.tsx

@@ -1,13 +1,13 @@
 import { LightAsync as SyntaxHighlighter } from "react-syntax-highlighter";
 import { atomOneDark as style } from "react-syntax-highlighter/dist/esm/styles/hljs";
 import ReactMarkdown from "react-markdown";
-import remarkGfm from 'remark-gfm';
+import remarkGfm from "remark-gfm";
 import remarkMath from "remark-math";
 import rehypeKatex from "rehype-katex";
 import remarkFile from "./plugins/file.tsx";
 import "../assets/markdown/all.less";
-import {useEffect} from "react";
+import { useEffect } from "react";
-import {saveAsFile} from "../utils.ts";
+import { saveAsFile } from "../utils.ts";
 
 type MarkdownProps = {
   children: string;
@@ -23,7 +23,10 @@ function Markdown({ children, className }: MarkdownProps) {
         e.stopPropagation();
         // prevent double click
         // @ts-ignore
-        if (window.hasOwnProperty("file") && window.file + 250 > new Date().getTime()) {
+        if (
+          window.hasOwnProperty("file") &&
+          window.file + 250 > new Date().getTime()
+        ) {
           return;
         } else {
           // @ts-ignore

app/src/components/Message.tsx

@@ -14,7 +14,12 @@ import {
   ContextMenuItem,
   ContextMenuTrigger,
 } from "./ui/context-menu.tsx";
-import {copyClipboard, filterMessage, saveAsFile, useInputValue} from "../utils.ts";
+import {
+  copyClipboard,
+  filterMessage,
+  saveAsFile,
+  useInputValue,
+} from "../utils.ts";
 import { useTranslation } from "react-i18next";
 import {
   Tooltip,
@@ -56,12 +61,17 @@ function MessageSegment({ message }: MessageProps) {
         </div>
       </ContextMenuTrigger>
       <ContextMenuContent>
-        <ContextMenuItem onClick={() => copyClipboard(filterMessage(message.content))}>
+        <ContextMenuItem
+          onClick={() => copyClipboard(filterMessage(message.content))}
+        >
           <Copy className={`h-4 w-4 mr-2`} /> {t("message.copy")}
         </ContextMenuItem>
         <ContextMenuItem
           onClick={() =>
-            saveAsFile(`message-${message.role}.txt`, filterMessage(message.content))
+            saveAsFile(
+              `message-${message.role}.txt`,
+              filterMessage(message.content),
+            )
           }
         >
           <File className={`h-4 w-4 mr-2`} /> {t("message.save")}

app/src/components/plugins/file.tsx

@@ -1,4 +1,4 @@
-import {visit} from 'unist-util-visit';
+import { visit } from "unist-util-visit";
 
 /**
  * file format:
@@ -17,53 +17,53 @@ function fileMarkdownPlugin() {
       content: "",
       last: false,
       cursor: 0,
-    }
+    };
 
     function parse(data: string, index: number, parent: any) {
-        if (data.startsWith(':::file')) {
+      if (data.startsWith(":::file")) {
-          cache.name = "";
+        cache.name = "";
-          cache.content = "";
+        cache.content = "";
-          cache.last = true;
+        cache.last = true;
-          cache.cursor = index;
+        cache.cursor = index;
 
-          const part = data.slice(7);
+        const part = data.slice(7);
-          if (part.length > 0) {
+        if (part.length > 0) {
-            parse(part.trimStart(), index, parent);
+          parse(part.trimStart(), index, parent);
-          }
+        }
-        } else if (data.startsWith(':::')) {
+      } else if (data.startsWith(":::")) {
-          cache.last = false;
+        cache.last = false;
-          parent.children.splice(cache.cursor, index - cache.cursor + 1, {
+        parent.children.splice(cache.cursor, index - cache.cursor + 1, {
-            type: 'div',
+          type: "div",
-            data: {
+          data: {
-              hName: 'div',
+            hName: "div",
-              hProperties: {
+            hProperties: {
-                className: 'file-instance',
+              className: "file-instance",
-                file: cache.name,
+              file: cache.name,
-                content: cache.content,
+              content: cache.content,
-              },
+            },
-            }
+          },
-          });
+        });
-          cache.name = "";
+        cache.name = "";
-          cache.content = "";
+        cache.content = "";
-        } else if (cache.last) {
+      } else if (cache.last) {
-          if (cache.name.length === 0 && data.startsWith('[[')) {
+        if (cache.name.length === 0 && data.startsWith("[[")) {
-            // may contain content
+          // may contain content
-            const end = data.indexOf(']]');
+          const end = data.indexOf("]]");
-            if (end !== -1) {
+          if (end !== -1) {
-              cache.name = data.slice(2, end);
+            cache.name = data.slice(2, end);
-              parse(data.slice(end + 2).trimStart(), index, parent);
+            parse(data.slice(end + 2).trimStart(), index, parent);
-            } else {
-              cache.name = data.slice(2);
-            }
           } else {
-            cache.content += data;
+            cache.name = data.slice(2);
           }
+        } else {
+          cache.content += data;
         }
       }
+    }
 
-    visit(tree, 'paragraph', (node, index, parent) => {
+    visit(tree, "paragraph", (node, index, parent) => {
       for (const child of node.children) {
-        parse((child.value || ""), index as number, parent);
+        parse(child.value || "", index as number, parent);
       }
     });
   };

app/src/conf.ts

@@ -1,6 +1,6 @@
 import axios from "axios";
 
-export const version: string = "2.4.0";
+export const version: string = "2.5.0";
 export const deploy: boolean = true;
 export let rest_api: string = "http://localhost:8094";
 export let ws_api: string = "ws://localhost:8094";

app/src/i18n.ts

@@ -136,10 +136,12 @@ const resources = {
         type: "Currently only text files are supported for upload",
         drop: "Drag and drop files here or click to upload",
         "parse-error": "Parse Error",
-        "parse-error-prompt": "Parse error, currently only text files are supported",
+        "parse-error-prompt":
+          "Parse error, currently only text files are supported",
         "max-length": "Content too long",
-        "max-length-prompt": "The content has been truncated due to the context length limit",
+        "max-length-prompt":
-      }
+          "The content has been truncated due to the context length limit",
+      },
     },
   },
   cn: {

app/src/routes/Home.tsx

@@ -30,7 +30,13 @@ import {
   updateConversationList,
 } from "../conversation/history.ts";
 import React, { useEffect, useRef, useState } from "react";
-import {filterMessage, formatMessage, mobile, useAnimation, useEffectAsync} from "../utils.ts";
+import {
+  filterMessage,
+  formatMessage,
+  mobile,
+  useAnimation,
+  useEffectAsync,
+} from "../utils.ts";
 import { useToast } from "../components/ui/use-toast.ts";
 import { ConversationInstance, Message } from "../conversation/types.ts";
 import {
@@ -57,7 +63,7 @@ import { manager } from "../conversation/manager.ts";
 import { useTranslation } from "react-i18next";
 import MessageSegment from "../components/Message.tsx";
 import { setMenu } from "../store/menu.ts";
-import FileProvider, {FileObject} from "../components/FileProvider.tsx";
+import FileProvider, { FileObject } from "../components/FileProvider.tsx";
 
 function SideBar() {
   const { t } = useTranslation();
@@ -122,7 +128,9 @@ function SideBar() {
                 }}
               >
                 <MessageSquare className={`h-4 w-4 mr-1`} />
-                <div className={`title`}>{filterMessage(conversation.name)}</div>
+                <div className={`title`}>
+                  {filterMessage(conversation.name)}
+                </div>
                 <div className={`id`}>{conversation.id}</div>
                 <AlertDialog>
                   <AlertDialogTrigger asChild>
@@ -242,11 +250,11 @@ function ChatInterface() {
 
 function ChatWrapper() {
   const { t } = useTranslation();
-  const [ file, setFile ] = useState<FileObject>({
+  const [file, setFile] = useState<FileObject>({
     name: "",
     content: "",
   });
-  const [ clearEvent, setClearEvent ] = useState<() => void>(() => {});
+  const [clearEvent, setClearEvent] = useState<() => void>(() => {});
   const dispatch = useDispatch();
   const auth = useSelector(selectAuthenticated);
   const gpt4 = useSelector(selectGPT4);
@@ -311,8 +319,7 @@ function ChatWrapper() {
                   if (e.key === "Enter") await handleSend(auth, gpt4, web);
                 }}
               />
-              {
+              {auth && (
-                auth &&
                 <FileProvider
                   id={`file`}
                   className={`file`}
@@ -320,7 +327,7 @@ function ChatWrapper() {
                   maxLength={4000 * 1.25}
                   setClearEvent={setClearEvent}
                 />
-              }
+              )}
             </div>
             <Button
               size={`icon`}

app/src/utils.ts

@@ -1,5 +1,5 @@
 import React, { useEffect } from "react";
-import {FileObject} from "./components/FileProvider.tsx";
+import { FileObject } from "./components/FileProvider.tsx";
 
 export let mobile =
   window.innerWidth <= 468 ||
@@ -166,17 +166,14 @@ export function filterMessage(message: string): string {
    * :::
    */
 
-  return message.replace(
+  return message.replace(/:::file\n\[\[.*]]\n[\s\S]*?\n:::\n\n/g, "");
-    /:::file\n\[\[.*]]\n[\s\S]*?\n:::\n\n/g,
-    "",
-  );
 }
 
 export function useDraggableInput(
   t: any,
   toast: any,
   target: HTMLLabelElement,
-  handleChange: (filename?: string, content?: string) => void
+  handleChange: (filename?: string, content?: string) => void,
 ) {
   target.addEventListener("dragover", (e) => {
     e.preventDefault();