From e989003fb47981b8c4bde9770df4e677bc8bd69a Mon Sep 17 00:00:00 2001 From: Zhang Minghan Date: Wed, 13 Sep 2023 22:15:59 +0800 Subject: [PATCH] fix unicode detector --- app/src/components/FileProvider.tsx | 47 ++++++++++------- app/src/components/Markdown.tsx | 11 ++-- app/src/components/Message.tsx | 16 ++++-- app/src/components/plugins/file.tsx | 78 ++++++++++++++--------------- app/src/conf.ts | 2 +- app/src/i18n.ts | 8 +-- app/src/routes/Home.tsx | 23 ++++++--- app/src/utils.ts | 9 ++-- 8 files changed, 112 insertions(+), 82 deletions(-) diff --git a/app/src/components/FileProvider.tsx b/app/src/components/FileProvider.tsx index f9ae7c9..c526c64 100644 --- a/app/src/components/FileProvider.tsx +++ b/app/src/components/FileProvider.tsx @@ -1,5 +1,5 @@ -import React, {useEffect, useRef, useState} from "react"; -import {AlertCircle, File, FileCheck, Plus, X} from "lucide-react"; +import React, { useEffect, useRef, useState } from "react"; +import { AlertCircle, File, FileCheck, Plus, X } from "lucide-react"; import "../assets/file.less"; import { Dialog, @@ -12,12 +12,12 @@ import { import { useTranslation } from "react-i18next"; import { Alert, AlertTitle } from "./ui/alert.tsx"; import { useToast } from "./ui/use-toast.ts"; -import {useDraggableInput} from "../utils.ts"; +import { useDraggableInput } from "../utils.ts"; export type FileObject = { name: string; content: string; -} +}; type FileProviderProps = { id: string; @@ -34,6 +34,22 @@ type FileObjectProps = { onChange?: (filename?: string, data?: string) => void; }; +function isValidUnicode(str: string): boolean { + if (!Array.from(str).every(c => { + const code = c.codePointAt(0); + return c.length === 1 ? code <= 0xFFFF : code >= 0x010000 && code <= 0x10FFFF; + })) return false; + if (str.includes('\0')) { + return false; + } + + const binaryRegex = /[\0-\x1F\x7F-\xFF]/; + if (binaryRegex.test(str)) { + return false; + } + return true; +} + function FileProvider({ id, className, @@ -51,7 +67,7 @@ function FileProvider({ return () => { setClearEvent && setClearEvent(() => {}); - } + }; }, [setClearEvent]); function clear() { @@ -86,11 +102,11 @@ function FileProvider({
- { - active ? - : - - } + {active ? ( + + ) : ( + + )}
@@ -117,12 +133,7 @@ function FileProvider({ ); } -function FileObject({ - id, - filename, - className, - onChange, -}: FileObjectProps) { +function FileObject({ id, filename, className, onChange }: FileObjectProps) { const { t } = useTranslation(); const { toast } = useToast(); const ref = useRef(null); @@ -134,7 +145,7 @@ function FileObject({ return () => { target.removeEventListener("dragover", () => {}); target.removeEventListener("drop", () => {}); - } + }; }, [ref]); const handleChange = (e?: React.ChangeEvent) => { @@ -143,7 +154,7 @@ function FileObject({ const reader = new FileReader(); reader.onload = (e) => { const data = e.target?.result as string; - if (!/^[\x00-\x7F]*$/.test(data)) { + if (!isValidUnicode(data)) { toast({ title: t("file.parse-error"), description: t("file.parse-error-prompt"), diff --git a/app/src/components/Markdown.tsx b/app/src/components/Markdown.tsx index 72e0f8d..b7a4ea8 100644 --- a/app/src/components/Markdown.tsx +++ b/app/src/components/Markdown.tsx @@ -1,13 +1,13 @@ import { LightAsync as SyntaxHighlighter } from "react-syntax-highlighter"; import { atomOneDark as style } from "react-syntax-highlighter/dist/esm/styles/hljs"; import ReactMarkdown from "react-markdown"; -import remarkGfm from 'remark-gfm'; +import remarkGfm from "remark-gfm"; import remarkMath from "remark-math"; import rehypeKatex from "rehype-katex"; import remarkFile from "./plugins/file.tsx"; import "../assets/markdown/all.less"; -import {useEffect} from "react"; -import {saveAsFile} from "../utils.ts"; +import { useEffect } from "react"; +import { saveAsFile } from "../utils.ts"; type MarkdownProps = { children: string; @@ -23,7 +23,10 @@ function Markdown({ children, className }: MarkdownProps) { e.stopPropagation(); // prevent double click // @ts-ignore - if (window.hasOwnProperty("file") && window.file + 250 > new Date().getTime()) { + if ( + window.hasOwnProperty("file") && + window.file + 250 > new Date().getTime() + ) { return; } else { // @ts-ignore diff --git a/app/src/components/Message.tsx b/app/src/components/Message.tsx index 87e9cc8..aa6105c 100644 --- a/app/src/components/Message.tsx +++ b/app/src/components/Message.tsx @@ -14,7 +14,12 @@ import { ContextMenuItem, ContextMenuTrigger, } from "./ui/context-menu.tsx"; -import {copyClipboard, filterMessage, saveAsFile, useInputValue} from "../utils.ts"; +import { + copyClipboard, + filterMessage, + saveAsFile, + useInputValue, +} from "../utils.ts"; import { useTranslation } from "react-i18next"; import { Tooltip, @@ -56,12 +61,17 @@ function MessageSegment({ message }: MessageProps) { - copyClipboard(filterMessage(message.content))}> + copyClipboard(filterMessage(message.content))} + > {t("message.copy")} - saveAsFile(`message-${message.role}.txt`, filterMessage(message.content)) + saveAsFile( + `message-${message.role}.txt`, + filterMessage(message.content), + ) } > {t("message.save")} diff --git a/app/src/components/plugins/file.tsx b/app/src/components/plugins/file.tsx index 0d2bbf9..192efd5 100644 --- a/app/src/components/plugins/file.tsx +++ b/app/src/components/plugins/file.tsx @@ -1,4 +1,4 @@ -import {visit} from 'unist-util-visit'; +import { visit } from "unist-util-visit"; /** * file format: @@ -17,53 +17,53 @@ function fileMarkdownPlugin() { content: "", last: false, cursor: 0, - } + }; function parse(data: string, index: number, parent: any) { - if (data.startsWith(':::file')) { - cache.name = ""; - cache.content = ""; - cache.last = true; - cache.cursor = index; + if (data.startsWith(":::file")) { + cache.name = ""; + cache.content = ""; + cache.last = true; + cache.cursor = index; - const part = data.slice(7); - if (part.length > 0) { - parse(part.trimStart(), index, parent); - } - } else if (data.startsWith(':::')) { - cache.last = false; - parent.children.splice(cache.cursor, index - cache.cursor + 1, { - type: 'div', - data: { - hName: 'div', - hProperties: { - className: 'file-instance', - file: cache.name, - content: cache.content, - }, - } - }); - cache.name = ""; - cache.content = ""; - } else if (cache.last) { - if (cache.name.length === 0 && data.startsWith('[[')) { - // may contain content - const end = data.indexOf(']]'); - if (end !== -1) { - cache.name = data.slice(2, end); - parse(data.slice(end + 2).trimStart(), index, parent); - } else { - cache.name = data.slice(2); - } + const part = data.slice(7); + if (part.length > 0) { + parse(part.trimStart(), index, parent); + } + } else if (data.startsWith(":::")) { + cache.last = false; + parent.children.splice(cache.cursor, index - cache.cursor + 1, { + type: "div", + data: { + hName: "div", + hProperties: { + className: "file-instance", + file: cache.name, + content: cache.content, + }, + }, + }); + cache.name = ""; + cache.content = ""; + } else if (cache.last) { + if (cache.name.length === 0 && data.startsWith("[[")) { + // may contain content + const end = data.indexOf("]]"); + if (end !== -1) { + cache.name = data.slice(2, end); + parse(data.slice(end + 2).trimStart(), index, parent); } else { - cache.content += data; + cache.name = data.slice(2); } + } else { + cache.content += data; } } + } - visit(tree, 'paragraph', (node, index, parent) => { + visit(tree, "paragraph", (node, index, parent) => { for (const child of node.children) { - parse((child.value || ""), index as number, parent); + parse(child.value || "", index as number, parent); } }); }; diff --git a/app/src/conf.ts b/app/src/conf.ts index c03d330..b79a3bd 100644 --- a/app/src/conf.ts +++ b/app/src/conf.ts @@ -1,6 +1,6 @@ import axios from "axios"; -export const version: string = "2.4.0"; +export const version: string = "2.5.0"; export const deploy: boolean = true; export let rest_api: string = "http://localhost:8094"; export let ws_api: string = "ws://localhost:8094"; diff --git a/app/src/i18n.ts b/app/src/i18n.ts index 9b3e06c..0342957 100644 --- a/app/src/i18n.ts +++ b/app/src/i18n.ts @@ -136,10 +136,12 @@ const resources = { type: "Currently only text files are supported for upload", drop: "Drag and drop files here or click to upload", "parse-error": "Parse Error", - "parse-error-prompt": "Parse error, currently only text files are supported", + "parse-error-prompt": + "Parse error, currently only text files are supported", "max-length": "Content too long", - "max-length-prompt": "The content has been truncated due to the context length limit", - } + "max-length-prompt": + "The content has been truncated due to the context length limit", + }, }, }, cn: { diff --git a/app/src/routes/Home.tsx b/app/src/routes/Home.tsx index e173f44..7910df2 100644 --- a/app/src/routes/Home.tsx +++ b/app/src/routes/Home.tsx @@ -30,7 +30,13 @@ import { updateConversationList, } from "../conversation/history.ts"; import React, { useEffect, useRef, useState } from "react"; -import {filterMessage, formatMessage, mobile, useAnimation, useEffectAsync} from "../utils.ts"; +import { + filterMessage, + formatMessage, + mobile, + useAnimation, + useEffectAsync, +} from "../utils.ts"; import { useToast } from "../components/ui/use-toast.ts"; import { ConversationInstance, Message } from "../conversation/types.ts"; import { @@ -57,7 +63,7 @@ import { manager } from "../conversation/manager.ts"; import { useTranslation } from "react-i18next"; import MessageSegment from "../components/Message.tsx"; import { setMenu } from "../store/menu.ts"; -import FileProvider, {FileObject} from "../components/FileProvider.tsx"; +import FileProvider, { FileObject } from "../components/FileProvider.tsx"; function SideBar() { const { t } = useTranslation(); @@ -122,7 +128,9 @@ function SideBar() { }} > -
{filterMessage(conversation.name)}
+
+ {filterMessage(conversation.name)} +
{conversation.id}
@@ -242,11 +250,11 @@ function ChatInterface() { function ChatWrapper() { const { t } = useTranslation(); - const [ file, setFile ] = useState({ + const [file, setFile] = useState({ name: "", content: "", }); - const [ clearEvent, setClearEvent ] = useState<() => void>(() => {}); + const [clearEvent, setClearEvent] = useState<() => void>(() => {}); const dispatch = useDispatch(); const auth = useSelector(selectAuthenticated); const gpt4 = useSelector(selectGPT4); @@ -311,8 +319,7 @@ function ChatWrapper() { if (e.key === "Enter") await handleSend(auth, gpt4, web); }} /> - { - auth && + {auth && ( - } + )}