GithubProxy/coai
1
0
Fork 0
mirror of https://github.com/coaidev/coai.git synced 2025-05-22 22:40:14 +09:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix unicode detector

Browse Source
This commit is contained in:
Zhang Minghan 2023-09-13 22:15:59 +08:00
parent bd959d89db
commit e989003fb4
8 changed files with 112 additions and 82 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

47
app/src/components/FileProvider.tsx
View File

@ -1,5 +1,5 @@
import React, {useEffect, useRef, useState} from "react";
import {AlertCircle, File, FileCheck, Plus, X} from "lucide-react";
import React, { useEffect, useRef, useState } from "react";
import { AlertCircle, File, FileCheck, Plus, X } from "lucide-react";
import "../assets/file.less";
import {
Dialog,
@ -12,12 +12,12 @@ import {
import { useTranslation } from "react-i18next";
import { Alert, AlertTitle } from "./ui/alert.tsx";
import { useToast } from "./ui/use-toast.ts";
import {useDraggableInput} from "../utils.ts";
import { useDraggableInput } from "../utils.ts";
export type FileObject = {
name: string;
content: string;
}
};
type FileProviderProps = {
id: string;
@ -34,6 +34,22 @@ type FileObjectProps = {
onChange?: (filename?: string, data?: string) => void;
};
function isValidUnicode(str: string): boolean {
if (!Array.from(str).every(c => {
const code = c.codePointAt(0);
return c.length === 1 ? code <= 0xFFFF : code >= 0x010000 && code <= 0x10FFFF;
})) return false;
if (str.includes('\0')) {
return false;
}
const binaryRegex = /[\0-\x1F\x7F-\xFF]/;
if (binaryRegex.test(str)) {
return false;
}
return true;
}
function FileProvider({
id,
className,
@ -51,7 +67,7 @@ function FileProvider({
return () => {
setClearEvent && setClearEvent(() => {});
}
};
}, [setClearEvent]);
function clear() {
@ -86,11 +102,11 @@ function FileProvider({
<Dialog>
<DialogTrigger asChild>
<div className={`file-action`}>
{
active ?
<FileCheck className={`h-3.5 w-3.5`} /> :
<Plus className={`h-3.5 w-3.5`} />
}
{active ? (
<FileCheck className={`h-3.5 w-3.5`} />
) : (
<Plus className={`h-3.5 w-3.5`} />
)}
</div>
</DialogTrigger>
<DialogContent className={`file-dialog flex-dialog`}>
@ -117,12 +133,7 @@ function FileProvider({
);
}
function FileObject({
id,
filename,
className,
onChange,
}: FileObjectProps) {
function FileObject({ id, filename, className, onChange }: FileObjectProps) {
const { t } = useTranslation();
const { toast } = useToast();
const ref = useRef(null);
@ -134,7 +145,7 @@ function FileObject({
return () => {
target.removeEventListener("dragover", () => {});
target.removeEventListener("drop", () => {});
}
};
}, [ref]);
const handleChange = (e?: React.ChangeEvent<HTMLInputElement>) => {
@ -143,7 +154,7 @@ function FileObject({
const reader = new FileReader();
reader.onload = (e) => {
const data = e.target?.result as string;
if (!/^[\x00-\x7F]*$/.test(data)) {
if (!isValidUnicode(data)) {
toast({
title: t("file.parse-error"),
description: t("file.parse-error-prompt"),

11
app/src/components/Markdown.tsx
View File

@ -1,13 +1,13 @@
import { LightAsync as SyntaxHighlighter } from "react-syntax-highlighter";
import { atomOneDark as style } from "react-syntax-highlighter/dist/esm/styles/hljs";
import ReactMarkdown from "react-markdown";
import remarkGfm from 'remark-gfm';
import remarkGfm from "remark-gfm";
import remarkMath from "remark-math";
import rehypeKatex from "rehype-katex";
import remarkFile from "./plugins/file.tsx";
import "../assets/markdown/all.less";
import {useEffect} from "react";
import {saveAsFile} from "../utils.ts";
import { useEffect } from "react";
import { saveAsFile } from "../utils.ts";
type MarkdownProps = {
children: string;
@ -23,7 +23,10 @@ function Markdown({ children, className }: MarkdownProps) {
e.stopPropagation();
// prevent double click
// @ts-ignore
if (window.hasOwnProperty("file") && window.file + 250 > new Date().getTime()) {
if (
window.hasOwnProperty("file") &&
window.file + 250 > new Date().getTime()
) {
return;
} else {
// @ts-ignore

16
app/src/components/Message.tsx
View File

@ -14,7 +14,12 @@ import {
ContextMenuItem,
ContextMenuTrigger,
} from "./ui/context-menu.tsx";
import {copyClipboard, filterMessage, saveAsFile, useInputValue} from "../utils.ts";
import {
copyClipboard,
filterMessage,
saveAsFile,
useInputValue,
} from "../utils.ts";
import { useTranslation } from "react-i18next";
import {
Tooltip,
@ -56,12 +61,17 @@ function MessageSegment({ message }: MessageProps) {
</div>
</ContextMenuTrigger>
<ContextMenuContent>
<ContextMenuItem onClick={() => copyClipboard(filterMessage(message.content))}>
<ContextMenuItem
onClick={() => copyClipboard(filterMessage(message.content))}
>
<Copy className={`h-4 w-4 mr-2`} /> {t("message.copy")}
</ContextMenuItem>
<ContextMenuItem
onClick={() =>
saveAsFile(`message-${message.role}.txt`, filterMessage(message.content))
saveAsFile(
`message-${message.role}.txt`,
filterMessage(message.content),
)
}
>
<File className={`h-4 w-4 mr-2`} /> {t("message.save")}

78
app/src/components/plugins/file.tsx
View File

@ -1,4 +1,4 @@
import {visit} from 'unist-util-visit';
import { visit } from "unist-util-visit";
/**
* file format:
@ -17,53 +17,53 @@ function fileMarkdownPlugin() {
content: "",
last: false,
cursor: 0,
}
};
function parse(data: string, index: number, parent: any) {
if (data.startsWith(':::file')) {
cache.name = "";
cache.content = "";
cache.last = true;
cache.cursor = index;
if (data.startsWith(":::file")) {
cache.name = "";
cache.content = "";
cache.last = true;
cache.cursor = index;
const part = data.slice(7);
if (part.length > 0) {
parse(part.trimStart(), index, parent);
}
} else if (data.startsWith(':::')) {
cache.last = false;
parent.children.splice(cache.cursor, index - cache.cursor + 1, {
type: 'div',
data: {
hName: 'div',
hProperties: {
className: 'file-instance',
file: cache.name,
content: cache.content,
},
}
});
cache.name = "";
cache.content = "";
} else if (cache.last) {
if (cache.name.length === 0 && data.startsWith('[[')) {
// may contain content
const end = data.indexOf(']]');
if (end !== -1) {
cache.name = data.slice(2, end);
parse(data.slice(end + 2).trimStart(), index, parent);
} else {
cache.name = data.slice(2);
}
const part = data.slice(7);
if (part.length > 0) {
parse(part.trimStart(), index, parent);
}
} else if (data.startsWith(":::")) {
cache.last = false;
parent.children.splice(cache.cursor, index - cache.cursor + 1, {
type: "div",
data: {
hName: "div",
hProperties: {
className: "file-instance",
file: cache.name,
content: cache.content,
},
},
});
cache.name = "";
cache.content = "";
} else if (cache.last) {
if (cache.name.length === 0 && data.startsWith("[[")) {
// may contain content
const end = data.indexOf("]]");
if (end !== -1) {
cache.name = data.slice(2, end);
parse(data.slice(end + 2).trimStart(), index, parent);
} else {
cache.content += data;
cache.name = data.slice(2);
}
} else {
cache.content += data;
}
}
}
visit(tree, 'paragraph', (node, index, parent) => {
visit(tree, "paragraph", (node, index, parent) => {
for (const child of node.children) {
parse((child.value || ""), index as number, parent);
parse(child.value || "", index as number, parent);
}
});
};

2
app/src/conf.ts
View File

@ -1,6 +1,6 @@
import axios from "axios";
export const version: string = "2.4.0";
export const version: string = "2.5.0";
export const deploy: boolean = true;
export let rest_api: string = "http://localhost:8094";
export let ws_api: string = "ws://localhost:8094";

8
app/src/i18n.ts
View File

@ -136,10 +136,12 @@ const resources = {
type: "Currently only text files are supported for upload",
drop: "Drag and drop files here or click to upload",
"parse-error": "Parse Error",
"parse-error-prompt": "Parse error, currently only text files are supported",
"parse-error-prompt":
"Parse error, currently only text files are supported",
"max-length": "Content too long",
"max-length-prompt": "The content has been truncated due to the context length limit",
}
"max-length-prompt":
"The content has been truncated due to the context length limit",
},
},
},
cn: {

23
app/src/routes/Home.tsx
View File

@ -30,7 +30,13 @@ import {
updateConversationList,
} from "../conversation/history.ts";
import React, { useEffect, useRef, useState } from "react";
import {filterMessage, formatMessage, mobile, useAnimation, useEffectAsync} from "../utils.ts";
import {
filterMessage,
formatMessage,
mobile,
useAnimation,
useEffectAsync,
} from "../utils.ts";
import { useToast } from "../components/ui/use-toast.ts";
import { ConversationInstance, Message } from "../conversation/types.ts";
import {
@ -57,7 +63,7 @@ import { manager } from "../conversation/manager.ts";
import { useTranslation } from "react-i18next";
import MessageSegment from "../components/Message.tsx";
import { setMenu } from "../store/menu.ts";
import FileProvider, {FileObject} from "../components/FileProvider.tsx";
import FileProvider, { FileObject } from "../components/FileProvider.tsx";
function SideBar() {
const { t } = useTranslation();
@ -122,7 +128,9 @@ function SideBar() {
}}
>
<MessageSquare className={`h-4 w-4 mr-1`} />
<div className={`title`}>{filterMessage(conversation.name)}</div>
<div className={`title`}>
{filterMessage(conversation.name)}
</div>
<div className={`id`}>{conversation.id}</div>
<AlertDialog>
<AlertDialogTrigger asChild>
@ -242,11 +250,11 @@ function ChatInterface() {
function ChatWrapper() {
const { t } = useTranslation();
const [ file, setFile ] = useState<FileObject>({
const [file, setFile] = useState<FileObject>({
name: "",
content: "",
});
const [ clearEvent, setClearEvent ] = useState<() => void>(() => {});
const [clearEvent, setClearEvent] = useState<() => void>(() => {});
const dispatch = useDispatch();
const auth = useSelector(selectAuthenticated);
const gpt4 = useSelector(selectGPT4);
@ -311,8 +319,7 @@ function ChatWrapper() {
if (e.key === "Enter") await handleSend(auth, gpt4, web);
}}
/>
{
auth &&
{auth && (
<FileProvider
id={`file`}
className={`file`}
@ -320,7 +327,7 @@ function ChatWrapper() {
maxLength={4000 * 1.25}
setClearEvent={setClearEvent}
/>
}
)}
</div>
<Button
size={`icon`}

9
app/src/utils.ts
View File

@ -1,5 +1,5 @@
import React, { useEffect } from "react";
import {FileObject} from "./components/FileProvider.tsx";
import { FileObject } from "./components/FileProvider.tsx";
export let mobile =
window.innerWidth <= 468 ||
@ -166,17 +166,14 @@ export function filterMessage(message: string): string {
* :::
*/
return message.replace(
/:::file\n\[\[.*]]\n[\s\S]*?\n:::\n\n/g,
"",
);
return message.replace(/:::file\n\[\[.*]]\n[\s\S]*?\n:::\n\n/g, "");
}
export function useDraggableInput(
t: any,
toast: any,
target: HTMLLabelElement,
handleChange: (filename?: string, content?: string) => void
handleChange: (filename?: string, content?: string) => void,
) {
target.addEventListener("dragover", (e) => {
e.preventDefault();