feat: add arxiv plugin (#55)

README.md

@@ -53,8 +53,7 @@
       - 本插件目前为测试版本，后续可能会有较大的变更，请谨慎使用
       - 使用本插件需要一定的专业知识，Stable Diffusion 本身的相关问题不在本项目的解答范围内，如果您确定要使用本插件请参考 [Stable Diffusion 插件配置指南](./docs/stable-diffusion-plugin-cn.md) 文档进行配置
       - StableDiffusion 插件需要配置 R2 存储，请参考 [Cloudflare R2 服务配置指南](./docs/cloudflare-r2-cn.md) 配置
-  
+    - Arxiv
-
 
 ## 开发计划
 

app/api/langchain-tools/arxiv.ts

@@ -0,0 +1,77 @@
+import { StructuredTool } from "langchain/tools";
+import { z } from "zod";
+
+export class ArxivAPIWrapper extends StructuredTool {
+  get lc_namespace() {
+    return [...super.lc_namespace, "test"];
+  }
+
+  name = "arxiv";
+  description = "Run Arxiv search and get the article information.";
+
+  SORT_BY = {
+    RELEVANCE: "relevance",
+    LAST_UPDATED_DATE: "lastUpdatedDate",
+    SUBMITTED_DATE: "submittedDate",
+  };
+
+  SORT_ORDER = {
+    ASCENDING: "ascending",
+    DESCENDING: "descending",
+  };
+
+  schema = z.object({
+    searchQuery: z
+      .string()
+      .describe("same as the search_query parameter rules of the arxiv API."),
+    sortBy: z
+      .string()
+      .describe('can be "relevance", "lastUpdatedDate", "submittedDate".'),
+    sortOrder: z
+      .string()
+      .describe('can be either "ascending" or "descending".'),
+    start: z
+      .number()
+      .default(0)
+      .describe("the index of the first returned result."),
+    maxResults: z
+      .number()
+      .default(10)
+      .describe("the number of results returned by the query."),
+  });
+
+  async _call({
+    searchQuery,
+    sortBy,
+    sortOrder,
+    start,
+    maxResults,
+  }: z.infer<typeof this.schema>) {
+    if (sortBy && !Object.values(this.SORT_BY).includes(sortBy)) {
+      throw new Error(
+        `unsupported sort by option. should be one of: ${Object.values(
+          this.SORT_BY,
+        ).join(" ")}`,
+      );
+    }
+    if (sortOrder && !Object.values(this.SORT_ORDER).includes(sortOrder)) {
+      throw new Error(
+        `unsupported sort order option. should be one of: ${Object.values(
+          this.SORT_ORDER,
+        ).join(" ")}`,
+      );
+    }
+    try {
+      let url = `http://export.arxiv.org/api/query?search_query=${searchQuery}&start=${start}&max_results=${maxResults}${
+        sortBy ? `&sortBy=${sortBy}` : ""
+      }${sortOrder ? `&sortOrder=${sortOrder}` : ""}`;
+      console.error("[arxiv]", url);
+      const response = await fetch(url);
+      const data = await response.text();
+      return data;
+    } catch (e) {
+      console.error("[arxiv]", e);
+    }
+    return "not found";
+  }
+}

app/api/langchain/tool/agent/route.ts

@@ -22,6 +22,7 @@ import { DallEAPIWrapper } from "@/app/api/langchain-tools/dalle_image_generator
 import { BaiduSearch } from "@/app/api/langchain-tools/baidu_search";
 import { GoogleSearch } from "@/app/api/langchain-tools/google_search";
 import { StableDiffusionWrapper } from "@/app/api/langchain-tools/stable_diffusion_image_generator";
+import { ArxivAPIWrapper } from "@/app/api/langchain-tools/arxiv";
 
 const serverConfig = getServerSideConfig();
 
@@ -95,8 +96,8 @@ async function handle(req: NextRequest) {
 
     const handler = BaseCallbackHandler.fromMethods({
       async handleLLMNewToken(token: string) {
+        // console.log("[Token]", token);
         if (token) {
-          console.log("[Token]", token);
           var response = new ResponseBody();
           response.message = token;
           await writer.ready;
@@ -151,8 +152,7 @@ async function handle(req: NextRequest) {
           if (!reqBody.returnIntermediateSteps) return;
           var response = new ResponseBody();
           response.isToolMessage = true;
-          let toolInput = <ToolInput>(<unknown>action.toolInput);
+          response.message = JSON.stringify(action.toolInput);
-          response.message = toolInput.input;
           response.toolName = action.tool;
           await writer.ready;
           await writer.write(
@@ -230,12 +230,14 @@ async function handle(req: NextRequest) {
     const calculatorTool = new Calculator();
     const dallEAPITool = new DallEAPIWrapper(apiKey, baseUrl);
     const stableDiffusionTool = new StableDiffusionWrapper();
+    const arxivAPITool = new ArxivAPIWrapper();
     if (useTools.includes("web-search")) tools.push(searchTool);
     if (useTools.includes(webBrowserTool.name)) tools.push(webBrowserTool);
     if (useTools.includes(calculatorTool.name)) tools.push(calculatorTool);
     if (useTools.includes(dallEAPITool.name)) tools.push(dallEAPITool);
     if (useTools.includes(stableDiffusionTool.name))
       tools.push(stableDiffusionTool);
+    if (useTools.includes(arxivAPITool.name)) tools.push(arxivAPITool);
 
     useTools.forEach((toolName) => {
       if (toolName) {

app/plugins/cn.ts

@@ -58,4 +58,13 @@ export const CN_PLUGINS: BuiltinPlugin[] = [
     createdAt: 1688899480510,
     enable: false,
   },
+  {
+    name: "Arxiv",
+    toolName: "arxiv",
+    lang: "cn",
+    description: "使用 Arxiv 接口搜索并获取文章信息。",
+    builtin: true,
+    createdAt: 1699265115000,
+    enable: false,
+  },
 ];

app/plugins/en.ts

@@ -60,4 +60,13 @@ export const EN_PLUGINS: BuiltinPlugin[] = [
     createdAt: 1688899480510,
     enable: false,
   },
+  {
+    name: "Arxiv",
+    toolName: "arxiv",
+    lang: "en",
+    description: "Arxiv search and get the article information.",
+    builtin: true,
+    createdAt: 1699265115000,
+    enable: false,
+  },
 ];

yarn.lock

@@ -16,9 +16,9 @@
     "@jridgewell/trace-mapping" "^0.3.9"
 
 "@anthropic-ai/sdk@^0.6.2":
-  version "0.6.2"
+  version "0.6.8"
-  resolved "https://registry.yarnpkg.com/@anthropic-ai/sdk/-/sdk-0.6.2.tgz#4be415e6b1d948df6f8e03af84aedf102ec74b70"
+  resolved "https://registry.yarnpkg.com/@anthropic-ai/sdk/-/sdk-0.6.8.tgz#670ecb3275e5f63f19f6decdd2ca94f4f8ac5f05"
-  integrity sha512-fB9PUj9RFT+XjkL+E9Ol864ZIJi+1P8WnbHspN3N3/GK2uSzjd0cbVIKTGgf4v3N8MwaQu+UWnU7C4BG/fap/g==
+  integrity sha512-z4gDFrBf+W2wOVvwA3CA+5bfKOxQhPeXQo7+ITWj3r3XPulIMEasVT0KrD41G+anr5Yc3d2PKvXKB6b1LSon5w==
   dependencies:
     "@types/node" "^18.11.18"
     "@types/node-fetch" "^2.6.4"
@@ -28,6 +28,7 @@
     form-data-encoder "1.7.2"
     formdata-node "^4.3.2"
     node-fetch "^2.6.7"
+    web-streams-polyfill "^3.2.1"
 
 "@aws-crypto/crc32@3.0.0":
   version "3.0.0"
@@ -2551,24 +2552,26 @@
   integrity sha512-iiUgKzV9AuaEkZqkOLDIvlQiL6ltuZd9tGcW3gwpnX8JbuiuhFlEGmmFXEXkN50Cvq7Os88IY2v0dkDqXYWVgA==
 
 "@types/node-fetch@^2.6.4":
-  version "2.6.6"
+  version "2.6.8"
-  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.6.tgz#b72f3f4bc0c0afee1c0bc9cff68e041d01e3e779"
+  resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.6.8.tgz#9a2993583975849c2e1f360b6ca2f11755b2c504"
-  integrity sha512-95X8guJYhfqiuVVhRFxVQcf4hW/2bCuoPwDasMf/531STFoNoWTT7YDnWdXHEZKqAGUigmpG31r2FE70LwnzJw==
+  integrity sha512-nnH5lV9QCMPsbEVdTb5Y+F3GQxLSw1xQgIydrb2gSfEavRPs50FnMr+KUaa+LoPSqibm2N+ZZxH7lavZlAT4GA==
   dependencies:
     "@types/node" "*"
     form-data "^4.0.0"
 
 "@types/node@*":
-  version "20.8.5"
+  version "20.8.10"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.8.5.tgz#13352ae1f80032171616910e8aba2e3e52e57d96"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.8.10.tgz#a5448b895c753ae929c26ce85cab557c6d4a365e"
-  integrity sha512-SPlobFgbidfIeOYlzXiEjSYeIJiOCthv+9tSQVpvk4PAdIIc+2SmjNVzWXk9t0Y7dl73Zdf+OgXKHX9XtkqUpw==
+  integrity sha512-TlgT8JntpcbmKUFzjhsyhGfP2fsiz1Mv56im6enJ905xG1DAYesxJaeSbGqQmAw8OWPdhyJGhGSQGKRNJ45u9w==
   dependencies:
-    undici-types "~5.25.1"
+    undici-types "~5.26.4"
 
 "@types/node@^18.11.18":
-  version "18.18.5"
+  version "18.18.8"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.18.5.tgz#afc0fd975df946d6e1add5bbf98264225b212244"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.18.8.tgz#2b285361f2357c8c8578ec86b5d097c7f464cfd6"
-  integrity sha512-4slmbtwV59ZxitY4ixUZdy1uRLf9eSIvBWPQxNjhHYWEtn0FryfKpyS2cvADYXTayWdKEIsJengncrVvkI4I6A==
+  integrity sha512-OLGBaaK5V3VRBS1bAkMVP2/W9B+H8meUfl866OrMNQqt7wDgdpWPp5o6gmIc9pB+lIQHSq4ZL8ypeH1vPxcPaQ==
+  dependencies:
+    undici-types "~5.26.4"
 
 "@types/node@^20.3.3":
   version "20.3.3"
@@ -2634,9 +2637,9 @@
   integrity sha512-EwmlvuaxPNej9+T4v5AuBPJa2x2UOJVdjCtDHgcDqitUeOtjnJKJ+apYjVcAoBEMjKW1VVFGZLUb5+qqa09XFA==
 
 "@types/uuid@^9.0.1":
-  version "9.0.5"
+  version "9.0.6"
-  resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.5.tgz#25a71eb73eba95ac0e559ff3dd018fc08294acf6"
+  resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-9.0.6.tgz#c91ae743d8344a54b2b0c691195f5ff5265f6dfb"
-  integrity sha512-xfHdwa1FMJ082prjSJpoEI57GZITiQz10r3vEJCHa2khEFQjKy91aWKz6+zybzssCvXUwE1LQWgWVwZ4nYUvHQ==
+  integrity sha512-BT2Krtx4xaO6iwzwMFUYvWBWkV2pr37zD68Vmp1CDV196MzczBRxuEpD6Pr395HAgebC/co7hOphs53r8V7jew==
 
 "@typescript-eslint/parser@^5.4.2 || ^6.0.0":
   version "6.4.0"
@@ -5397,9 +5400,9 @@ langchainhub@~0.0.6:
   integrity sha512-SW6105T+YP1cTe0yMf//7kyshCgvCTyFBMTgH2H3s9rTAR4e+78DA/BBrUL/Mt4Q5eMWui7iGuAYb3pgGsdQ9w==
 
 langsmith@~0.0.31:
-  version "0.0.42"
+  version "0.0.48"
-  resolved "https://registry.yarnpkg.com/langsmith/-/langsmith-0.0.42.tgz#e20e3e261c87282ec5ba6342c1f4ee19f10b91a0"
+  resolved "https://registry.yarnpkg.com/langsmith/-/langsmith-0.0.48.tgz#3a9a8ce257271ddb43d01ebf585c4370a3a3ba79"
-  integrity sha512-sFuN+e7E+pPBIRaRgFqZh/BRBWNHTZNAwi6uj4kydQawooCZYoJmM5snOkiQrhVSvAhgu6xFhLvmfvkPcKzD7w==
+  integrity sha512-s0hW8iZ90Q9XLTnDK0Pgee245URV3b1cXQjPDj5OKm1+KN7iSK1pKx+4CO7RcFLz58Ixe7Mt+mVcomYqUuryxQ==
   dependencies:
     "@types/uuid" "^9.0.1"
     commander "^10.0.1"
@@ -7414,10 +7417,10 @@ unbox-primitive@^1.0.2:
     has-symbols "^1.0.3"
     which-boxed-primitive "^1.0.2"
 
-undici-types@~5.25.1:
+undici-types@~5.26.4:
-  version "5.25.3"
+  version "5.26.5"
-  resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.25.3.tgz#e044115914c85f0bcbb229f346ab739f064998c3"
+  resolved "https://registry.yarnpkg.com/undici-types/-/undici-types-5.26.5.tgz#bcd539893d00b56e964fd2657a4866b221a65617"
-  integrity sha512-Ga1jfYwRn7+cP9v8auvEXN1rX3sWqlayd4HP7OKk4mZWylEmu3KzXDUGrQUN6Ol7qo1gPvB2e5gX6udnyEPgdA==
+  integrity sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==
 
 unicode-canonical-property-names-ecmascript@^2.0.0:
   version "2.0.0"
@@ -7616,7 +7619,7 @@ web-streams-polyfill@4.0.0-beta.3:
   resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-4.0.0-beta.3.tgz#2898486b74f5156095e473efe989dcf185047a38"
   integrity sha512-QW95TCTaHmsYfHDybGMwO5IJIM93I/6vTRk+daHTWFPhwh+C8Cg7j7XyKrwrj8Ib6vYXe0ocYNrmzY4xAAN6ug==
 
-web-streams-polyfill@^3.0.3:
+web-streams-polyfill@^3.0.3, web-streams-polyfill@^3.2.1:
   version "3.2.1"
   resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.2.1.tgz#71c2718c52b45fd49dbeee88634b3a60ceab42a6"
   integrity sha512-e0MO3wdXWKrLbL0DgGnUV7WHVuw9OUvL4hjgnPkIeEvESk74gAITi5G606JtZPp39cd8HA9VQzCIvA49LpPN5Q==
@@ -7753,9 +7756,9 @@ yaml@^1.10.0:
   integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==
 
 yaml@^2.2.1:
-  version "2.3.2"
+  version "2.3.4"
-  resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.3.2.tgz#f522db4313c671a0ca963a75670f1c12ea909144"
+  resolved "https://registry.yarnpkg.com/yaml/-/yaml-2.3.4.tgz#53fc1d514be80aabf386dc6001eb29bf3b7523b2"
-  integrity sha512-N/lyzTPaJasoDmfV7YTrYCI0G/3ivm/9wdG0aHuheKowWQwGTsK0Eoiw6utmzAnI6pkJa0DUVygvp3spqqEKXg==
+  integrity sha512-8aAvwVUSHpfEqTQ4w/KMlf3HcRdt50E5ODIQJBw1fQ5RL34xabzxtUlzTXVqc4rkZsPbvrXKWnABCD7kWSmocA==
 
 yaml@^2.2.2:
   version "2.3.1"