feat: support tool call

app/api/langchain/tool/agent/agentapi.ts

@@ -7,7 +7,10 @@ import { BaseCallbackHandler } from "langchain/callbacks";
 
 import { AIMessage, HumanMessage, SystemMessage } from "langchain/schema";
 import { BufferMemory, ChatMessageHistory } from "langchain/memory";
-import { initializeAgentExecutorWithOptions } from "langchain/agents";
+import {
+  AgentExecutor,
+  initializeAgentExecutorWithOptions,
+} from "langchain/agents";
 import { ACCESS_CODE_PREFIX, ServiceProvider } from "@/app/constant";
 
 import * as langchainTools from "langchain/tools";
@@ -17,6 +20,14 @@ import { DynamicTool, Tool } from "langchain/tools";
 import { BaiduSearch } from "@/app/api/langchain-tools/baidu_search";
 import { GoogleSearch } from "@/app/api/langchain-tools/google_search";
 import { useAccessStore } from "@/app/store";
+import { DynamicStructuredTool, formatToOpenAITool } from "langchain/tools";
+import { formatToOpenAIToolMessages } from "langchain/agents/format_scratchpad/openai_tools";
+import {
+  OpenAIToolsAgentOutputParser,
+  type ToolsAgentStep,
+} from "langchain/agents/openai/output_parser";
+import { RunnableSequence } from "langchain/schema/runnable";
+import { ChatPromptTemplate, MessagesPlaceholder } from "langchain/prompts";
 
 export interface RequestMessage {
   role: string;
@@ -92,9 +103,9 @@ export class AgentApi {
         await writer.close();
       },
       async handleChainEnd(outputs, runId, parentRunId, tags) {
-        console.log("[handleChainEnd]");
+        // console.log("[handleChainEnd]");
-        await writer.ready;
+        // await writer.ready;
-        await writer.close();
+        // await writer.close();
       },
       async handleLLMEnd() {
         // await writer.ready;
@@ -111,10 +122,10 @@ export class AgentApi {
         );
         await writer.close();
       },
-      handleLLMStart(llm, _prompts: string[]) {
+      async handleLLMStart(llm, _prompts: string[]) {
         // console.log("handleLLMStart: I'm the second handler!!", { llm });
       },
-      handleChainStart(chain) {
+      async handleChainStart(chain) {
         // console.log("handleChainStart: I'm the second handler!!", { chain });
       },
       async handleAgentAction(action) {
@@ -141,14 +152,16 @@ export class AgentApi {
           await writer.close();
         }
       },
-      handleToolStart(tool, input) {
+      async handleToolStart(tool, input) {
         // console.log("[handleToolStart]", { tool });
       },
       async handleToolEnd(output, runId, parentRunId, tags) {
         // console.log("[handleToolEnd]", { output, runId, parentRunId, tags });
       },
-      handleAgentEnd(action, runId, parentRunId, tags) {
+      async handleAgentEnd(action, runId, parentRunId, tags) {
-        // console.log("[handleAgentEnd]");
+        console.log("[handleAgentEnd]");
+        await writer.ready;
+        await writer.close();
       },
     });
   }
@@ -288,13 +301,13 @@ export class AgentApi {
             pastMessages.push(new AIMessage(message.content));
         });
 
-      const memory = new BufferMemory({
+      // const memory = new BufferMemory({
-        memoryKey: "chat_history",
+      //   memoryKey: "chat_history",
-        returnMessages: true,
+      //   returnMessages: true,
-        inputKey: "input",
+      //   inputKey: "input",
-        outputKey: "output",
+      //   outputKey: "output",
-        chatHistory: new ChatMessageHistory(pastMessages),
+      //   chatHistory: new ChatMessageHistory(pastMessages),
-      });
+      // });
 
       let llm = new ChatOpenAI(
         {
@@ -324,13 +337,48 @@ export class AgentApi {
           azureOpenAIBasePath: baseUrl,
         });
       }
-
+      const memory = new BufferMemory({
-      const executor = await initializeAgentExecutorWithOptions(tools, llm, {
+        memoryKey: "history",
-        agentType: "openai-functions",
+        inputKey: "question",
-        returnIntermediateSteps: reqBody.returnIntermediateSteps,
+        outputKey: "answer",
-        maxIterations: reqBody.maxIterations,
+        returnMessages: true,
-        memory: memory,
+        chatHistory: new ChatMessageHistory(pastMessages),
       });
+      const prompt = ChatPromptTemplate.fromMessages([
+        new MessagesPlaceholder("chat_history"),
+        ["human", "{input}"],
+        new MessagesPlaceholder("agent_scratchpad"),
+      ]);
+      const modelWithTools = llm.bind({ tools: tools.map(formatToOpenAITool) });
+      const runnableAgent = RunnableSequence.from([
+        {
+          input: (i: { input: string; steps: ToolsAgentStep[] }) => i.input,
+          agent_scratchpad: (i: { input: string; steps: ToolsAgentStep[] }) =>
+            formatToOpenAIToolMessages(i.steps),
+          chat_history: async (_: {
+            input: string;
+            steps: ToolsAgentStep[];
+          }) => {
+            const { history } = await memory.loadMemoryVariables({});
+            return history;
+          },
+        },
+        prompt,
+        modelWithTools,
+        new OpenAIToolsAgentOutputParser(),
+      ]).withConfig({ runName: "OpenAIToolsAgent" });
+
+      const executor = AgentExecutor.fromAgentAndTools({
+        agent: runnableAgent,
+        tools,
+      });
+
+      // const executor = await initializeAgentExecutorWithOptions(tools, llm, {
+      //   agentType: "openai-functions",
+      //   returnIntermediateSteps: reqBody.returnIntermediateSteps,
+      //   maxIterations: reqBody.maxIterations,
+      //   memory: memory,
+      // });
 
       executor.call(
         {

app/client/platforms/google.ts

@@ -16,6 +16,7 @@ import { prettyObject } from "@/app/utils/format";
 import { getClientConfig } from "@/app/config/client";
 import Locale from "../../locales";
 import { getServerSideConfig } from "@/app/config/server";
+
 export class GeminiProApi implements LLMApi {
   toolAgentChat(options: AgentChatOptions): Promise<void> {
     throw new Error("Method not implemented.");

package.json

@@ -27,12 +27,12 @@
     "duck-duck-scrape": "^2.2.4",
     "emoji-picker-react": "^4.5.15",
     "encoding": "^0.1.13",
-    "html-entities": "^2.4.0",
     "fuse.js": "^7.0.0",
+    "html-entities": "^2.4.0",
     "html-to-image": "^1.11.11",
     "html-to-text": "^9.0.5",
     "https-proxy-agent": "^7.0.2",
-    "langchain": "^0.0.210",
+    "langchain": "^0.0.212",
     "mermaid": "^10.6.1",
     "nanoid": "^5.0.3",
     "next": "^13.4.9",
@@ -54,8 +54,8 @@
     "zustand": "^4.3.8"
   },
   "devDependencies": {
-    "@types/html-to-text": "^9.0.1",
     "@tauri-apps/cli": "^1.5.8",
+    "@types/html-to-text": "^9.0.1",
     "@types/node": "^20.9.0",
     "@types/react": "^18.2.14",
     "@types/react-dom": "^18.2.7",

yarn.lock

@@ -1814,9 +1814,9 @@
     "@jridgewell/sourcemap-codec" "1.4.14"
 
 "@langchain/community@~0.0.8":
-  version "0.0.8"
+  version "0.0.11"
-  resolved "https://registry.yarnpkg.com/@langchain/community/-/community-0.0.8.tgz#3427321a3262cdb362d22be79d6cc6ee1904a16f"
+  resolved "https://registry.yarnpkg.com/@langchain/community/-/community-0.0.11.tgz#5e1dce3e1ec636e015c503593fdf7383d4478159"
-  integrity sha512-nBJiEQgAFy1Wovyoxcl48rK8LC0L1HC/gN5kplf8tVSBQEpMjHsGAWBN3PlXMhJj+JNX/4wcqcfMsyCLkgC2wg==
+  integrity sha512-8L19AyzBueHrRtawxU4JstRtmzqsHK2jDJtSg/bv9ivto0kDKlILnauIDijcb09V/4mql2XolaS9r7eGe99yGA==
   dependencies:
     "@langchain/core" "~0.1.3"
     "@langchain/openai" "~0.0.7"
@@ -1826,9 +1826,9 @@
     zod "^3.22.3"
 
 "@langchain/core@~0.1.3":
-  version "0.1.3"
+  version "0.1.4"
-  resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.1.3.tgz#6415ed458e70b5a2414c2be7c870dd0d3f25c913"
+  resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.1.4.tgz#f923c0d844faf6de2069d86d785e4ef3b1381257"
-  integrity sha512-nsQbakY0P0ErBSzlFf1HsgNXSAxQNYLfzNkcqpEEr4kCH0PMw5lmyROYN9LMds+JXhM2/AOE/VP4HYN3WlxaJA==
+  integrity sha512-Y3/mQLEiQ78ZbsTGYvPRj5bpvrhpTAcsbdyEtlYEvjMLbehEADfjQ41G9zc7U/emkrGtHRmxWO1ISmoeXeDmyQ==
   dependencies:
     ansi-styles "^5.0.0"
     camelcase "6"
@@ -1842,9 +1842,9 @@
     zod "^3.22.3"
 
 "@langchain/openai@~0.0.7":
-  version "0.0.7"
+  version "0.0.8"
-  resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.0.7.tgz#9615a7cc61b3f9a10006de3cfb888e448dd8c870"
+  resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.0.8.tgz#77e4a216f9ba5a347ee24b7f28a526fd57c24c8d"
-  integrity sha512-m/UjOf9SdIZhoR3RALgUS78+v4r/RJQhyQbvGLbaCcAwbCFjUohmESW6Y1n5dIhwk5rVazprG2oL0O1ZSAwrgw==
+  integrity sha512-F8DD6vs5BvMAHiDprINWCVf7jyNaYHp2zAqHwYij3+YAKsgIORm2EQvI6YtT3qPnGghpE3qz8H3DKjTNPywIzQ==
   dependencies:
     "@langchain/core" "~0.1.3"
     js-tiktoken "^1.0.7"
@@ -2686,16 +2686,16 @@
     form-data "^4.0.0"
 
 "@types/node@*":
-  version "20.9.4"
+  version "20.10.5"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.9.4.tgz#cc8f970e869c26834bdb7ed480b30ede622d74c7"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-20.10.5.tgz#47ad460b514096b7ed63a1dae26fad0914ed3ab2"
-  integrity sha512-wmyg8HUhcn6ACjsn8oKYjkN/zUzQeNtMy44weTJSM6p4MMzEOuKbA3OjJ267uPCOW7Xex9dyrNTful8XTQYoDA==
+  integrity sha512-nNPsNE65wjMxEKI93yOP+NPGGBJz/PoN3kZsVLee0XMiJolxSekEVD8wRwBUBqkwc7UWop0edW50yrCQW4CyRw==
   dependencies:
     undici-types "~5.26.4"
 
 "@types/node@^18.11.18":
-  version "18.18.12"
+  version "18.19.3"
-  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.18.12.tgz#0c40e52e5ff2569386b160f6f6bb019ff1361cb4"
+  resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.3.tgz#e4723c4cb385641d61b983f6fe0b716abd5f8fc0"
-  integrity sha512-G7slVfkwOm7g8VqcEF1/5SXiMjP3Tbt+pXDU3r/qhlM2KkGm786DUD4xyMA2QzEElFrv/KZV9gjygv4LnkpbMQ==
+  integrity sha512-k5fggr14DwAytoA/t8rPrIz++lXK7/DqckthCmoZOKNsEbJkId4Z//BqgApXBUGrGddrigYa1oqheo/7YmW4rg==
   dependencies:
     undici-types "~5.26.4"
 
@@ -5487,10 +5487,10 @@ kleur@^4.0.3:
   resolved "https://registry.yarnpkg.com/kleur/-/kleur-4.1.5.tgz#95106101795f7050c6c650f350c683febddb1780"
   integrity sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ==
 
-langchain@^0.0.210:
+langchain@^0.0.212:
-  version "0.0.210"
+  version "0.0.212"
-  resolved "https://registry.yarnpkg.com/langchain/-/langchain-0.0.210.tgz#8eaac00bf70985231904a8d9acdcba7926b285f9"
+  resolved "https://registry.yarnpkg.com/langchain/-/langchain-0.0.212.tgz#68ddeb6bb83bc7714a6b8d97ace464951d10ea6d"
-  integrity sha512-5DTf3VlsTVV+I4aQ8sj6DPYaHjT1cgYYlXNMVrQ7/2oG2b3CFBUbH4svn+LVc0aoZx5yneIY71PGaMVh4nYJHA==
+  integrity sha512-zPcnAX3t1RuyWkbEkyqlIdR29j51Eylz7rcDcqET1baquR/r9xuk7P/HAJ6TFowiXDUg10d5YBD6VBvjwF/WDA==
   dependencies:
     "@anthropic-ai/sdk" "^0.9.1"
     "@langchain/community" "~0.0.8"
@@ -5517,9 +5517,9 @@ langchainhub@~0.0.6:
   integrity sha512-SW6105T+YP1cTe0yMf//7kyshCgvCTyFBMTgH2H3s9rTAR4e+78DA/BBrUL/Mt4Q5eMWui7iGuAYb3pgGsdQ9w==
 
 langsmith@~0.0.48:
-  version "0.0.48"
+  version "0.0.53"
-  resolved "https://registry.yarnpkg.com/langsmith/-/langsmith-0.0.48.tgz#3a9a8ce257271ddb43d01ebf585c4370a3a3ba79"
+  resolved "https://registry.yarnpkg.com/langsmith/-/langsmith-0.0.53.tgz#70e8bbbb6fdde8a5217321a0d47199cc15699d9e"
-  integrity sha512-s0hW8iZ90Q9XLTnDK0Pgee245URV3b1cXQjPDj5OKm1+KN7iSK1pKx+4CO7RcFLz58Ixe7Mt+mVcomYqUuryxQ==
+  integrity sha512-w2Rgx4ixE+wuItFVKGnc+Wmzj91RevMd7sev9BHWC8VFztDPuyrNZQD55QpxphM6JLb9hF9osxvAiYDGtDZbuQ==
   dependencies:
     "@types/uuid" "^9.0.1"
     commander "^10.0.1"
@@ -6487,9 +6487,9 @@ open@^8.4.0:
     is-wsl "^2.2.0"
 
 openai@^4.19.0:
-  version "4.19.1"
+  version "4.24.1"
-  resolved "https://registry.yarnpkg.com/openai/-/openai-4.19.1.tgz#229d6e994248966f255f6a5b849dcec28e3b3439"
+  resolved "https://registry.yarnpkg.com/openai/-/openai-4.24.1.tgz#3759001eca835228289fcf18c1bd8d35dae538ba"
-  integrity sha512-9TddzuZBn2xxhghGGTHLZ4EeNBGTLs3xVzh266NiSJvtUsCsZQ5yVV6H5NhnhyAkKK8uUiZOUUlUAk3HdV+4xg==
+  integrity sha512-ezm/O3eiZMnyBqirUnWm9N6INJU1WhNtz+nK/Zj/2oyKvRz9pgpViDxa5wYOtyGYXPn1sIKBV0I/S4BDhtydqw==
   dependencies:
     "@types/node" "^18.11.18"
     "@types/node-fetch" "^2.6.4"