From 8cdbc231ca151d27ae32d1f8338d48d5100c1c8e Mon Sep 17 00:00:00 2001 From: Hk-Gosuto Date: Fri, 29 Dec 2023 09:43:37 +0800 Subject: [PATCH] feat: support tool call --- app/api/langchain/tool/agent/agentapi.ts | 92 ++++++++++++++++++------ app/client/platforms/google.ts | 1 + package.json | 6 +- yarn.lock | 50 ++++++------- 4 files changed, 99 insertions(+), 50 deletions(-) diff --git a/app/api/langchain/tool/agent/agentapi.ts b/app/api/langchain/tool/agent/agentapi.ts index 147dbbc7e..b66c0c80c 100644 --- a/app/api/langchain/tool/agent/agentapi.ts +++ b/app/api/langchain/tool/agent/agentapi.ts @@ -7,7 +7,10 @@ import { BaseCallbackHandler } from "langchain/callbacks"; import { AIMessage, HumanMessage, SystemMessage } from "langchain/schema"; import { BufferMemory, ChatMessageHistory } from "langchain/memory"; -import { initializeAgentExecutorWithOptions } from "langchain/agents"; +import { + AgentExecutor, + initializeAgentExecutorWithOptions, +} from "langchain/agents"; import { ACCESS_CODE_PREFIX, ServiceProvider } from "@/app/constant"; import * as langchainTools from "langchain/tools"; @@ -17,6 +20,14 @@ import { DynamicTool, Tool } from "langchain/tools"; import { BaiduSearch } from "@/app/api/langchain-tools/baidu_search"; import { GoogleSearch } from "@/app/api/langchain-tools/google_search"; import { useAccessStore } from "@/app/store"; +import { DynamicStructuredTool, formatToOpenAITool } from "langchain/tools"; +import { formatToOpenAIToolMessages } from "langchain/agents/format_scratchpad/openai_tools"; +import { + OpenAIToolsAgentOutputParser, + type ToolsAgentStep, +} from "langchain/agents/openai/output_parser"; +import { RunnableSequence } from "langchain/schema/runnable"; +import { ChatPromptTemplate, MessagesPlaceholder } from "langchain/prompts"; export interface RequestMessage { role: string; @@ -92,9 +103,9 @@ export class AgentApi { await writer.close(); }, async handleChainEnd(outputs, runId, parentRunId, tags) { - console.log("[handleChainEnd]"); - await writer.ready; - await writer.close(); + // console.log("[handleChainEnd]"); + // await writer.ready; + // await writer.close(); }, async handleLLMEnd() { // await writer.ready; @@ -111,10 +122,10 @@ export class AgentApi { ); await writer.close(); }, - handleLLMStart(llm, _prompts: string[]) { + async handleLLMStart(llm, _prompts: string[]) { // console.log("handleLLMStart: I'm the second handler!!", { llm }); }, - handleChainStart(chain) { + async handleChainStart(chain) { // console.log("handleChainStart: I'm the second handler!!", { chain }); }, async handleAgentAction(action) { @@ -141,14 +152,16 @@ export class AgentApi { await writer.close(); } }, - handleToolStart(tool, input) { + async handleToolStart(tool, input) { // console.log("[handleToolStart]", { tool }); }, async handleToolEnd(output, runId, parentRunId, tags) { // console.log("[handleToolEnd]", { output, runId, parentRunId, tags }); }, - handleAgentEnd(action, runId, parentRunId, tags) { - // console.log("[handleAgentEnd]"); + async handleAgentEnd(action, runId, parentRunId, tags) { + console.log("[handleAgentEnd]"); + await writer.ready; + await writer.close(); }, }); } @@ -288,13 +301,13 @@ export class AgentApi { pastMessages.push(new AIMessage(message.content)); }); - const memory = new BufferMemory({ - memoryKey: "chat_history", - returnMessages: true, - inputKey: "input", - outputKey: "output", - chatHistory: new ChatMessageHistory(pastMessages), - }); + // const memory = new BufferMemory({ + // memoryKey: "chat_history", + // returnMessages: true, + // inputKey: "input", + // outputKey: "output", + // chatHistory: new ChatMessageHistory(pastMessages), + // }); let llm = new ChatOpenAI( { @@ -324,13 +337,48 @@ export class AgentApi { azureOpenAIBasePath: baseUrl, }); } - - const executor = await initializeAgentExecutorWithOptions(tools, llm, { - agentType: "openai-functions", - returnIntermediateSteps: reqBody.returnIntermediateSteps, - maxIterations: reqBody.maxIterations, - memory: memory, + const memory = new BufferMemory({ + memoryKey: "history", + inputKey: "question", + outputKey: "answer", + returnMessages: true, + chatHistory: new ChatMessageHistory(pastMessages), }); + const prompt = ChatPromptTemplate.fromMessages([ + new MessagesPlaceholder("chat_history"), + ["human", "{input}"], + new MessagesPlaceholder("agent_scratchpad"), + ]); + const modelWithTools = llm.bind({ tools: tools.map(formatToOpenAITool) }); + const runnableAgent = RunnableSequence.from([ + { + input: (i: { input: string; steps: ToolsAgentStep[] }) => i.input, + agent_scratchpad: (i: { input: string; steps: ToolsAgentStep[] }) => + formatToOpenAIToolMessages(i.steps), + chat_history: async (_: { + input: string; + steps: ToolsAgentStep[]; + }) => { + const { history } = await memory.loadMemoryVariables({}); + return history; + }, + }, + prompt, + modelWithTools, + new OpenAIToolsAgentOutputParser(), + ]).withConfig({ runName: "OpenAIToolsAgent" }); + + const executor = AgentExecutor.fromAgentAndTools({ + agent: runnableAgent, + tools, + }); + + // const executor = await initializeAgentExecutorWithOptions(tools, llm, { + // agentType: "openai-functions", + // returnIntermediateSteps: reqBody.returnIntermediateSteps, + // maxIterations: reqBody.maxIterations, + // memory: memory, + // }); executor.call( { diff --git a/app/client/platforms/google.ts b/app/client/platforms/google.ts index 83619774d..3b20a6a39 100644 --- a/app/client/platforms/google.ts +++ b/app/client/platforms/google.ts @@ -16,6 +16,7 @@ import { prettyObject } from "@/app/utils/format"; import { getClientConfig } from "@/app/config/client"; import Locale from "../../locales"; import { getServerSideConfig } from "@/app/config/server"; + export class GeminiProApi implements LLMApi { toolAgentChat(options: AgentChatOptions): Promise { throw new Error("Method not implemented."); diff --git a/package.json b/package.json index 9203e217c..b10b026f1 100644 --- a/package.json +++ b/package.json @@ -27,12 +27,12 @@ "duck-duck-scrape": "^2.2.4", "emoji-picker-react": "^4.5.15", "encoding": "^0.1.13", - "html-entities": "^2.4.0", "fuse.js": "^7.0.0", + "html-entities": "^2.4.0", "html-to-image": "^1.11.11", "html-to-text": "^9.0.5", "https-proxy-agent": "^7.0.2", - "langchain": "^0.0.210", + "langchain": "^0.0.212", "mermaid": "^10.6.1", "nanoid": "^5.0.3", "next": "^13.4.9", @@ -54,8 +54,8 @@ "zustand": "^4.3.8" }, "devDependencies": { - "@types/html-to-text": "^9.0.1", "@tauri-apps/cli": "^1.5.8", + "@types/html-to-text": "^9.0.1", "@types/node": "^20.9.0", "@types/react": "^18.2.14", "@types/react-dom": "^18.2.7", diff --git a/yarn.lock b/yarn.lock index 553b6e3f5..23dbe0462 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1814,9 +1814,9 @@ "@jridgewell/sourcemap-codec" "1.4.14" "@langchain/community@~0.0.8": - version "0.0.8" - resolved "https://registry.yarnpkg.com/@langchain/community/-/community-0.0.8.tgz#3427321a3262cdb362d22be79d6cc6ee1904a16f" - integrity sha512-nBJiEQgAFy1Wovyoxcl48rK8LC0L1HC/gN5kplf8tVSBQEpMjHsGAWBN3PlXMhJj+JNX/4wcqcfMsyCLkgC2wg== + version "0.0.11" + resolved "https://registry.yarnpkg.com/@langchain/community/-/community-0.0.11.tgz#5e1dce3e1ec636e015c503593fdf7383d4478159" + integrity sha512-8L19AyzBueHrRtawxU4JstRtmzqsHK2jDJtSg/bv9ivto0kDKlILnauIDijcb09V/4mql2XolaS9r7eGe99yGA== dependencies: "@langchain/core" "~0.1.3" "@langchain/openai" "~0.0.7" @@ -1826,9 +1826,9 @@ zod "^3.22.3" "@langchain/core@~0.1.3": - version "0.1.3" - resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.1.3.tgz#6415ed458e70b5a2414c2be7c870dd0d3f25c913" - integrity sha512-nsQbakY0P0ErBSzlFf1HsgNXSAxQNYLfzNkcqpEEr4kCH0PMw5lmyROYN9LMds+JXhM2/AOE/VP4HYN3WlxaJA== + version "0.1.4" + resolved "https://registry.yarnpkg.com/@langchain/core/-/core-0.1.4.tgz#f923c0d844faf6de2069d86d785e4ef3b1381257" + integrity sha512-Y3/mQLEiQ78ZbsTGYvPRj5bpvrhpTAcsbdyEtlYEvjMLbehEADfjQ41G9zc7U/emkrGtHRmxWO1ISmoeXeDmyQ== dependencies: ansi-styles "^5.0.0" camelcase "6" @@ -1842,9 +1842,9 @@ zod "^3.22.3" "@langchain/openai@~0.0.7": - version "0.0.7" - resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.0.7.tgz#9615a7cc61b3f9a10006de3cfb888e448dd8c870" - integrity sha512-m/UjOf9SdIZhoR3RALgUS78+v4r/RJQhyQbvGLbaCcAwbCFjUohmESW6Y1n5dIhwk5rVazprG2oL0O1ZSAwrgw== + version "0.0.8" + resolved "https://registry.yarnpkg.com/@langchain/openai/-/openai-0.0.8.tgz#77e4a216f9ba5a347ee24b7f28a526fd57c24c8d" + integrity sha512-F8DD6vs5BvMAHiDprINWCVf7jyNaYHp2zAqHwYij3+YAKsgIORm2EQvI6YtT3qPnGghpE3qz8H3DKjTNPywIzQ== dependencies: "@langchain/core" "~0.1.3" js-tiktoken "^1.0.7" @@ -2686,16 +2686,16 @@ form-data "^4.0.0" "@types/node@*": - version "20.9.4" - resolved "https://registry.yarnpkg.com/@types/node/-/node-20.9.4.tgz#cc8f970e869c26834bdb7ed480b30ede622d74c7" - integrity sha512-wmyg8HUhcn6ACjsn8oKYjkN/zUzQeNtMy44weTJSM6p4MMzEOuKbA3OjJ267uPCOW7Xex9dyrNTful8XTQYoDA== + version "20.10.5" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.10.5.tgz#47ad460b514096b7ed63a1dae26fad0914ed3ab2" + integrity sha512-nNPsNE65wjMxEKI93yOP+NPGGBJz/PoN3kZsVLee0XMiJolxSekEVD8wRwBUBqkwc7UWop0edW50yrCQW4CyRw== dependencies: undici-types "~5.26.4" "@types/node@^18.11.18": - version "18.18.12" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.18.12.tgz#0c40e52e5ff2569386b160f6f6bb019ff1361cb4" - integrity sha512-G7slVfkwOm7g8VqcEF1/5SXiMjP3Tbt+pXDU3r/qhlM2KkGm786DUD4xyMA2QzEElFrv/KZV9gjygv4LnkpbMQ== + version "18.19.3" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.19.3.tgz#e4723c4cb385641d61b983f6fe0b716abd5f8fc0" + integrity sha512-k5fggr14DwAytoA/t8rPrIz++lXK7/DqckthCmoZOKNsEbJkId4Z//BqgApXBUGrGddrigYa1oqheo/7YmW4rg== dependencies: undici-types "~5.26.4" @@ -5487,10 +5487,10 @@ kleur@^4.0.3: resolved "https://registry.yarnpkg.com/kleur/-/kleur-4.1.5.tgz#95106101795f7050c6c650f350c683febddb1780" integrity sha512-o+NO+8WrRiQEE4/7nwRJhN1HWpVmJm511pBHUxPLtp0BUISzlBplORYSmTclCnJvQq2tKu/sgl3xVpkc7ZWuQQ== -langchain@^0.0.210: - version "0.0.210" - resolved "https://registry.yarnpkg.com/langchain/-/langchain-0.0.210.tgz#8eaac00bf70985231904a8d9acdcba7926b285f9" - integrity sha512-5DTf3VlsTVV+I4aQ8sj6DPYaHjT1cgYYlXNMVrQ7/2oG2b3CFBUbH4svn+LVc0aoZx5yneIY71PGaMVh4nYJHA== +langchain@^0.0.212: + version "0.0.212" + resolved "https://registry.yarnpkg.com/langchain/-/langchain-0.0.212.tgz#68ddeb6bb83bc7714a6b8d97ace464951d10ea6d" + integrity sha512-zPcnAX3t1RuyWkbEkyqlIdR29j51Eylz7rcDcqET1baquR/r9xuk7P/HAJ6TFowiXDUg10d5YBD6VBvjwF/WDA== dependencies: "@anthropic-ai/sdk" "^0.9.1" "@langchain/community" "~0.0.8" @@ -5517,9 +5517,9 @@ langchainhub@~0.0.6: integrity sha512-SW6105T+YP1cTe0yMf//7kyshCgvCTyFBMTgH2H3s9rTAR4e+78DA/BBrUL/Mt4Q5eMWui7iGuAYb3pgGsdQ9w== langsmith@~0.0.48: - version "0.0.48" - resolved "https://registry.yarnpkg.com/langsmith/-/langsmith-0.0.48.tgz#3a9a8ce257271ddb43d01ebf585c4370a3a3ba79" - integrity sha512-s0hW8iZ90Q9XLTnDK0Pgee245URV3b1cXQjPDj5OKm1+KN7iSK1pKx+4CO7RcFLz58Ixe7Mt+mVcomYqUuryxQ== + version "0.0.53" + resolved "https://registry.yarnpkg.com/langsmith/-/langsmith-0.0.53.tgz#70e8bbbb6fdde8a5217321a0d47199cc15699d9e" + integrity sha512-w2Rgx4ixE+wuItFVKGnc+Wmzj91RevMd7sev9BHWC8VFztDPuyrNZQD55QpxphM6JLb9hF9osxvAiYDGtDZbuQ== dependencies: "@types/uuid" "^9.0.1" commander "^10.0.1" @@ -6487,9 +6487,9 @@ open@^8.4.0: is-wsl "^2.2.0" openai@^4.19.0: - version "4.19.1" - resolved "https://registry.yarnpkg.com/openai/-/openai-4.19.1.tgz#229d6e994248966f255f6a5b849dcec28e3b3439" - integrity sha512-9TddzuZBn2xxhghGGTHLZ4EeNBGTLs3xVzh266NiSJvtUsCsZQ5yVV6H5NhnhyAkKK8uUiZOUUlUAk3HdV+4xg== + version "4.24.1" + resolved "https://registry.yarnpkg.com/openai/-/openai-4.24.1.tgz#3759001eca835228289fcf18c1bd8d35dae538ba" + integrity sha512-ezm/O3eiZMnyBqirUnWm9N6INJU1WhNtz+nK/Zj/2oyKvRz9pgpViDxa5wYOtyGYXPn1sIKBV0I/S4BDhtydqw== dependencies: "@types/node" "^18.11.18" "@types/node-fetch" "^2.6.4"