From 1d5e8a9cbf10bb6a3530daa6c30f470e30a42423 Mon Sep 17 00:00:00 2001 From: Hk-Gosuto Date: Sat, 29 Jul 2023 19:02:57 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81duckduckgo=E6=90=9C?= =?UTF-8?q?=E7=B4=A2=E6=8F=92=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/langchain-tools/duckduckgo.ts | 32 +++++++++ app/api/langchain/tool/agent/route.ts | 25 ++++--- app/api/tools/ddg/route.ts | 50 -------------- package.json | 2 + yarn.lock | 93 ++++++++++++++++++++++++++- 5 files changed, 141 insertions(+), 61 deletions(-) create mode 100644 app/api/langchain-tools/duckduckgo.ts delete mode 100644 app/api/tools/ddg/route.ts diff --git a/app/api/langchain-tools/duckduckgo.ts b/app/api/langchain-tools/duckduckgo.ts new file mode 100644 index 000000000..4febe5d29 --- /dev/null +++ b/app/api/langchain-tools/duckduckgo.ts @@ -0,0 +1,32 @@ +import { SafeSearchType, search } from "duck-duck-scrape"; +import { convert as htmlToText } from "html-to-text"; +import { Tool } from "langchain/tools"; + +export class DuckDuckGo extends Tool { + name = "duckduckgo_search"; + maxResults = 4; + + /** @ignore */ + async _call(input: string) { + const searchResults = await search(input, { + safeSearch: SafeSearchType.OFF, + }); + + if (searchResults.noResults) { + return "No good search result found"; + } + + const results = searchResults.results + .slice(0, this.maxResults) + .map( + ({ title, description, url }) => + `title:${title}\ncontent:${htmlToText(description)}\nurl:${url}`, + ) + .join("\n\n"); + + return results; + } + + description = + "a search engine. useful for when you need to answer questions about current events. input should be a search query."; +} diff --git a/app/api/langchain/tool/agent/route.ts b/app/api/langchain/tool/agent/route.ts index 5465bfbc4..0cf5a16df 100644 --- a/app/api/langchain/tool/agent/route.ts +++ b/app/api/langchain/tool/agent/route.ts @@ -9,12 +9,14 @@ import { DynamicTool, RequestsGetTool, RequestsPostTool, + Tool, } from "langchain/tools"; -import { SerpAPI } from "langchain/tools"; -import { Calculator } from "langchain/tools/calculator"; import { AIMessage, HumanMessage, SystemMessage } from "langchain/schema"; import { BufferMemory, ChatMessageHistory } from "langchain/memory"; import { initializeAgentExecutorWithOptions } from "langchain/agents"; +import { SerpAPI } from "langchain/tools"; +import { Calculator } from "langchain/tools/calculator"; +import { DuckDuckGo } from "@/app/api/tools/duckduckgo"; const serverConfig = getServerSideConfig(); @@ -120,16 +122,21 @@ async function handle(req: NextRequest) { }, }); + let searchTool: Tool = new DuckDuckGo(); + if (process.env.SERPAPI_API_KEY) { + let serpAPITool = new SerpAPI(process.env.SERPAPI_API_KEY); + searchTool = new DynamicTool({ + name: "google_search", + description: serpAPITool.description, + func: async (input: string) => serpAPITool.call(input), + }); + } + const tools = [ + searchTool, new RequestsGetTool(), new RequestsPostTool(), - new SerpAPI(process.env.SERPAPI_API_KEY), new Calculator(), - // new DynamicTool({ - // name: ddg.name, - // description: ddg.description, - // func: async (input: string) => ddg.call(input), - // }), ]; const pastMessages = new Array(); @@ -192,4 +199,4 @@ async function handle(req: NextRequest) { export const GET = handle; export const POST = handle; -export const runtime = "edge"; +export const runtime = "nodejs"; diff --git a/app/api/tools/ddg/route.ts b/app/api/tools/ddg/route.ts deleted file mode 100644 index 19a87061d..000000000 --- a/app/api/tools/ddg/route.ts +++ /dev/null @@ -1,50 +0,0 @@ -import { prettyObject } from "@/app/utils/format"; -import { NextRequest, NextResponse } from "next/server"; -import { auth } from "../../auth"; - -import { search, SafeSearchType } from "duck-duck-scrape"; - -async function handle(req: NextRequest) { - if (req.method === "OPTIONS") { - return NextResponse.json({ body: "OK" }, { status: 200 }); - } - - let query = req.nextUrl.searchParams.get("query") ?? ""; - let maxResults = req.nextUrl.searchParams.get( - "max_results", - ) as unknown as number; - if (!maxResults) maxResults = 3; - console.log("[Tools Route] query ", query); - - const authResult = auth(req); - if (authResult.error) { - return NextResponse.json(authResult, { - status: 401, - }); - } - - try { - const searchResults = await search(query, { - safeSearch: SafeSearchType.OFF, - }); - const result = searchResults.results - .slice(0, maxResults) - .map(({ title, description, url }) => ({ - title, - content: description, - url, - })); - const res = new NextResponse(JSON.stringify(result)); - res.headers.set("Content-Type", "application/json"); - res.headers.set("Cache-Control", "no-cache"); - return res; - } catch (e) { - console.error("[Tools] ", e); - return NextResponse.json(prettyObject(e)); - } -} - -export const GET = handle; -export const POST = handle; - -export const runtime = "nodejs"; diff --git a/package.json b/package.json index c1900165e..b30410cea 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "emoji-picker-react": "^4.4.7", "fuse.js": "^6.6.2", "html-to-image": "^1.11.11", + "html-to-text": "^9.0.5", "langchain": "^0.0.114", "mermaid": "^10.2.3", "nanoid": "^4.0.2", @@ -45,6 +46,7 @@ }, "devDependencies": { "@tauri-apps/cli": "^1.4.0", + "@types/html-to-text": "^9.0.1", "@types/node": "^20.3.3", "@types/react": "^18.2.14", "@types/react-dom": "^18.2.7", diff --git a/yarn.lock b/yarn.lock index 723cd579e..5a3dbdef1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1245,6 +1245,14 @@ resolved "https://registry.yarnpkg.com/@rushstack/eslint-patch/-/eslint-patch-1.2.0.tgz#8be36a1f66f3265389e90b5f9c9962146758f728" integrity sha512-sXo/qW2/pAcmT43VoRKOJbDOfV3cYpq3szSVfIThQXNt+E4DfKj361vaAt3c88U5tPUxzEswam7GW48PJqtKAg== +"@selderee/plugin-htmlparser2@^0.11.0": + version "0.11.0" + resolved "https://registry.yarnpkg.com/@selderee/plugin-htmlparser2/-/plugin-htmlparser2-0.11.0.tgz#d5b5e29a7ba6d3958a1972c7be16f4b2c188c517" + integrity sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ== + dependencies: + domhandler "^5.0.3" + selderee "^0.11.0" + "@svgr/babel-plugin-add-jsx-attribute@^6.5.1": version "6.5.1" resolved "https://registry.yarnpkg.com/@svgr/babel-plugin-add-jsx-attribute/-/babel-plugin-add-jsx-attribute-6.5.1.tgz#74a5d648bd0347bda99d82409d87b8ca80b9a1ba" @@ -1472,6 +1480,11 @@ "@types/react" "*" hoist-non-react-statics "^3.3.0" +"@types/html-to-text@^9.0.1": + version "9.0.1" + resolved "https://registry.yarnpkg.com/@types/html-to-text/-/html-to-text-9.0.1.tgz#d4f8b1844464df3a13ca14134f5970c847de6751" + integrity sha512-sHu702QGb0SP2F0Zt+CxdCmGZIZ0gEaaCjqOh/V4iba1wTxPVntEPOM/vHm5bel08TILhB3+OxUTkDJWnr/zHQ== + "@types/json-schema@*", "@types/json-schema@^7.0.8": version "7.0.12" resolved "https://registry.npmmirror.com/@types/json-schema/-/json-schema-7.0.12.tgz#d70faba7039d5fca54c83c7dbab41051d2b6f6cb" @@ -2718,7 +2731,7 @@ deep-is@^0.1.3: resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== -deepmerge@^4.2.2: +deepmerge@^4.2.2, deepmerge@^4.3.1: version "4.3.1" resolved "https://registry.yarnpkg.com/deepmerge/-/deepmerge-4.3.1.tgz#44b5f2147cd3b00d4b56137685966f26fd25dd4a" integrity sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A== @@ -2801,7 +2814,16 @@ dom-serializer@^1.0.1: domhandler "^4.2.0" entities "^2.0.0" -domelementtype@^2.0.1, domelementtype@^2.2.0: +dom-serializer@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/dom-serializer/-/dom-serializer-2.0.0.tgz#e41b802e1eedf9f6cae183ce5e622d789d7d8e53" + integrity sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.2" + entities "^4.2.0" + +domelementtype@^2.0.1, domelementtype@^2.2.0, domelementtype@^2.3.0: version "2.3.0" resolved "https://registry.yarnpkg.com/domelementtype/-/domelementtype-2.3.0.tgz#5c45e8e869952626331d7aab326d01daf65d589d" integrity sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw== @@ -2813,6 +2835,13 @@ domhandler@^4.2.0, domhandler@^4.3.1: dependencies: domelementtype "^2.2.0" +domhandler@^5.0.2, domhandler@^5.0.3: + version "5.0.3" + resolved "https://registry.yarnpkg.com/domhandler/-/domhandler-5.0.3.tgz#cc385f7f751f1d1fc650c21374804254538c7d31" + integrity sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w== + dependencies: + domelementtype "^2.3.0" + dompurify@3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/dompurify/-/dompurify-3.0.3.tgz#4b115d15a091ddc96f232bcef668550a2f6f1430" @@ -2827,6 +2856,15 @@ domutils@^2.8.0: domelementtype "^2.2.0" domhandler "^4.2.0" +domutils@^3.0.1: + version "3.1.0" + resolved "https://registry.yarnpkg.com/domutils/-/domutils-3.1.0.tgz#c47f551278d3dc4b0b1ab8cbb42d751a6f0d824e" + integrity sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA== + dependencies: + dom-serializer "^2.0.0" + domelementtype "^2.3.0" + domhandler "^5.0.3" + duck-duck-scrape@^2.2.4: version "2.2.4" resolved "https://registry.yarnpkg.com/duck-duck-scrape/-/duck-duck-scrape-2.2.4.tgz#4281311ddd51997af6dab6a7f0e7b953a34c78c0" @@ -2893,6 +2931,11 @@ entities@^2.0.0: resolved "https://registry.yarnpkg.com/entities/-/entities-2.2.0.tgz#098dc90ebb83d8dffa089d55256b351d34c4da55" integrity sha512-p92if5Nz619I0w+akJrLZH0MX0Pb5DX39XOwQTtXSdQQOaYH03S1uIQp4mhOZtAXrxq4ViO67YTiLBo2638o9A== +entities@^4.2.0: + version "4.5.0" + resolved "https://registry.yarnpkg.com/entities/-/entities-4.5.0.tgz#5d268ea5e7113ec74c4d033b79ea5a35a488fb48" + integrity sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw== + entities@^4.4.0: version "4.4.0" resolved "https://registry.yarnpkg.com/entities/-/entities-4.4.0.tgz#97bdaba170339446495e653cfd2db78962900174" @@ -3762,6 +3805,27 @@ html-to-image@^1.11.11: resolved "https://registry.npmmirror.com/html-to-image/-/html-to-image-1.11.11.tgz#c0f8a34dc9e4b97b93ff7ea286eb8562642ebbea" integrity sha512-9gux8QhvjRO/erSnDPv28noDZcPZmYE7e1vFsBLKLlRlKDSqNJYebj6Qz1TGd5lsRV+X+xYyjCKjuZdABinWjA== +html-to-text@^9.0.5: + version "9.0.5" + resolved "https://registry.yarnpkg.com/html-to-text/-/html-to-text-9.0.5.tgz#6149a0f618ae7a0db8085dca9bbf96d32bb8368d" + integrity sha512-qY60FjREgVZL03vJU6IfMV4GDjGBIoOyvuFdpBDIX9yTlDw0TjxVBQp+P8NvpdIXNJvfWBTNul7fsAQJq2FNpg== + dependencies: + "@selderee/plugin-htmlparser2" "^0.11.0" + deepmerge "^4.3.1" + dom-serializer "^2.0.0" + htmlparser2 "^8.0.2" + selderee "^0.11.0" + +htmlparser2@^8.0.2: + version "8.0.2" + resolved "https://registry.yarnpkg.com/htmlparser2/-/htmlparser2-8.0.2.tgz#f002151705b383e62433b5cf466f5b716edaec21" + integrity sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA== + dependencies: + domelementtype "^2.3.0" + domhandler "^5.0.3" + domutils "^3.0.1" + entities "^4.4.0" + human-signals@^4.3.0: version "4.3.1" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-4.3.1.tgz#ab7f811e851fca97ffbd2c1fe9a958964de321b2" @@ -4232,6 +4296,11 @@ layout-base@^2.0.0: resolved "https://registry.npmmirror.com/layout-base/-/layout-base-2.0.1.tgz#d0337913586c90f9c2c075292069f5c2da5dd285" integrity sha512-dp3s92+uNI1hWIpPGH3jK2kxE2lMjdXdr+DH8ynZHpd6PUlH6x6cbuXnoMmiNumznqaNO31xu9e79F0uuZ0JFg== +leac@^0.6.0: + version "0.6.0" + resolved "https://registry.yarnpkg.com/leac/-/leac-0.6.0.tgz#dcf136e382e666bd2475f44a1096061b70dc0912" + integrity sha512-y+SqErxb8h7nE/fiEX07jsbuhrpO9lL8eca7/Y1nuWV2moNlXhyd59iDGcRf6moVyDMbmTNzL40SUyrFU/yDpg== + levn@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/levn/-/levn-0.4.1.tgz#ae4562c007473b932a6200d403268dd2fffc6ade" @@ -5259,6 +5328,14 @@ parse5@^7.0.0: dependencies: entities "^4.4.0" +parseley@^0.12.0: + version "0.12.1" + resolved "https://registry.yarnpkg.com/parseley/-/parseley-0.12.1.tgz#4afd561d50215ebe259e3e7a853e62f600683aef" + integrity sha512-e6qHKe3a9HWr0oMRVDTRhKce+bRO8VGQR3NyVwcjwrbhMmFCX9KszEV35+rn4AdilFAq9VPxP/Fe1wC9Qjd2lw== + dependencies: + leac "^0.6.0" + peberminta "^0.9.0" + path-exists@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-4.0.0.tgz#513bdbe2d3b95d7762e8c1137efa195c6c61b5b3" @@ -5289,6 +5366,11 @@ path-type@^4.0.0: resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== +peberminta@^0.9.0: + version "0.9.0" + resolved "https://registry.yarnpkg.com/peberminta/-/peberminta-0.9.0.tgz#8ec9bc0eb84b7d368126e71ce9033501dca2a352" + integrity sha512-XIxfHpEuSJbITd1H3EeQwpcZbTLHc+VVr8ANI9t5sit565tsI4/xK3KWTUFE2e6QiangUkh3B0jihzmGnNrRsQ== + picocolors@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" @@ -5709,6 +5791,13 @@ schema-utils@^3.1.1, schema-utils@^3.2.0: ajv "^6.12.5" ajv-keywords "^3.5.2" +selderee@^0.11.0: + version "0.11.0" + resolved "https://registry.yarnpkg.com/selderee/-/selderee-0.11.0.tgz#6af0c7983e073ad3e35787ffe20cefd9daf0ec8a" + integrity sha512-5TF+l7p4+OsnP8BCCvSyZiSPc4x4//p5uPwK8TCnVPJYRmU2aYKMpOXvw8zM5a5JvuuCGN1jmsMwuU2W02ukfA== + dependencies: + parseley "^0.12.0" + semver@^6.1.1, semver@^6.1.2, semver@^6.3.0: version "6.3.1" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4"