From 1e8068436f609995c6910a9a68e1e343b2988387 Mon Sep 17 00:00:00 2001 From: Hk-Gosuto Date: Tue, 12 Sep 2023 13:02:41 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20ddg=E6=8F=92=E4=BB=B6=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=8D=95=E7=8B=AC=E9=85=8D=E7=BD=AEhttp=E4=BB=A3=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- app/api/langchain-tools/duckduckgo_search.ts | 15 ++++++++++ app/api/langchain/tool/agent/route.ts | 5 ++-- package.json | 1 + yarn.lock | 29 +++++++++++++++----- 4 files changed, 41 insertions(+), 9 deletions(-) diff --git a/app/api/langchain-tools/duckduckgo_search.ts b/app/api/langchain-tools/duckduckgo_search.ts index 58437d97e..95e4eaf11 100644 --- a/app/api/langchain-tools/duckduckgo_search.ts +++ b/app/api/langchain-tools/duckduckgo_search.ts @@ -1,6 +1,9 @@ import { decode } from "html-entities"; import { convert as htmlToText } from "html-to-text"; import { Tool } from "langchain/tools"; +import fetch from "node-fetch"; + +const DDG_PROXY_URL = process.env.DDG_PROXY_URL; const SEARCH_REGEX = /DDG\.pageLayout\.load\('d',(\[.+\])\);DDG\.duckbar\.load\('images'/; @@ -323,9 +326,15 @@ async function search( // cdrexp: 'b' }), }; + let proxyAgent; + if (DDG_PROXY_URL) { + const { HttpsProxyAgent } = require("https-proxy-agent"); + proxyAgent = new HttpsProxyAgent(DDG_PROXY_URL); + } const response = await fetch( `https://links.duckduckgo.com/d.js?${queryString(queryObject)}`, + { agent: proxyAgent }, ); const data = await response.text(); @@ -455,8 +464,14 @@ function queryString(query: Record) { async function getVQD(query: string, ia = "web") { try { + let proxyAgent; + if (DDG_PROXY_URL) { + const { HttpsProxyAgent } = require("https-proxy-agent"); + proxyAgent = new HttpsProxyAgent(DDG_PROXY_URL); + } const response = await fetch( `https://duckduckgo.com/?${queryString({ q: query, ia })}`, + { agent: proxyAgent }, ); const data = await response.text(); return VQD_REGEX.exec(data)![1]; diff --git a/app/api/langchain/tool/agent/route.ts b/app/api/langchain/tool/agent/route.ts index 64f4c45de..5e8f9a665 100644 --- a/app/api/langchain/tool/agent/route.ts +++ b/app/api/langchain/tool/agent/route.ts @@ -181,7 +181,8 @@ async function handle(req: NextRequest) { description: bingSearchTool.description, func: async (input: string) => bingSearchTool.call(input), }); - } else if (process.env.SERPAPI_API_KEY) { + } + if (process.env.SERPAPI_API_KEY) { let serpAPITool = new langchainTools["SerpAPI"]( process.env.SERPAPI_API_KEY, ); @@ -290,4 +291,4 @@ async function handle(req: NextRequest) { export const GET = handle; export const POST = handle; -export const runtime = "edge"; +export const runtime = process.env.DDG_PROXY_URL ? "nodejs" : "edge"; diff --git a/package.json b/package.json index 299d12456..60492872e 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "html-entities": "^2.4.0", "html-to-image": "^1.11.11", "html-to-text": "^9.0.5", + "https-proxy-agent": "^7.0.2", "langchain": "^0.0.114", "mermaid": "^10.3.1", "nanoid": "^4.0.2", diff --git a/yarn.lock b/yarn.lock index 04620de4d..0c3d1dcfb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1828,6 +1828,13 @@ acorn@^8.9.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.10.0.tgz#8be5b3907a67221a81ab23c7889c4c5526b62ec5" integrity sha512-F0SAmZ8iUtS//m8DmCTA0jlh6TDKkHQyK6xc6V4KDTyZKA9dnvX9/3sRTVQrWm79glUAZbnmmNcdYwUIHWVybw== +agent-base@^7.0.2: + version "7.1.0" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-7.1.0.tgz#536802b76bc0b34aa50195eb2442276d613e3434" + integrity sha512-o/zjMZRhJxny7OyEF+Op8X+efiELC7k7yOjMzgfzVqOzXqkBkWI79YoTdOtsuWd5BWhAGAuOY/Xa6xpiaWXiNg== + dependencies: + debug "^4.3.4" + agentkeepalive@^4.2.1: version "4.5.0" resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-4.5.0.tgz#2673ad1389b3c418c5a20c5d7364f93ca04be923" @@ -2770,6 +2777,13 @@ dayjs@^1.11.7: resolved "https://registry.npmmirror.com/dayjs/-/dayjs-1.11.7.tgz#4b296922642f70999544d1144a2c25730fce63e2" integrity sha512-+Yw9U6YO5TQohxLcIkrXBeY73WP3ejHWVvx8XCk3gxvQDCTEmS48ZrSZCKciI7Bhl/uCMyxYtE9UqRILmFphkQ== +debug@4, debug@^4.0.0, debug@^4.1.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.4: + version "4.3.4" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" + integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== + dependencies: + ms "2.1.2" + debug@^3.2.6, debug@^3.2.7: version "3.2.7" resolved "https://registry.yarnpkg.com/debug/-/debug-3.2.7.tgz#72580b7e9145fb39b6676f9c5e5fb100b934179a" @@ -2777,13 +2791,6 @@ debug@^3.2.6, debug@^3.2.7: dependencies: ms "^2.1.1" -debug@^4.0.0, debug@^4.1.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.4: - version "4.3.4" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" - integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== - dependencies: - ms "2.1.2" - decamelize@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" @@ -3914,6 +3921,14 @@ htmlparser2@^8.0.1, htmlparser2@^8.0.2: domutils "^3.0.1" entities "^4.4.0" +https-proxy-agent@^7.0.2: + version "7.0.2" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz#e2645b846b90e96c6e6f347fb5b2e41f1590b09b" + integrity sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA== + dependencies: + agent-base "^7.0.2" + debug "4" + human-signals@^4.3.0: version "4.3.1" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-4.3.1.tgz#ab7f811e851fca97ffbd2c1fe9a958964de321b2"