diff --git a/app/api/langchain-tools/bilibili_music_recognition.ts b/app/api/langchain-tools/bilibili_music_recognition.ts new file mode 100644 index 000000000..cc03047da --- /dev/null +++ b/app/api/langchain-tools/bilibili_music_recognition.ts @@ -0,0 +1,97 @@ +import { Tool } from "@langchain/core/tools"; +import { getRandomUserAgent } from "./ua_tools"; +import { encWbi, getWbiKeys } from "./bili_wbi_tools"; + +export interface Headers { + [key: string]: string; +} + +export interface RequestTool { + headers: Headers; + maxOutputLength?: number; + timeout: number; +} + +export class BilibiliMusicRecognitionTool extends Tool implements RequestTool { + name = "bilibili_music_recognition"; + + maxOutputLength = Infinity; + + timeout = 300000; + + constructor( + public headers: Headers = {}, + { maxOutputLength }: { maxOutputLength?: number } = {}, + { timeout }: { timeout?: number } = {}, + ) { + super(...arguments); + + this.maxOutputLength = maxOutputLength ?? this.maxOutputLength; + this.timeout = timeout ?? this.timeout; + } + + /** @ignore */ + async _call(query: string) { + try { + // let result = await this.doAcrcloudRecognitionUsingMetaprocAPI(searchQuery); + // parse query + const [videoAid, pid, targetSec] = query.split(","); + const result = await this.doAcrcloudRecognitionUsingMetaprocAPI( + videoAid, + parseInt(pid), + parseInt(targetSec), + ); + // console.log(result) + return JSON.stringify(result); + } catch (error) { + console.error(error); + return (error as Error).toString(); + } + } + + async doAcrcloudRecognitionUsingMetaprocAPI( + videoAid: string, + pid: number, + targetSec: number, + ) { + // get http://10.0.1.3:32345/api/recog_music_in_bili_video?video_aid=170001&pid=1&target_sec=14 + + const url = `http://10.0.1.3:32345/api/recog_music_in_bili_video?video_aid=${videoAid}&pid=${pid}&target_sec=${targetSec}`; + + const headers = { + "User-Agent": getRandomUserAgent(), + }; + + const response = await this.fetchWithTimeout( + url, + { headers }, + this.timeout, + ); + + if (!response.ok) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + var data = await response.json(); + + return data; + } + + async fetchWithTimeout( + resource: RequestInfo | URL, + options = {}, + timeout: number = 30000, + ) { + const controller = new AbortController(); + const id = setTimeout(() => controller.abort(), timeout); + const response = await fetch(resource, { + ...options, + signal: controller.signal, + }); + clearTimeout(id); + return response; + } + + description = `A tool that recognizes music in Bilibili videos using ACRCloud API. Input string is in this format: video_aid,pid,target_sec. video_aid is extracted from av{video_aid}, e.g. av170001 means video_aid=170001. pid is the page ID of the video(note: pid starts from 1, not 0). and target_sec is the time in seconds where the recognition is expected to start from. +Trick: If you only have a BVID of the video, you can convert it to video_aid by triggering the video info API and extract the video_aid from the response, before using this tool.`; +} diff --git a/app/api/langchain-tools/nodejs_tools.ts b/app/api/langchain-tools/nodejs_tools.ts index 996af7952..6b8d9c084 100644 --- a/app/api/langchain-tools/nodejs_tools.ts +++ b/app/api/langchain-tools/nodejs_tools.ts @@ -9,6 +9,7 @@ import { WebBrowser } from "langchain/tools/webbrowser"; import { WolframAlphaTool } from "@/app/api/langchain-tools/wolframalpha"; import { BilibiliVideoInfoTool } from "./bilibili_vid_info"; import { BilibiliVideoSearchTool } from "./bilibili_vid_search"; +import { BilibiliMusicRecognitionTool } from "./bilibili_music_recognition"; export class NodeJSTool { private apiKey: string | undefined; @@ -52,6 +53,7 @@ export class NodeJSTool { const pdfBrowserTool = new PDFBrowser(this.model, this.embeddings); const bilibiliVideoInfoTool = new BilibiliVideoInfoTool(); const bilibiliVideoSearchTool = new BilibiliVideoSearchTool(); + const bilibiliMusicRecognitionTool = new BilibiliMusicRecognitionTool(); let tools = [ calculatorTool, webBrowserTool, @@ -62,6 +64,7 @@ export class NodeJSTool { pdfBrowserTool, bilibiliVideoInfoTool, bilibiliVideoSearchTool, + bilibiliMusicRecognitionTool, ]; return tools; } diff --git a/app/plugins/cn.ts b/app/plugins/cn.ts index a82fa6794..8595d2395 100644 --- a/app/plugins/cn.ts +++ b/app/plugins/cn.ts @@ -52,6 +52,16 @@ export const CN_PLUGINS: BuiltinPlugin[] = [ enable: true, onlyNodeRuntime: true, }, + { + name: "Bilibili听歌识曲", + toolName: "bilibili_music_recognition", + lang: "cn", + description: "通过Bilibili视频ID进行听歌识曲识别。", + builtin: true, + createdAt: 1712394126000, + enable: true, + onlyNodeRuntime: true, + }, { name: "维基百科", toolName: "WikipediaQueryRun",