add plugin: bili music recognition

app/api/langchain-tools/bilibili_music_recognition.ts

@@ -0,0 +1,97 @@
+import { Tool } from "@langchain/core/tools";
+import { getRandomUserAgent } from "./ua_tools";
+import { encWbi, getWbiKeys } from "./bili_wbi_tools";
+
+export interface Headers {
+  [key: string]: string;
+}
+
+export interface RequestTool {
+  headers: Headers;
+  maxOutputLength?: number;
+  timeout: number;
+}
+
+export class BilibiliMusicRecognitionTool extends Tool implements RequestTool {
+  name = "bilibili_music_recognition";
+
+  maxOutputLength = Infinity;
+
+  timeout = 300000;
+
+  constructor(
+    public headers: Headers = {},
+    { maxOutputLength }: { maxOutputLength?: number } = {},
+    { timeout }: { timeout?: number } = {},
+  ) {
+    super(...arguments);
+
+    this.maxOutputLength = maxOutputLength ?? this.maxOutputLength;
+    this.timeout = timeout ?? this.timeout;
+  }
+
+  /** @ignore */
+  async _call(query: string) {
+    try {
+      // let result = await this.doAcrcloudRecognitionUsingMetaprocAPI(searchQuery);
+      // parse query
+      const [videoAid, pid, targetSec] = query.split(",");
+      const result = await this.doAcrcloudRecognitionUsingMetaprocAPI(
+        videoAid,
+        parseInt(pid),
+        parseInt(targetSec),
+      );
+      // console.log(result)
+      return JSON.stringify(result);
+    } catch (error) {
+      console.error(error);
+      return (error as Error).toString();
+    }
+  }
+
+  async doAcrcloudRecognitionUsingMetaprocAPI(
+    videoAid: string,
+    pid: number,
+    targetSec: number,
+  ) {
+    // get http://10.0.1.3:32345/api/recog_music_in_bili_video?video_aid=170001&pid=1&target_sec=14
+
+    const url = `http://10.0.1.3:32345/api/recog_music_in_bili_video?video_aid=${videoAid}&pid=${pid}&target_sec=${targetSec}`;
+
+    const headers = {
+      "User-Agent": getRandomUserAgent(),
+    };
+
+    const response = await this.fetchWithTimeout(
+      url,
+      { headers },
+      this.timeout,
+    );
+
+    if (!response.ok) {
+      throw new Error(`HTTP error! status: ${response.status}`);
+    }
+
+    var data = await response.json();
+
+    return data;
+  }
+
+  async fetchWithTimeout(
+    resource: RequestInfo | URL,
+    options = {},
+    timeout: number = 30000,
+  ) {
+    const controller = new AbortController();
+    const id = setTimeout(() => controller.abort(), timeout);
+    const response = await fetch(resource, {
+      ...options,
+      signal: controller.signal,
+    });
+    clearTimeout(id);
+    return response;
+  }
+
+  description = `A tool that recognizes music in Bilibili videos using ACRCloud API. Input string is in this format: video_aid,pid,target_sec. video_aid is extracted from av{video_aid}, e.g. av170001 means video_aid=170001. pid is the page ID of the video(note: pid starts from 1, not 0). and target_sec is the time in seconds where the recognition is expected to start from.
+Trick: If you only have a BVID of the video, you can convert it to video_aid by triggering the video info API and extract the video_aid from the response, before using this tool.`;
+}

app/api/langchain-tools/nodejs_tools.ts

@@ -9,6 +9,7 @@ import { WebBrowser } from "langchain/tools/webbrowser";
 import { WolframAlphaTool } from "@/app/api/langchain-tools/wolframalpha";
 import { BilibiliVideoInfoTool } from "./bilibili_vid_info";
 import { BilibiliVideoSearchTool } from "./bilibili_vid_search";
+import { BilibiliMusicRecognitionTool } from "./bilibili_music_recognition";
 
 export class NodeJSTool {
   private apiKey: string | undefined;
@@ -52,6 +53,7 @@ export class NodeJSTool {
     const pdfBrowserTool = new PDFBrowser(this.model, this.embeddings);
     const bilibiliVideoInfoTool = new BilibiliVideoInfoTool();
     const bilibiliVideoSearchTool = new BilibiliVideoSearchTool();
+    const bilibiliMusicRecognitionTool = new BilibiliMusicRecognitionTool();
     let tools = [
       calculatorTool,
       webBrowserTool,
@@ -62,6 +64,7 @@ export class NodeJSTool {
       pdfBrowserTool,
       bilibiliVideoInfoTool,
       bilibiliVideoSearchTool,
+      bilibiliMusicRecognitionTool,
     ];
     return tools;
   }

app/plugins/cn.ts

@@ -52,6 +52,16 @@ export const CN_PLUGINS: BuiltinPlugin[] = [
     enable: true,
     onlyNodeRuntime: true,
   },
+  {
+    name: "Bilibili听歌识曲",
+    toolName: "bilibili_music_recognition",
+    lang: "cn",
+    description: "通过Bilibili视频ID进行听歌识曲识别。",
+    builtin: true,
+    createdAt: 1712394126000,
+    enable: true,
+    onlyNodeRuntime: true,
+  },
   {
     name: "维基百科",
     toolName: "WikipediaQueryRun",