add video conclusion plugin

app/api/langchain-tools/bilibili_vid_conclusion.ts

@@ -0,0 +1,138 @@
+import { Tool } from "@langchain/core/tools";
+import { getRandomUserAgent } from "./ua_tools";
+import { encWbi, getWbiKeys } from "./bili_wbi_tools";
+
+export interface Headers {
+  [key: string]: string;
+}
+
+export interface RequestTool {
+  headers: Headers;
+  maxOutputLength?: number;
+  timeout: number;
+}
+
+export class BilibiliVideoConclusionTool extends Tool implements RequestTool {
+  name = "bilibili_video_conclusion";
+
+  maxOutputLength = Infinity;
+
+  timeout = 90000;
+
+  constructor(
+    public headers: Headers = {},
+    { maxOutputLength }: { maxOutputLength?: number } = {},
+    { timeout }: { timeout?: number } = {},
+  ) {
+    super(...arguments);
+
+    this.maxOutputLength = maxOutputLength ?? this.maxOutputLength;
+    this.timeout = timeout ?? this.timeout;
+  }
+
+  /** @ignore */
+  async _call(query: string) {
+    try {
+      var [videoAid, pid] = query.split(",");
+      // check if arguments are valid
+      // is videoAid a string of numbers?
+      if (!(/^\d+$/.test(videoAid) || /^av\d+$/.test(videoAid))) {
+        throw new Error(
+          "Invalid videoAid: It should be a string of numbers. If a BVid or a short link is given, please convert it to Aid using av{Aid} format using BiliVideoInfo tool.",
+        );
+      }
+      if (!/^\d+$/.test(pid)) {
+        throw new Error(
+          "Invalid pid: it should be a number representing the page number of the video, starting from 1.",
+        );
+      }
+
+      if (videoAid.startsWith("av")) videoAid = videoAid.slice(2);
+
+      let result = await this.fetchVideoInfo(+videoAid, +pid);
+      // console.log(result)
+      return result;
+    } catch (error) {
+      console.error(error);
+      return (error as Error).toString();
+    }
+  }
+
+  async fetchVideoInfo(aid: number, pid: number): Promise<string> {
+    const headers = new Headers();
+    headers.append("User-Agent", getRandomUserAgent());
+    headers.append("Referer", "https://www.bilibili.com/video/av" + aid);
+    headers.append("Origin", "https://www.bilibili.com");
+    headers.append("Cookie", process.env.BILIBILI_COOKIES || "");
+
+    const vidinfo_resp = await this.fetchWithTimeout(
+      `https://api.bilibili.com/x/web-interface/view?aid=${aid}`,
+      {
+        headers: headers,
+      },
+    );
+
+    const vidinfo = await vidinfo_resp.json();
+    const mid = vidinfo.data.owner.mid;
+    const cid = vidinfo.data.pages[pid - 1].cid;
+
+    // get https://api.bilibili.com/x/web-interface/view/conclusion/get?aid={aid}&cid={cid}&up_mid={mid}
+    const { img_key, sub_key } = await getWbiKeys();
+    var queryStr = encWbi(
+      {
+        aid: aid,
+        cid: cid,
+        up_mid: mid,
+      },
+      img_key,
+      sub_key,
+    );
+
+    const resp = await this.fetchWithTimeout(
+      `https://api.bilibili.com/x/web-interface/view/conclusion/get?${queryStr}`,
+      {
+        headers: headers,
+      },
+    );
+
+    const data = await resp.json();
+    const model_result = data.data.model_result;
+    switch (model_result.result_type) {
+      case 0:
+        return "BiliAPI returned result: Unable to provide a conclusion for this video, or the conclusion cannot be generated for this video due to the video quality or the video content.";
+      case 1:
+        return model_result.summary;
+      case 2:
+        var resText =
+          model_result.summary + "\n\nOutline (generated by BiliAPI): \n";
+
+        for (var i = 0; i < model_result.outline.length; i++) {
+          let ol = model_result.outline[i];
+          resText += `## [position: ${ol.timestamp}s] ${ol.title}\n`;
+          ol.part_outline.forEach((olpart: any) => {
+            resText += `- [position: ${olpart.timestamp}s] ${olpart.content}`;
+          });
+        }
+        return resText;
+      default:
+        return "BiliAPI returned unknown result type. Please report this issue to the developer.";
+    }
+  }
+
+  async fetchWithTimeout(
+    resource: RequestInfo | URL,
+    options = {},
+    timeout: number = 30000,
+  ) {
+    const controller = new AbortController();
+    const id = setTimeout(() => controller.abort(), timeout);
+    const response = await fetch(resource, {
+      ...options,
+      signal: controller.signal,
+    });
+    clearTimeout(id);
+    return response;
+  }
+
+  description = `A tool to fetch the conclusion of a Bilibili video. It uses the BiliAPI to fetch the conclusion of the video. Input string should be in the format of "videoAid,pid" where videoAid is the Aid of the video and pid is the page number of the video, starting from 1.`;
+}

app/api/langchain-tools/nodejs_tools.ts

@@ -11,6 +11,7 @@ import { BilibiliVideoInfoTool } from "./bilibili_vid_info";
 import { BilibiliVideoSearchTool } from "./bilibili_vid_search";
 import { BilibiliMusicRecognitionTool } from "./bilibili_music_recognition";
 import { RAGSearch } from "./rag_search";
+import { BilibiliVideoConclusionTool } from "./bilibili_vid_conclusion";
 
 export class NodeJSTool {
   private apiKey: string | undefined;
@@ -57,6 +58,7 @@ export class NodeJSTool {
     const bilibiliVideoInfoTool = new BilibiliVideoInfoTool();
     const bilibiliVideoSearchTool = new BilibiliVideoSearchTool();
     const bilibiliMusicRecognitionTool = new BilibiliMusicRecognitionTool();
+    const bilibiliVideoConclusionTool = new BilibiliVideoConclusionTool();
     let tools = [
       calculatorTool,
       webBrowserTool,
@@ -68,6 +70,7 @@ export class NodeJSTool {
       bilibiliVideoInfoTool,
       bilibiliVideoSearchTool,
       bilibiliMusicRecognitionTool,
+      bilibiliVideoConclusionTool,
     ];
     if (!!process.env.ENABLE_RAG) {
       tools.push(new RAGSearch(this.sessionId, this.model, this.ragEmbeddings));

app/plugins/cn.ts

@@ -62,6 +62,16 @@ export const CN_PLUGINS: BuiltinPlugin[] = [
     enable: true,
     onlyNodeRuntime: true,
   },
+  {
+    name: "Bilibili视频总结",
+    toolName: "bilibili_video_conclusion",
+    lang: "cn",
+    description: "通过Bilibili视频ID进行视频总结。",
+    builtin: true,
+    createdAt: 1712394126000,
+    enable: true,
+    onlyNodeRuntime: true,
+  },
   {
     name: "维基百科",
     toolName: "WikipediaQueryRun",