Skip to content

Commit e5ac058

Browse files
🎥 feat: YouTube Tool (danny-avila#5582)
* adding youtube tool * refactor: use short `url` param instead of `videoUrl` * refactor: move API key retrieval to a separate credentials module * refactor: remove unnecessary `isEdited` message property * refactor: remove unnecessary `isEdited` message property pt. 2 * refactor: YouTube Tool with new `tool()` generator, handle tools already created by new `tool` generator * fix: only reset request data for multi-convo messages * refactor: enhance YouTube tool by adding transcript parsing and returning structured JSON responses * refactor: update transcript parsing to handle raw response and clean up text output * feat: support toolkits and refactor YouTube tool as a toolkit for better LLM usage * refactor: remove unused OpenAPI specs and streamline tools transformation in loadAsyncEndpoints * refactor: implement manifestToolMap for better tool management and streamline authentication handling * feat: support toolkits for assistants * refactor: rename loadedTools to toolDefinitions for clarity in PluginController and assistant controllers * feat: complete support of toolkits for assistants --------- Co-authored-by: Danilo Pejakovic <[email protected]>
1 parent a50123a commit e5ac058

File tree

29 files changed

+456
-102
lines changed

29 files changed

+456
-102
lines changed

‎.env.example

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,10 @@ AZURE_AI_SEARCH_SEARCH_OPTION_SELECT=
255255
GOOGLE_SEARCH_API_KEY=
256256
GOOGLE_CSE_ID=
257257

258+
# YOUTUBE
259+
#-----------------
260+
YOUTUBE_API_KEY=
261+
258262
# SerpAPI
259263
#-----------------
260264
SERPAPI_API_KEY=

‎api/app/clients/PluginsClient.js

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -280,7 +280,6 @@ class PluginsClient extends OpenAIClient {
280280
logger.debug('[PluginsClient] sendMessage', { userMessageText: message, opts });
281281
const {
282282
user,
283-
isEdited,
284283
conversationId,
285284
responseMessageId,
286285
saveOptions,
@@ -359,7 +358,6 @@ class PluginsClient extends OpenAIClient {
359358
conversationId,
360359
parentMessageId: userMessage.messageId,
361360
isCreatedByUser: false,
362-
isEdited,
363361
model: this.modelOptions.model,
364362
sender: this.sender,
365363
promptTokens,

‎api/app/clients/prompts/formatMessages.spec.js

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@ describe('formatMessage', () => {
6060
error: false,
6161
finish_reason: null,
6262
isCreatedByUser: true,
63-
isEdited: false,
6463
model: null,
6564
parentMessageId: Constants.NO_PARENT,
6665
sender: 'User',

‎api/app/clients/tools/index.js

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,40 @@ const availableTools = require('./manifest.json');
22

33
// Structured Tools
44
const DALLE3 = require('./structured/DALLE3');
5+
const OpenWeather = require('./structured/OpenWeather');
6+
const createYouTubeTools = require('./structured/YouTube');
57
const StructuredWolfram = require('./structured/Wolfram');
68
const StructuredACS = require('./structured/AzureAISearch');
79
const StructuredSD = require('./structured/StableDiffusion');
810
const GoogleSearchAPI = require('./structured/GoogleSearch');
911
const TraversaalSearch = require('./structured/TraversaalSearch');
1012
const TavilySearchResults = require('./structured/TavilySearchResults');
11-
const OpenWeather = require('./structured/OpenWeather');
13+
14+
/** @type {Record<string, TPlugin | undefined>} */
15+
const manifestToolMap = {};
16+
17+
/** @type {Array<TPlugin>} */
18+
const toolkits = [];
19+
20+
availableTools.forEach((tool) => {
21+
manifestToolMap[tool.pluginKey] = tool;
22+
if (tool.toolkit === true) {
23+
toolkits.push(tool);
24+
}
25+
});
1226

1327
module.exports = {
28+
toolkits,
1429
availableTools,
30+
manifestToolMap,
1531
// Structured Tools
1632
DALLE3,
33+
OpenWeather,
1734
StructuredSD,
1835
StructuredACS,
1936
GoogleSearchAPI,
2037
TraversaalSearch,
2138
StructuredWolfram,
39+
createYouTubeTools,
2240
TavilySearchResults,
23-
OpenWeather,
2441
};

‎api/app/clients/tools/manifest.json

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,20 @@
3030
}
3131
]
3232
},
33+
{
34+
"name": "YouTube",
35+
"pluginKey": "youtube",
36+
"toolkit": true,
37+
"description": "Get YouTube video information, retrieve comments, analyze transcripts and search for videos.",
38+
"icon": "https://www.youtube.com/s/desktop/7449ebf7/img/favicon_144x144.png",
39+
"authConfig": [
40+
{
41+
"authField": "YOUTUBE_API_KEY",
42+
"label": "YouTube API Key",
43+
"description": "Your YouTube Data API v3 key."
44+
}
45+
]
46+
},
3347
{
3448
"name": "Wolfram",
3549
"pluginKey": "wolfram",

‎api/app/clients/tools/structured/TavilySearch.js

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,13 @@
11
const { z } = require('zod');
22
const { tool } = require('@langchain/core/tools');
3-
const { getEnvironmentVariable } = require('@langchain/core/utils/env');
3+
const { getApiKey } = require('./credentials');
44

55
function createTavilySearchTool(fields = {}) {
66
const envVar = 'TAVILY_API_KEY';
77
const override = fields.override ?? false;
88
const apiKey = fields.apiKey ?? getApiKey(envVar, override);
99
const kwargs = fields?.kwargs ?? {};
1010

11-
function getApiKey(envVar, override) {
12-
const key = getEnvironmentVariable(envVar);
13-
if (!key && !override) {
14-
throw new Error(`Missing ${envVar} environment variable.`);
15-
}
16-
return key;
17-
}
18-
1911
return tool(
2012
async (input) => {
2113
const { query, ...rest } = input;
Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
const { z } = require('zod');
2+
const { tool } = require('@langchain/core/tools');
3+
const { youtube } = require('@googleapis/youtube');
4+
const { YoutubeTranscript } = require('youtube-transcript');
5+
const { getApiKey } = require('./credentials');
6+
const { logger } = require('~/config');
7+
8+
function extractVideoId(url) {
9+
const rawIdRegex = /^[a-zA-Z0-9_-]{11}$/;
10+
if (rawIdRegex.test(url)) {
11+
return url;
12+
}
13+
14+
const regex = new RegExp(
15+
'(?:youtu\\.be/|youtube(?:\\.com)?/(?:' +
16+
'(?:watch\\?v=)|(?:embed/)|(?:shorts/)|(?:live/)|(?:v/)|(?:/))?)' +
17+
'([a-zA-Z0-9_-]{11})(?:\\S+)?$',
18+
);
19+
const match = url.match(regex);
20+
return match ? match[1] : null;
21+
}
22+
23+
function parseTranscript(transcriptResponse) {
24+
if (!Array.isArray(transcriptResponse)) {
25+
return '';
26+
}
27+
28+
return transcriptResponse
29+
.map((entry) => entry.text.trim())
30+
.filter((text) => text)
31+
.join(' ')
32+
.replaceAll('&amp;#39;', '\'');
33+
}
34+
35+
function createYouTubeTools(fields = {}) {
36+
const envVar = 'YOUTUBE_API_KEY';
37+
const override = fields.override ?? false;
38+
const apiKey = fields.apiKey ?? fields[envVar] ?? getApiKey(envVar, override);
39+
40+
const youtubeClient = youtube({
41+
version: 'v3',
42+
auth: apiKey,
43+
});
44+
45+
const searchTool = tool(
46+
async ({ query, maxResults = 5 }) => {
47+
const response = await youtubeClient.search.list({
48+
part: 'snippet',
49+
q: query,
50+
type: 'video',
51+
maxResults: maxResults || 5,
52+
});
53+
const result = response.data.items.map((item) => ({
54+
title: item.snippet.title,
55+
description: item.snippet.description,
56+
url: `https://www.youtube.com/watch?v=${item.id.videoId}`,
57+
}));
58+
return JSON.stringify(result, null, 2);
59+
},
60+
{
61+
name: 'youtube_search',
62+
description: `Search for YouTube videos by keyword or phrase.
63+
- Required: query (search terms to find videos)
64+
- Optional: maxResults (number of videos to return, 1-50, default: 5)
65+
- Returns: List of videos with titles, descriptions, and URLs
66+
- Use for: Finding specific videos, exploring content, research
67+
Example: query="cooking pasta tutorials" maxResults=3`,
68+
schema: z.object({
69+
query: z.string().describe('Search query terms'),
70+
maxResults: z.number().int().min(1).max(50).optional().describe('Number of results (1-50)'),
71+
}),
72+
},
73+
);
74+
75+
const infoTool = tool(
76+
async ({ url }) => {
77+
const videoId = extractVideoId(url);
78+
if (!videoId) {
79+
throw new Error('Invalid YouTube URL or video ID');
80+
}
81+
82+
const response = await youtubeClient.videos.list({
83+
part: 'snippet,statistics',
84+
id: videoId,
85+
});
86+
87+
if (!response.data.items?.length) {
88+
throw new Error('Video not found');
89+
}
90+
const video = response.data.items[0];
91+
92+
const result = {
93+
title: video.snippet.title,
94+
description: video.snippet.description,
95+
views: video.statistics.viewCount,
96+
likes: video.statistics.likeCount,
97+
comments: video.statistics.commentCount,
98+
};
99+
return JSON.stringify(result, null, 2);
100+
},
101+
{
102+
name: 'youtube_info',
103+
description: `Get detailed metadata and statistics for a specific YouTube video.
104+
- Required: url (full YouTube URL or video ID)
105+
- Returns: Video title, description, view count, like count, comment count
106+
- Use for: Getting video metrics and basic metadata
107+
- DO NOT USE FOR VIDEO SUMMARIES, USE TRANSCRIPTS FOR COMPREHENSIVE ANALYSIS
108+
- Accepts both full URLs and video IDs
109+
Example: url="https://youtube.com/watch?v=abc123" or url="abc123"`,
110+
schema: z.object({
111+
url: z.string().describe('YouTube video URL or ID'),
112+
}),
113+
},
114+
);
115+
116+
const commentsTool = tool(
117+
async ({ url, maxResults = 10 }) => {
118+
const videoId = extractVideoId(url);
119+
if (!videoId) {
120+
throw new Error('Invalid YouTube URL or video ID');
121+
}
122+
123+
const response = await youtubeClient.commentThreads.list({
124+
part: 'snippet',
125+
videoId,
126+
maxResults: maxResults || 10,
127+
});
128+
129+
const result = response.data.items.map((item) => ({
130+
author: item.snippet.topLevelComment.snippet.authorDisplayName,
131+
text: item.snippet.topLevelComment.snippet.textDisplay,
132+
likes: item.snippet.topLevelComment.snippet.likeCount,
133+
}));
134+
return JSON.stringify(result, null, 2);
135+
},
136+
{
137+
name: 'youtube_comments',
138+
description: `Retrieve top-level comments from a YouTube video.
139+
- Required: url (full YouTube URL or video ID)
140+
- Optional: maxResults (number of comments, 1-50, default: 10)
141+
- Returns: Comment text, author names, like counts
142+
- Use for: Sentiment analysis, audience feedback, engagement review
143+
Example: url="abc123" maxResults=20`,
144+
schema: z.object({
145+
url: z.string().describe('YouTube video URL or ID'),
146+
maxResults: z
147+
.number()
148+
.int()
149+
.min(1)
150+
.max(50)
151+
.optional()
152+
.describe('Number of comments to retrieve'),
153+
}),
154+
},
155+
);
156+
157+
const transcriptTool = tool(
158+
async ({ url }) => {
159+
const videoId = extractVideoId(url);
160+
if (!videoId) {
161+
throw new Error('Invalid YouTube URL or video ID');
162+
}
163+
164+
try {
165+
try {
166+
const transcript = await YoutubeTranscript.fetchTranscript(videoId, { lang: 'en' });
167+
return parseTranscript(transcript);
168+
} catch (e) {
169+
logger.error(e);
170+
}
171+
172+
try {
173+
const transcript = await YoutubeTranscript.fetchTranscript(videoId, { lang: 'de' });
174+
return parseTranscript(transcript);
175+
} catch (e) {
176+
logger.error(e);
177+
}
178+
179+
const transcript = await YoutubeTranscript.fetchTranscript(videoId);
180+
return parseTranscript(transcript);
181+
} catch (error) {
182+
throw new Error(`Failed to fetch transcript: ${error.message}`);
183+
}
184+
},
185+
{
186+
name: 'youtube_transcript',
187+
description: `Fetch and parse the transcript/captions of a YouTube video.
188+
- Required: url (full YouTube URL or video ID)
189+
- Returns: Full video transcript as plain text
190+
- Use for: Content analysis, summarization, translation reference
191+
- This is the "Go-to" tool for analyzing actual video content
192+
- Attempts to fetch English first, then German, then any available language
193+
Example: url="https://youtube.com/watch?v=abc123"`,
194+
schema: z.object({
195+
url: z.string().describe('YouTube video URL or ID'),
196+
}),
197+
},
198+
);
199+
200+
return [searchTool, infoTool, commentsTool, transcriptTool];
201+
}
202+
203+
module.exports = createYouTubeTools;
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
const { getEnvironmentVariable } = require('@langchain/core/utils/env');
2+
3+
function getApiKey(envVar, override) {
4+
const key = getEnvironmentVariable(envVar);
5+
if (!key && !override) {
6+
throw new Error(`Missing ${envVar} environment variable.`);
7+
}
8+
return key;
9+
}
10+
11+
module.exports = {
12+
getApiKey,
13+
};

0 commit comments

Comments
 (0)