fix: better extraction for youtube thumbnails. #2204

Author: MohamedBassem

apps/workers/package.json

@@ -41,6 +41,7 @@
     "metascraper-title": "^5.49.5",
     "metascraper-twitter": "^5.45.6",
     "metascraper-url": "^5.49.5",
+    "metascraper-youtube": "^5.49.7",
     "node-cron": "^3.0.3",
     "node-fetch": "^3.3.2",
     "pdf2json": "^3.1.5",

apps/workers/workers/crawlerWorker.ts

@@ -26,6 +26,7 @@ import metascraperPublisher from "metascraper-publisher";
 import metascraperTitle from "metascraper-title";
 import metascraperTwitter from "metascraper-twitter";
 import metascraperUrl from "metascraper-url";
+import metascraperYoutube from "metascraper-youtube";
 import { crawlerStatusCodeCounter, workerStatsCounter } from "metrics";
 import {
   fetchWithProxy,
@@ -125,6 +126,18 @@ const metascraperParser = metascraper([
     datePublished: true,
   }),
   metascraperAmazon(),
+  metascraperYoutube({
+    gotOpts: {
+      agent: {
+        http: serverConfig.proxy.httpProxy
+          ? new HttpProxyAgent(getRandomProxy(serverConfig.proxy.httpProxy))
+          : undefined,
+        https: serverConfig.proxy.httpsProxy
+          ? new HttpsProxyAgent(getRandomProxy(serverConfig.proxy.httpsProxy))
+          : undefined,
+      },
+    },
+  }),
   metascraperReddit(),
   metascraperAuthor(),
   metascraperPublisher(),