create a GraphQL API to create youtube transcript job on demand

packages/api/package.json

@@ -67,6 +67,7 @@
     "express-http-context2": "^1.0.0",
     "express-prom-bundle": "^7.0.0",
     "express-rate-limit": "^6.3.0",
+    "express-validator": "^7.1.0",
     "fast-safe-stringify": "^2.1.1",
     "fingerprint-generator": "^2.1.52",
     "firebase-admin": "^11.5.0",

packages/api/src/pubsub.ts

@@ -5,13 +5,11 @@ import { env } from './env'
 import { ReportType } from './generated/graphql'
 import {
   enqueueGeneratePreviewContentJob,
-  enqueueProcessYouTubeVideo,
   enqueueScoreJob,
   enqueueThumbnailJob,
   enqueueTriggerRuleJob,
 } from './utils/createTask'
 import { logger } from './utils/logger'
-import { isYouTubeVideoURL } from './utils/youtube'
 
 export type EntityEvent = { id: string }
 
@@ -60,24 +58,6 @@ export const createPubSubClient = (): PubsubClient => {
       })
 
       if (type === EntityType.ITEM) {
-        // if (await findGrantedFeatureByName(FeatureName.AISummaries, userId)) {
-        // await enqueueAISummarizeJob({
-        //   userId,
-        //   libraryItemId,
-        // })
-        // }
-
-        const isItemWithURL = (data: any): data is { originalUrl: string } => {
-          return 'originalUrl' in data
-        }
-
-        if (isItemWithURL(data) && isYouTubeVideoURL(data['originalUrl'])) {
-          await enqueueProcessYouTubeVideo({
-            userId,
-            libraryItemId: data.id,
-          })
-        }
-
         await enqueueScoreJob({
           userId,
           libraryItemId: data.id,

packages/api/src/routers/youtube_transcript_router.ts

@@ -0,0 +1,72 @@
+import express from 'express'
+import { body, matchedData, validationResult } from 'express-validator'
+import { userRepository } from '../repository/user'
+import { findLibraryItemById } from '../services/library_item'
+import { getClaimsByToken, getTokenByRequest } from '../utils/auth'
+import { enqueueProcessYouTubeVideo } from '../utils/createTask'
+import { logger } from '../utils/logger'
+import { isYouTubeVideoURL } from '../utils/youtube'
+
+interface RequestData {
+  libraryItemId: string
+}
+
+export function youtubeTranscriptRouter() {
+  const router = express.Router()
+
+  router.put(
+    '/',
+    body('libraryItemId').isString().notEmpty(),
+    async (req: express.Request, res) => {
+      const errors = validationResult(req)
+      if (!errors.isEmpty()) {
+        return res
+          .status(400)
+          .json({ errorCodes: errors.array().map((e) => e.msg as string) })
+      }
+
+      const token = getTokenByRequest(req)
+
+      let claims
+      try {
+        claims = await getClaimsByToken(token)
+        if (!claims) {
+          logger.info('failed to authorize')
+          return res.sendStatus(401)
+        }
+      } catch (e) {
+        logger.info('failed to authorize', e)
+        return res.sendStatus(401)
+      }
+
+      const { uid } = claims
+      const user = await userRepository.findById(uid)
+      if (!user) {
+        return res.status(404).send({ errorCodes: ['USER_NOT_FOUND'] })
+      }
+
+      const { libraryItemId } = matchedData<RequestData>(req)
+      const libraryItem = await findLibraryItemById(libraryItemId, uid, {
+        select: ['originalUrl'],
+      })
+      if (!libraryItem) {
+        return res.status(404).send({ errorCodes: ['LIBRARY_ITEM_NOT_FOUND'] })
+      }
+
+      if (!isYouTubeVideoURL(libraryItem.originalUrl)) {
+        return res.status(400).send({ errorCodes: ['NOT_YOUTUBE_VIDEO'] })
+      }
+
+      try {
+        await enqueueProcessYouTubeVideo({ libraryItemId, userId: uid })
+      } catch (error) {
+        logger.error('Error processing youtube video:', error)
+        return res.status(500).send({ errors: ['INTERNAL_SERVER_ERROR'] })
+      }
+
+      res.sendStatus(200)
+    }
+  )
+
+  return router
+}

packages/api/src/server.ts

@@ -6,6 +6,7 @@ import * as lw from '@google-cloud/logging-winston'
 import * as Sentry from '@sentry/node'
 import { json, urlencoded } from 'body-parser'
 import cookieParser from 'cookie-parser'
+import cors from 'cors'
 import express, { Express } from 'express'
 import * as httpContext from 'express-http-context2'
 import promBundle from 'express-prom-bundle'
@@ -27,6 +28,7 @@ import { integrationRouter } from './routers/integration_router'
 import { localDebugRouter } from './routers/local_debug_router'
 import { notificationRouter } from './routers/notification_router'
 import { pageRouter } from './routers/page_router'
+import { shortcutsRouter } from './routers/shortcuts_router'
 import { contentServiceRouter } from './routers/svc/content'
 import { emailsServiceRouter } from './routers/svc/emails'
 import { emailAttachmentRouter } from './routers/svc/email_attachment'
@@ -41,13 +43,13 @@ import { webhooksServiceRouter } from './routers/svc/webhooks'
 import { taskRouter } from './routers/task_router'
 import { textToSpeechRouter } from './routers/text_to_speech'
 import { userRouter } from './routers/user_router'
+import { youtubeTranscriptRouter } from './routers/youtube_transcript_router'
 import { sentryConfig } from './sentry'
 import { analytics } from './utils/analytics'
 import { corsConfig } from './utils/corsConfig'
 import { getClientFromUserAgent } from './utils/helpers'
 import { buildLogger, buildLoggerTransport, logger } from './utils/logger'
 import { apiLimiter, authLimiter } from './utils/rate_limit'
-import { shortcutsRouter } from './routers/shortcuts_router'
 
 const PORT = process.env.PORT || 4000
 
@@ -63,6 +65,7 @@ export const createApp = (): Express => {
   app.use(cookieParser())
   app.use(json({ limit: '100mb' }))
   app.use(urlencoded({ limit: '100mb', extended: true }))
+  app.use(cors(corsConfig))
 
   // set to true if behind a reverse proxy/load balancer
   app.set('trust proxy', env.server.trustProxy)
@@ -115,9 +118,9 @@ export const createApp = (): Express => {
   app.use('/svc/pubsub/webhooks', webhooksServiceRouter())
   app.use('/svc/pubsub/rss-feed', rssFeedRouter())
   app.use('/svc/pubsub/user', userServiceRouter())
-  // app.use('/svc/reminders', remindersServiceRouter())
   app.use('/svc/email-attachment', emailAttachmentRouter())
   app.use('/svc/following', followingServiceRouter())
+  app.use('/api/youtube-transcript', youtubeTranscriptRouter())
 
   if (env.dev.isLocal) {
     app.use('/local/debug', localDebugRouter())

yarn.lock

@@ -15611,6 +15611,14 @@ express-rate-limit@^6.3.0:
   resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-6.11.1.tgz#52e05c5d379cd5d06ae29665862436eb712e414a"
   integrity sha512-8+UpWtQY25lJaa4+3WxDBGDcAu4atcTruSs3QSL5VPEplYy6kmk84wutG9rUkkK5LmMQQ7TFHWLZYITwVNbbEg==
 
+express-validator@^7.1.0:
+  version "7.1.0"
+  resolved "https://registry.yarnpkg.com/express-validator/-/express-validator-7.1.0.tgz#e6571f6a4520560e1f3fae2ceab6f56c8f26c27b"
+  integrity sha512-ePn6NXjHRZiZkwTiU1Rl2hy6aUqmi6Cb4/s8sfUsKH7j2yYl9azSpl8xEHcOj1grzzQ+UBEoLWtE1s6FDxW++g==
+  dependencies:
+    lodash "^4.17.21"
+    validator "~13.12.0"
+
 express@^4.16.4, express@^4.17.1, express@^4.18.2:
   version "4.19.2"
   resolved "https://registry.yarnpkg.com/express/-/express-4.19.2.tgz#e25437827a3aa7f2a827bc8171bbbb664a356465"
@@ -29262,7 +29270,7 @@ string-template@~0.2.1:
   resolved "https://registry.yarnpkg.com/string-template/-/string-template-0.2.1.tgz#42932e598a352d01fc22ec3367d9d84eec6c9add"
   integrity sha1-QpMuWYo1LQH8IuwzZ9nYTuxsmt0=
 
-"string-width-cjs@npm:string-width@^4.2.0":
+"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.2.2, string-width@^4.2.3:
   version "4.2.3"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
   integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
@@ -29288,15 +29296,6 @@ string-width@^1.0.1:
     is-fullwidth-code-point "^2.0.0"
     strip-ansi "^4.0.0"
 
-"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.2.2, string-width@^4.2.3:
-  version "4.2.3"
-  resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010"
-  integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==
-  dependencies:
-    emoji-regex "^8.0.0"
-    is-fullwidth-code-point "^3.0.0"
-    strip-ansi "^6.0.1"
-
 string-width@^3.0.0:
   version "3.1.0"
   resolved "https://registry.yarnpkg.com/string-width/-/string-width-3.1.0.tgz#22767be21b62af1081574306f69ac51b62203961"
@@ -29451,7 +29450,7 @@ string_decoder@~1.1.1:
   dependencies:
     safe-buffer "~5.1.0"
 
-"strip-ansi-cjs@npm:strip-ansi@^6.0.1":
+"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.1:
   version "6.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
   integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
@@ -29486,13 +29485,6 @@ strip-ansi@^6.0.0:
   dependencies:
     ansi-regex "^5.0.0"
 
-strip-ansi@^6.0.1:
-  version "6.0.1"
-  resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9"
-  integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==
-  dependencies:
-    ansi-regex "^5.0.1"
-
 strip-ansi@^7.0.0:
   version "7.0.1"
   resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-7.0.1.tgz#61740a08ce36b61e50e65653f07060d000975fb2"
@@ -31464,6 +31456,11 @@ validate-npm-package-name@^4.0.0:
   dependencies:
     builtins "^5.0.0"
 
+validator@~13.12.0:
+  version "13.12.0"
+  resolved "https://registry.yarnpkg.com/validator/-/validator-13.12.0.tgz#7d78e76ba85504da3fee4fd1922b385914d4b35f"
+  integrity sha512-c1Q0mCiPlgdTVVVIJIrBuxNicYE+t/7oKeI9MWLj3fh/uq2Pxh/3eeWbVZ4OcGW1TUf53At0njHw5SMdA3tmMg==
+
 [email protected], value-or-promise@^1.0.11:
   version "1.0.11"
   resolved "https://registry.yarnpkg.com/value-or-promise/-/value-or-promise-1.0.11.tgz#3e90299af31dd014fe843fe309cefa7c1d94b140"
@@ -32171,7 +32168,7 @@ [email protected]:
   resolved "https://registry.yarnpkg.com/workerpool/-/workerpool-6.2.1.tgz#46fc150c17d826b86a008e5a4508656777e9c343"
   integrity sha512-ILEIE97kDZvF9Wb9f6h5aXK4swSlKGUcOEGiIYb2OOu/IrDU9iwj0fD//SsA6E5ibwJxpEvhullJY4Sl4GcpAw==
 
-"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0":
+"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0:
   version "7.0.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
   integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
@@ -32197,15 +32194,6 @@ wrap-ansi@^6.0.1, wrap-ansi@^6.2.0:
     string-width "^4.1.0"
     strip-ansi "^6.0.0"
 
-wrap-ansi@^7.0.0:
-  version "7.0.0"
-  resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-7.0.0.tgz#67e145cff510a6a6984bdf1152911d69d2eb9e43"
-  integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==
-  dependencies:
-    ansi-styles "^4.0.0"
-    string-width "^4.1.0"
-    strip-ansi "^6.0.0"
-
 wrap-ansi@^8.1.0:
   version "8.1.0"
   resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-8.1.0.tgz#56dc22368ee570face1b49819975d9b9a5ead214"
@@ -32546,7 +32534,7 @@ zod-to-json-schema@^3.22.4:
   resolved "https://registry.yarnpkg.com/zod-to-json-schema/-/zod-to-json-schema-3.23.0.tgz#4fc60e88d3c709eedbfaae3f92f8a7bf786469f2"
   integrity sha512-az0uJ243PxsRIa2x1WmNE/pnuA05gUq/JB8Lwe1EDCCL/Fz9MgjYQ0fPlyc2Tcv6aF2ZA7WM5TWaRZVEFaAIag==
 
-[email protected]:
+[email protected], zod@^3.23.8:
   version "3.23.8"
   resolved "https://registry.yarnpkg.com/zod/-/zod-3.23.8.tgz#e37b957b5d52079769fb8097099b592f0ef4067d"
   integrity sha512-XBx9AXhXktjUqnepgTiE5flcKIYWi/rme0Eaj+5Y0lftuGBq+jyRu/md4WnuxqgP1ubdpNCsYEYPxrzVHD8d6g==