danny-avila
diff --git a/‎.env.example
Lines changed: 8 additions & 0 deletions b/‎.env.example
Lines changed: 8 additions & 0 deletions
diff --git a/‎api/app/clients/BaseClient.js
Lines changed: 8 additions & 0 deletions b/‎api/app/clients/BaseClient.js
Lines changed: 8 additions & 0 deletions
diff --git a/‎api/app/clients/OpenAIClient.js
Lines changed: 3 additions & 0 deletions b/‎api/app/clients/OpenAIClient.js
Lines changed: 3 additions & 0 deletions
diff --git a/‎api/app/clients/PluginsClient.js
Lines changed: 9 additions & 0 deletions b/‎api/app/clients/PluginsClient.js
Lines changed: 9 additions & 0 deletions
diff --git a/‎api/cache/getLogStores.js
Lines changed: 6 additions & 0 deletions b/‎api/cache/getLogStores.js
Lines changed: 6 additions & 0 deletions
diff --git a/‎api/package.json
Lines changed: 1 addition & 0 deletions b/‎api/package.json
Lines changed: 1 addition & 0 deletions
diff --git a/‎api/server/controllers/AskController.js
Lines changed: 4 additions & 3 deletions b/‎api/server/controllers/AskController.js
Lines changed: 4 additions & 3 deletions
diff --git a/‎api/server/controllers/EditController.js
Lines changed: 4 additions & 3 deletions b/‎api/server/controllers/EditController.js
Lines changed: 4 additions & 3 deletions
diff --git a/‎api/server/controllers/assistants/chatV2.js
Lines changed: 3 additions & 0 deletions b/‎api/server/controllers/assistants/chatV2.js
Lines changed: 3 additions & 0 deletions
diff --git a/‎api/server/routes/ask/gptPlugins.js
Lines changed: 4 additions & 3 deletions b/‎api/server/routes/ask/gptPlugins.js
Lines changed: 4 additions & 3 deletions
@@ -257,6 +257,14 @@ MEILI_NO_ANALYTICS=true
 MEILI_HOST=http://0.0.0.0:7700
 MEILI_MASTER_KEY=DrhYf7zENyR6AlUCKmnz0eYASOQdl6zxH7s7MKFSfFCt
 
+
+#==================================================#
+#          Speech to Text & Text to Speech         #
+#==================================================#
+
+STT_API_KEY=
+TTS_API_KEY=
+
 #===================================================#
 #                    User System                    #
 #===================================================#
 
@@ -373,6 +373,14 @@ class BaseClient {
     const { user, head, isEdited, conversationId, responseMessageId, saveOptions, userMessage } =
       await this.handleStartMethods(message, opts);
 
+    if (opts.progressCallback) {
+      opts.onProgress = opts.progressCallback.call(null, {
+        ...(opts.progressOptions ?? {}),
+        parentMessageId: userMessage.messageId,
+        messageId: responseMessageId,
+      });
+    }
+
     const { generation = '' } = opts;
 
     // It's not necessary to push to currentMessages
 
@@ -27,6 +27,7 @@ const {
   createContextHandlers,
 } = require('./prompts');
 const { encodeAndFormat } = require('~/server/services/Files/images/encode');
+const { updateTokenWebsocket } = require('~/server/services/Files/Audio');
 const { isEnabled, sleep } = require('~/server/utils');
 const { handleOpenAIErrors } = require('./tools/util');
 const spendTokens = require('~/models/spendTokens');
@@ -594,6 +595,7 @@ class OpenAIClient extends BaseClient {
         payload,
         (progressMessage) => {
           if (progressMessage === '[DONE]') {
+            updateTokenWebsocket('[DONE]');
             return;
           }
 
@@ -1216,6 +1218,7 @@ ${convo}
           });
 
         const azureDelay = this.modelOptions.model?.includes('gpt-4') ? 30 : 17;
+
         for await (const chunk of stream) {
           const token = chunk.choices[0]?.delta?.content || '';
           intermediateReply += token;
 
@@ -250,6 +250,7 @@ class PluginsClient extends OpenAIClient {
       this.setOptions(opts);
       return super.sendMessage(message, opts);
     }
+
     logger.debug('[PluginsClient] sendMessage', { userMessageText: message, opts });
     const {
       user,
@@ -264,6 +265,14 @@ class PluginsClient extends OpenAIClient {
       onToolEnd,
     } = await this.handleStartMethods(message, opts);
 
+    if (opts.progressCallback) {
+      opts.onProgress = opts.progressCallback.call(null, {
+        ...(opts.progressOptions ?? {}),
+        parentMessageId: opts.progressOptions?.parentMessageId ?? userMessage.messageId,
+        messageId: responseMessageId,
+      });
+    }
+
     this.currentMessages.push(userMessage);
 
     let {
 
@@ -7,6 +7,7 @@ const keyvMongo = require('./keyvMongo');
 
 const { BAN_DURATION, USE_REDIS } = process.env ?? {};
 const THIRTY_MINUTES = 1800000;
+const TEN_MINUTES = 600000;
 
 const duration = math(BAN_DURATION, 7200000);
 
@@ -24,6 +25,10 @@ const config = isEnabled(USE_REDIS)
   ? new Keyv({ store: keyvRedis })
   : new Keyv({ namespace: CacheKeys.CONFIG_STORE });
 
+const audioRuns = isEnabled(USE_REDIS) // ttl: 30 minutes
+  ? new Keyv({ store: keyvRedis, ttl: TEN_MINUTES })
+  : new Keyv({ namespace: CacheKeys.AUDIO_RUNS, ttl: TEN_MINUTES });
+
 const tokenConfig = isEnabled(USE_REDIS) // ttl: 30 minutes
   ? new Keyv({ store: keyvRedis, ttl: THIRTY_MINUTES })
   : new Keyv({ namespace: CacheKeys.TOKEN_CONFIG, ttl: THIRTY_MINUTES });
@@ -64,6 +69,7 @@ const namespaces = {
   [CacheKeys.TOKEN_CONFIG]: tokenConfig,
   [CacheKeys.GEN_TITLE]: genTitle,
   [CacheKeys.MODEL_QUERIES]: modelQueries,
+  [CacheKeys.AUDIO_RUNS]: audioRuns,
 };
 
 /**
 
@@ -94,6 +94,7 @@
     "ua-parser-js": "^1.0.36",
     "winston": "^3.11.0",
     "winston-daily-rotate-file": "^4.7.1",
+    "ws": "^8.17.0",
     "zod": "^3.22.4"
   },
   "devDependencies": {
 
@@ -105,11 +105,12 @@ const AskController = async (req, res, next, initializeClient, addTitle) => {
       getReqData,
       onStart,
       abortController,
-      onProgress: progressCallback.call(null, {
+      progressCallback,
+      progressOptions: {
         res,
         text,
-        parentMessageId: overrideParentMessageId || userMessageId,
-      }),
+        // parentMessageId: overrideParentMessageId || userMessageId,
+      },
     };
 
     let response = await client.sendMessage(text, messageOptions);
 
@@ -112,11 +112,12 @@ const EditController = async (req, res, next, initializeClient) => {
       getReqData,
       onStart,
       abortController,
-      onProgress: progressCallback.call(null, {
+      progressCallback,
+      progressOptions: {
         res,
         text,
-        parentMessageId: overrideParentMessageId || userMessageId,
-      }),
+        // parentMessageId: overrideParentMessageId || userMessageId,
+      },
     });
 
     const conversation = await getConvo(user, conversationId);
 
@@ -520,6 +520,7 @@ const chatV2 = async (req, res) => {
         handlers,
         thread_id,
         attachedFileIds,
+        parentMessageId,
         responseMessage: openai.responseMessage,
         // streamOptions: {
 
@@ -532,6 +533,7 @@ const chatV2 = async (req, res) => {
       });
 
       response = streamRunManager;
+      response.text = streamRunManager.intermediateText;
     };
 
     await processRun();
@@ -554,6 +556,7 @@ const chatV2 = async (req, res) => {
     /** @type {ResponseMessage} */
     const responseMessage = {
       ...(response.responseMessage ?? response.finalMessage),
+      text: response.text,
       parentMessageId: userMessageId,
       conversationId,
       user: req.user.id,
 
@@ -174,12 +174,13 @@ router.post(
         onStart,
         getPartialText,
         ...endpointOption,
-        onProgress: progressCallback.call(null, {
+        progressCallback,
+        progressOptions: {
           res,
           text,
-          parentMessageId: overrideParentMessageId || userMessageId,
+          // parentMessageId: overrideParentMessageId || userMessageId,
           plugins,
-        }),
+        },
         abortController,
       });