feat: employ strategies for vision requests

Author: danny-avila

api/server/services/Files/Firebase/images.js

@@ -3,6 +3,7 @@ const path = require('path');
 const sharp = require('sharp');
 const { saveBufferToFirebase } = require('./crud');
 const { resizeImage } = require('../images/resize');
+const { updateFile } = require('~/models');
 
 /**
  * Converts an image file to the WebP format. The function first resizes the image based on the specified
@@ -50,4 +51,19 @@ async function uploadImageToFirebase(req, file, resolution = 'high') {
   return { filepath: downloadURL, bytes, width, height };
 }
 
-module.exports = { uploadImageToFirebase };
+/**
+ * Local: Updates the file and returns the URL in expected order/format
+ * for image payload handling: tuple order of [filepath, URL].
+ * @param {Object} req - The request object.
+ * @param {MongoFile} file - The file object.
+ * @returns {Promise<[MongoFile, string]>} - A promise that resolves to an array of results from updateFile and encodeImage.
+ */
+async function prepareImageURL(req, file) {
+  const { filepath } = file;
+  const promises = [];
+  promises.push(updateFile({ file_id: file.file_id }));
+  promises.push(filepath);
+  return await Promise.all(promises);
+}
+
+module.exports = { uploadImageToFirebase, prepareImageURL };

api/server/services/Files/Local/images.js

@@ -74,12 +74,13 @@ function encodeImage(imagePath) {
 }
 
 /**
- * Local: Updates the file and encodes the image.
+ * Local: Updates the file and encodes the image to base64,
+ * for image payload handling: tuple order of [filepath, base64].
  * @param {Object} req - The request object.
  * @param {MongoFile} file - The file object.
- * @returns {Promise<[MongoFile, string]>}  - A promise that resolves to an array of results from updateFile and encodeImage.
+ * @returns {Promise<[MongoFile, string]>} - A promise that resolves to an array of results from updateFile and encodeImage.
  */
-async function encodeLocal(req, file) {
+async function prepareImagesLocal(req, file) {
   const { publicPath, imageOutput } = req.app.locals.paths;
   const userPath = path.join(imageOutput, req.user.id);
 
@@ -94,4 +95,4 @@ async function encodeLocal(req, file) {
   return await Promise.all(promises);
 }
 
-module.exports = { uploadLocalImage, encodeImage, encodeLocal };
+module.exports = { uploadLocalImage, encodeImage, prepareImagesLocal };

api/server/services/Files/images/encode.js

@@ -1,9 +1,5 @@
 const { EModelEndpoint, FileSources } = require('librechat-data-provider');
-const { encodeLocal } = require('../Local/images');
-
-const encodeStrategies = {
-  [FileSources.local]: encodeLocal,
-};
+const { getStrategyFunctions } = require('../strategies');
 
 /**
  * Encodes and formats the given files.
@@ -13,32 +9,45 @@ const encodeStrategies = {
  * @returns {Promise<Object>} - A promise that resolves to the result object containing the encoded images and file details.
  */
 async function encodeAndFormat(req, files, endpoint) {
-  const { fileStrategy } = req.app.locals;
-  /**
-   * @type {function(Express.Request, MongoFile): Promise<[MongoFile, string]>}
-   */
-  const updateAndEncode = encodeStrategies[fileStrategy];
-
   const promises = [];
+  const encodingMethods = {};
+
   for (let file of files) {
-    promises.push(updateAndEncode(req, file));
+    const source = file.source ?? FileSources.local;
+
+    if (encodingMethods[source]) {
+      promises.push(encodingMethods[source](req, file));
+      continue;
+    }
+
+    const { prepareImagePayload } = getStrategyFunctions(source);
+    if (!prepareImagePayload) {
+      throw new Error(`Encoding function not implemented for ${source}`);
+    }
+
+    encodingMethods[source] = prepareImagePayload;
+    promises.push(prepareImagePayload(req, file));
   }
 
   // TODO: make detail configurable, as of now resizing is done
   // to prefer "high" but "low" may be used if the image is small enough
   const detail = req.body.detail ?? 'auto';
-  const encodedImages = await Promise.all(promises);
+
+  /** @type {Array<[MongoFile, string]>} */
+  const formattedImages = await Promise.all(promises);
 
   const result = {
     files: [],
     image_urls: [],
   };
 
-  for (const [file, base64] of encodedImages) {
+  for (const [file, imageContent] of formattedImages) {
     const imagePart = {
       type: 'image_url',
       image_url: {
-        url: `data:image/webp;base64,${base64}`,
+        url: imageContent.startsWith('http')
+          ? imageContent
+          : `data:image/webp;base64,${imageContent}`,
         detail,
       },
     };

api/server/services/Files/strategies.js

@@ -3,16 +3,24 @@ const {
   getFirebaseURL,
   saveURLToFirebase,
   deleteFirebaseFile,
+  prepareImageURL,
   uploadImageToFirebase,
 } = require('./Firebase');
-const { getLocalFileURL, saveFileFromURL, deleteLocalFile, uploadLocalImage } = require('./Local');
+const {
+  getLocalFileURL,
+  saveFileFromURL,
+  deleteLocalFile,
+  uploadLocalImage,
+  prepareImagesLocal,
+} = require('./Local');
 
 // Firebase Strategy Functions
 const firebaseStrategy = () => ({
   // saveFile:
   saveURL: saveURLToFirebase,
   getFileURL: getFirebaseURL,
   deleteFile: deleteFirebaseFile,
+  prepareImagePayload: prepareImageURL,
   handleImageUpload: uploadImageToFirebase,
 });
 
@@ -23,6 +31,7 @@ const localStrategy = () => ({
   getFileURL: getLocalFileURL,
   deleteFile: deleteLocalFile,
   handleImageUpload: uploadLocalImage,
+  prepareImagePayload: prepareImagesLocal,
 });
 
 // Strategy Selector