feat: employ file strategies for upload/delete files

Author: danny-avila

api/server/routes/files/files.js

@@ -1,33 +1,14 @@
 const { z } = require('zod');
-const path = require('path');
-const fs = require('fs').promises;
 const express = require('express');
+const { FileSources } = require('librechat-data-provider');
+const { getStrategyFunctions } = require('~/server/services/Files/strategies');
 const { deleteFiles, getFiles } = require('~/models');
 const { logger } = require('~/config');
 
 const router = express.Router();
 
 const isUUID = z.string().uuid();
 
-const isValidPath = (req, base, subfolder, filepath) => {
-  const normalizedBase = path.resolve(base, subfolder, req.user.id);
-  const normalizedFilepath = path.resolve(filepath);
-  return normalizedFilepath.startsWith(normalizedBase);
-};
-
-const deleteFile = async (req, file) => {
-  const { publicPath } = req.app.locals.paths;
-  const parts = file.filepath.split(path.sep);
-  const subfolder = parts[1];
-  const filepath = path.join(publicPath, file.filepath);
-
-  if (!isValidPath(req, publicPath, subfolder, filepath)) {
-    throw new Error('Invalid file path');
-  }
-
-  await fs.unlink(filepath);
-};
-
 router.get('/', async (req, res) => {
   try {
     const files = await getFiles({ user: req.user.id });
@@ -41,6 +22,8 @@ router.get('/', async (req, res) => {
 router.delete('/', async (req, res) => {
   try {
     const { files: _files } = req.body;
+
+    /** @type {MongoFile[]} */
     const files = _files.filter((file) => {
       if (!file.file_id) {
         return false;
@@ -57,9 +40,24 @@ router.delete('/', async (req, res) => {
     }
 
     const file_ids = files.map((file) => file.file_id);
+    const deletionMethods = {};
     const promises = [];
     promises.push(await deleteFiles(file_ids));
+
     for (const file of files) {
+      const source = file.source ?? FileSources.local;
+
+      if (deletionMethods[source]) {
+        promises.push(deletionMethods[source](req, file));
+        continue;
+      }
+
+      const { deleteFile } = getStrategyFunctions(source);
+      if (!deleteFile) {
+        throw new Error(`Delete function not implemented for ${source}`);
+      }
+
+      deletionMethods[source] = deleteFile;
       promises.push(deleteFile(req, file));
     }
 

api/server/routes/files/images.js

@@ -2,7 +2,7 @@ const { z } = require('zod');
 const fs = require('fs').promises;
 const express = require('express');
 const upload = require('./multer');
-const { localStrategy } = require('~/server/services/Files');
+const { processImageUpload } = require('~/server/services/Files/process');
 const { logger } = require('~/config');
 
 const router = express.Router();
@@ -35,7 +35,7 @@ router.post('/', upload.single('file'), async (req, res) => {
     metadata.temp_file_id = metadata.file_id;
     metadata.file_id = req.file_id;
 
-    await localStrategy({ req, res, file, metadata });
+    await processImageUpload({ req, res, file, metadata });
   } catch (error) {
     logger.error('[/files/images] Error processing file:', error);
     try {

api/server/services/Files/Firebase/crud.js

@@ -8,7 +8,7 @@ const { getFirebaseStorage } = require('./initialize');
  * @param {string} fileName - The name of the file to delete.
  * @returns {Promise<void>} A promise that resolves when the file is deleted.
  */
-async function deleteFileFromFirebase(basePath, fileName) {
+async function deleteFile(basePath, fileName) {
   const storage = getFirebaseStorage();
   if (!storage) {
     console.error('Firebase is not initialized. Cannot delete file from Firebase Storage.');
@@ -27,39 +27,38 @@ async function deleteFileFromFirebase(basePath, fileName) {
 }
 
 /**
- * Saves an image from a given URL to Firebase Storage. The function first initializes the Firebase Storage
- * reference, then uploads the image to a specified basePath in the Firebase Storage. It handles initialization
+ * Saves an file from a given URL to Firebase Storage. The function first initializes the Firebase Storage
+ * reference, then uploads the file to a specified basePath in the Firebase Storage. It handles initialization
  * errors and upload errors, logging them to the console. If the upload is successful, the file name is returned.
  *
  * @param {Object} params - The parameters object.
  * @param {string} params.userId - The user's unique identifier. This is used to create a user-specific basePath
  *                                 in Firebase Storage.
- * @param {string} params.URL - The URL of the image to be uploaded. The image at this URL will be fetched
+ * @param {string} params.URL - The URL of the file to be uploaded. The file at this URL will be fetched
  *                              and uploaded to Firebase Storage.
- * @param {string} params.fileName - The name that will be used to save the image in Firebase Storage. This
+ * @param {string} params.fileName - The name that will be used to save the file in Firebase Storage. This
  *                                   should include the file extension.
- * @param {string} [params.basePath='images'] - Optional. The base basePath in Firebase Storage where the image will
+ * @param {string} [params.basePath='images'] - Optional. The base basePath in Firebase Storage where the file will
  *                                          be stored. Defaults to 'images' if not specified.
  *
  * @returns {Promise<string|null>}
- *          A promise that resolves to the file name if the image is successfully uploaded, or null if there
+ *          A promise that resolves to the file name if the file is successfully uploaded, or null if there
  *          is an error in initialization or upload.
  */
 async function saveURLToFirebase({ userId, URL, fileName, basePath = 'images' }) {
   const storage = getFirebaseStorage();
   if (!storage) {
-    console.error('Firebase is not initialized. Cannot save image to Firebase Storage.');
+    console.error('Firebase is not initialized. Cannot save file to Firebase Storage.');
     return null;
   }
 
   const storageRef = ref(storage, `${basePath}/${userId.toString()}/${fileName}`);
 
   try {
-    // Upload image to Firebase Storage using the image URL
     await uploadBytes(storageRef, await fetch(URL).then((response) => response.buffer()));
     return fileName;
   } catch (error) {
-    console.error('Error uploading image to Firebase Storage:', error.message);
+    console.error('Error uploading file to Firebase Storage:', error.message);
     return null;
   }
 }
@@ -97,8 +96,78 @@ async function getFirebaseURL({ fileName, basePath = 'images' }) {
   }
 }
 
+/**
+ * Uploads a buffer to Firebase Storage.
+ *
+ * @param {Object} params - The parameters object.
+ * @param {string} params.userId - The user's unique identifier. This is used to create a user-specific basePath
+ *                                 in Firebase Storage.
+ * @param {string} params.fileName - The name of the file to be saved in Firebase Storage.
+ * @param {string} params.buffer - The buffer to be uploaded.
+ * @param {string} [params.basePath='images'] - Optional. The base basePath in Firebase Storage where the file will
+ *                                          be stored. Defaults to 'images' if not specified.
+ *
+ * @returns {Promise<string>} - A promise that resolves to the download URL of the uploaded file.
+ */
+async function saveBufferToFirebase({ userId, buffer, fileName, basePath = 'images' }) {
+  const storage = getFirebaseStorage();
+  if (!storage) {
+    throw new Error('Firebase is not initialized');
+  }
+
+  const storageRef = ref(storage, `${basePath}/${userId}/${fileName}`);
+  await uploadBytes(storageRef, buffer);
+
+  // Assuming you have a function to get the download URL
+  return await getFirebaseURL({ fileName, basePath: `${basePath}/${userId}` });
+}
+
+/**
+ * Extracts and decodes the file path from a Firebase Storage URL.
+ *
+ * @param {string} urlString - The Firebase Storage URL.
+ * @returns {string} The decoded file path.
+ */
+function extractFirebaseFilePath(urlString) {
+  try {
+    const url = new URL(urlString);
+    const pathRegex = /\/o\/(.+?)(\?|$)/;
+    const match = url.pathname.match(pathRegex);
+
+    if (match && match[1]) {
+      return decodeURIComponent(match[1]);
+    }
+
+    return '';
+  } catch (error) {
+    // If URL parsing fails, return an empty string
+    return '';
+  }
+}
+
+/**
+ * Deletes a file from Firebase storage. This function determines the subfolder (either 'images' or 'documents')
+ * based on the file type and constructs a Firebase storage path using the user's ID and the file name.
+ * It then deletes the file from Firebase storage.
+ *
+ * @param {Object} req - The request object from Express. It should contain a `user` object with an `id` property.
+ * @param {Object} file - The file object to be deleted. It should have `filepath` and `type` properties.
+ *                        `filepath` is a string representing the file's name, and `type` is used to determine
+ *                        the file's storage subfolder in Firebase.
+ *
+ * @returns {Promise<void>}
+ *          A promise that resolves when the file has been successfully deleted from Firebase storage.
+ *          Throws an error if there is an issue with deletion.
+ */
+const deleteFirebaseFile = async (req, file) => {
+  const fileName = extractFirebaseFilePath(file.filepath);
+  await deleteFile('', fileName);
+};
+
 module.exports = {
-  saveURLToFirebase,
+  deleteFile,
   getFirebaseURL,
-  deleteFileFromFirebase,
+  saveURLToFirebase,
+  deleteFirebaseFile,
+  saveBufferToFirebase,
 };

api/server/services/Files/Firebase/images.js

@@ -0,0 +1,53 @@
+const fs = require('fs');
+const path = require('path');
+const sharp = require('sharp');
+const { saveBufferToFirebase } = require('./crud');
+const { resizeImage } = require('../images/resize');
+
+/**
+ * Converts an image file to the WebP format. The function first resizes the image based on the specified
+ * resolution.
+ *
+ *
+ * @param {Object} req - The request object from Express. It should have a `user` property with an `id`
+ *                       representing the user, and an `app.locals.paths` object with an `imageOutput` path.
+ * @param {Express.Multer.File} file - The file object, which is part of the request. The file object should
+ *                                     have a `path` property that points to the location of the uploaded file.
+ * @param {string} [resolution='high'] - Optional. The desired resolution for the image resizing. Default is 'high'.
+ *
+ * @returns {Promise<{ filepath: string, bytes: number, width: number, height: number}>}
+ *          A promise that resolves to an object containing:
+ *            - filepath: The path where the converted WebP image is saved.
+ *            - bytes: The size of the converted image in bytes.
+ *            - width: The width of the converted image.
+ *            - height: The height of the converted image.
+ */
+async function uploadImageToFirebase(req, file, resolution = 'high') {
+  const inputFilePath = file.path;
+  const { buffer: resizedBuffer, width, height } = await resizeImage(inputFilePath, resolution);
+  const extension = path.extname(inputFilePath);
+  const userId = req.user.id;
+
+  let webPBuffer;
+  let fileName = path.basename(inputFilePath);
+  if (extension.toLowerCase() === '.webp') {
+    webPBuffer = resizedBuffer;
+  } else {
+    webPBuffer = await sharp(resizedBuffer).toFormat('webp').toBuffer();
+    // Replace or append the correct extension
+    const extRegExp = new RegExp(path.extname(fileName) + '$');
+    fileName = fileName.replace(extRegExp, '.webp');
+    if (!path.extname(fileName)) {
+      fileName += '.webp';
+    }
+  }
+
+  const downloadURL = await saveBufferToFirebase({ userId, buffer: webPBuffer, fileName });
+
+  await fs.promises.unlink(inputFilePath);
+
+  const bytes = Buffer.byteLength(webPBuffer);
+  return { filepath: downloadURL, bytes, width, height };
+}
+
+module.exports = { uploadImageToFirebase };

api/server/services/Files/Firebase/index.js

@@ -1,7 +1,9 @@
 const crud = require('./crud');
+const images = require('./images');
 const initialize = require('./initialize');
 
 module.exports = {
   ...crud,
+  ...images,
   ...initialize,
 };

api/server/services/Files/Local/crud.js

@@ -126,4 +126,49 @@ async function getLocalFileURL({ fileName, basePath = 'images' }) {
   return path.posix.join('/', basePath, fileName);
 }
 
-module.exports = { saveFile, saveLocalImage, saveFileFromURL, getLocalFileURL };
+/**
+ * Validates if a given filepath is within a specified subdirectory under a base path. This function constructs
+ * the expected base path using the base, subfolder, and user id from the request, and then checks if the
+ * provided filepath starts with this constructed base path.
+ *
+ * @param {Express.Request} req - The request object from Express. It should contain a `user` property with an `id`.
+ * @param {string} base - The base directory path.
+ * @param {string} subfolder - The subdirectory under the base path.
+ * @param {string} filepath - The complete file path to be validated.
+ *
+ * @returns {boolean}
+ *          Returns true if the filepath is within the specified base and subfolder, false otherwise.
+ */
+const isValidPath = (req, base, subfolder, filepath) => {
+  const normalizedBase = path.resolve(base, subfolder, req.user.id);
+  const normalizedFilepath = path.resolve(filepath);
+  return normalizedFilepath.startsWith(normalizedBase);
+};
+
+/**
+ * Deletes a file from the filesystem. This function takes a file object, constructs the full path, and
+ * verifies the path's validity before deleting the file. If the path is invalid, an error is thrown.
+ *
+ * @param {Express.Request} req - The request object from Express. It should have an `app.locals.paths` object with
+ *                       a `publicPath` property.
+ * @param {MongoFile} file - The file object to be deleted. It should have a `filepath` property that is
+ *                           a string representing the path of the file relative to the publicPath.
+ *
+ * @returns {Promise<void>}
+ *          A promise that resolves when the file has been successfully deleted, or throws an error if the
+ *          file path is invalid or if there is an error in deletion.
+ */
+const deleteLocalFile = async (req, file) => {
+  const { publicPath } = req.app.locals.paths;
+  const parts = file.filepath.split(path.sep);
+  const subfolder = parts[1];
+  const filepath = path.join(publicPath, file.filepath);
+
+  if (!isValidPath(req, publicPath, subfolder, filepath)) {
+    throw new Error('Invalid file path');
+  }
+
+  await fs.promises.unlink(filepath);
+};
+
+module.exports = { saveFile, saveLocalImage, saveFileFromURL, getLocalFileURL, deleteLocalFile };

api/server/services/Files/Local/images.js

@@ -1,8 +1,8 @@
 const fs = require('fs');
 const path = require('path');
 const sharp = require('sharp');
-const { updateFile, createFile } = require('~/models');
 const { resizeImage } = require('../images/resize');
+const { updateFile } = require('~/models');
 
 /**
  * Converts an image file to the WebP format. The function first resizes the image based on the specified
@@ -26,7 +26,7 @@ const { resizeImage } = require('../images/resize');
  *            - width: The width of the converted image.
  *            - height: The height of the converted image.
  */
-async function convertToWebP(req, file, resolution = 'high') {
+async function uploadLocalImage(req, file, resolution = 'high') {
   const inputFilePath = file.path;
   const { buffer: resizedBuffer, width, height } = await resizeImage(inputFilePath, resolution);
   const extension = path.extname(inputFilePath);
@@ -94,36 +94,4 @@ async function encodeLocal(req, file) {
   return await Promise.all(promises);
 }
 
-/**
- * Applies the local strategy for image uploads.
- * Saves file metadata to the database with an expiry TTL.
- * Files must be deleted from the server filesystem manually.
- *
- * @param {Object} params - The parameters object.
- * @param {Express.Request} params.req - The Express request object.
- * @param {Express.Response} params.res - The Express response object.
- * @param {Express.Multer.File} params.file - The uploaded file.
- * @param {ImageMetadata} params.metadata - Additional metadata for the file.
- * @returns {Promise<void>}
- */
-const saveLocalImage = async ({ req, res, file, metadata }) => {
-  const { file_id, temp_file_id } = metadata;
-  const { filepath, bytes, width, height } = await convertToWebP(req, file);
-  const result = await createFile(
-    {
-      user: req.user.id,
-      file_id,
-      temp_file_id,
-      bytes,
-      filepath,
-      filename: file.originalname,
-      type: 'image/webp',
-      width,
-      height,
-    },
-    true,
-  );
-  res.status(200).json({ message: 'File uploaded and processed successfully', ...result });
-};
-
-module.exports = { convertToWebP, encodeImage, encodeLocal, saveLocalImage };
+module.exports = { uploadLocalImage, encodeImage, encodeLocal };