🖇️ feat: Send Attachments Directly to Providers

api/server/controllers/agents/client.js

@@ -11,6 +11,9 @@ const {
   memoryInstructions,
   formatContentStrings,
   createMemoryProcessor,
+  encodeAndFormatAudios,
+  encodeAndFormatVideos,
+  encodeAndFormatDocuments,
 } = require('@librechat/api');
 const {
   Callback,
@@ -33,18 +36,21 @@ const {
   AgentCapabilities,
   bedrockInputSchema,
   removeNullishValues,
+  isDocumentSupportedEndpoint,
 } = require('librechat-data-provider');
 const { addCacheControl, createContextHandlers } = require('~/app/clients/prompts');
 const { initializeAgent } = require('~/server/services/Endpoints/agents/agent');
 const { spendTokens, spendStructuredTokens } = require('~/models/spendTokens');
 const { getFormattedMemories, deleteMemory, setMemory } = require('~/models');
 const { encodeAndFormat } = require('~/server/services/Files/images/encode');
 const { getProviderConfig } = require('~/server/services/Endpoints');
+const { getStrategyFunctions } = require('~/server/services/Files');
 const { checkCapability } = require('~/server/services/Config');
 const BaseClient = require('~/app/clients/BaseClient');
 const { getRoleByName } = require('~/models/Role');
 const { loadAgent } = require('~/models/Agent');
 const { getMCPManager } = require('~/config');
+const { getFiles } = require('~/models');
 
 const omitTitleOptions = new Set([
   'stream',
@@ -222,6 +228,168 @@ class AgentClient extends BaseClient {
     return files;
   }
 
+  async addDocuments(message, attachments) {
+    const documentResult = await encodeAndFormatDocuments(
+      this.options.req,
+      attachments,
+      this.options.agent.provider,
+      getStrategyFunctions,
+    );
+    message.documents =
+      documentResult.documents && documentResult.documents.length
+        ? documentResult.documents
+        : undefined;
+    return documentResult.files;
+  }
+
+  async addVideos(message, attachments) {
+    const videoResult = await encodeAndFormatVideos(
+      this.options.req,
+      attachments,
+      this.options.agent.provider,
+      getStrategyFunctions,
+    );
+    message.videos =
+      videoResult.videos && videoResult.videos.length ? videoResult.videos : undefined;
+    return videoResult.files;
+  }
+
+  async addAudios(message, attachments) {
+    const audioResult = await encodeAndFormatAudios(
+      this.options.req,
+      attachments,
+      this.options.agent.provider,
+      getStrategyFunctions,
+    );
+    message.audios =
+      audioResult.audios && audioResult.audios.length ? audioResult.audios : undefined;
+    return audioResult.files;
+  }
+
+  /**
+   * Override addPreviousAttachments to handle all file types, not just images
+   * @param {TMessage[]} _messages
+   * @returns {Promise<TMessage[]>}
+   */
+  async addPreviousAttachments(_messages) {
+    if (!this.options.resendFiles) {
+      return _messages;
+    }
+
+    const seen = new Set();
+    const attachmentsProcessed =
+      this.options.attachments && !(this.options.attachments instanceof Promise);
+    if (attachmentsProcessed) {
+      for (const attachment of this.options.attachments) {
+        seen.add(attachment.file_id);
+      }
+    }
+
+    /**
+     *
+     * @param {TMessage} message
+     */
+    const processMessage = async (message) => {
+      if (!this.message_file_map) {
+        /** @type {Record<string, MongoFile[]> */
+        this.message_file_map = {};
+      }
+
+      const fileIds = [];
+      for (const file of message.files) {
+        if (seen.has(file.file_id)) {
+          continue;
+        }
+        fileIds.push(file.file_id);
+        seen.add(file.file_id);
+      }
+
+      if (fileIds.length === 0) {
+        return message;
+      }
+
+      const files = await getFiles(
+        {
+          file_id: { $in: fileIds },
+        },
+        {},
+        {},
+      );
+
+      await this.processAttachments(message, files);
+
+      this.message_file_map[message.messageId] = files;
+      return message;
+    };
+
+    const promises = [];
+
+    for (const message of _messages) {
+      if (!message.files) {
+        promises.push(message);
+        continue;
+      }
+
+      promises.push(processMessage(message));
+    }
+
+    const messages = await Promise.all(promises);
+
+    this.checkVisionRequest(Object.values(this.message_file_map ?? {}).flat());
+    return messages;
+  }
+
+  async processAttachments(message, attachments) {
+    const categorizedAttachments = {
+      images: [],
+      documents: [],
+      videos: [],
+      audios: [],
+    };
+
+    for (const file of attachments) {
+      if (file.type.startsWith('image/')) {
+        categorizedAttachments.images.push(file);
+      } else if (file.type === 'application/pdf') {
+        categorizedAttachments.documents.push(file);
+      } else if (file.type.startsWith('video/')) {
+        categorizedAttachments.videos.push(file);
+      } else if (file.type.startsWith('audio/')) {
+        categorizedAttachments.audios.push(file);
+      }
+    }
+
+    const [imageFiles, documentFiles, videoFiles, audioFiles] = await Promise.all([
+      categorizedAttachments.images.length > 0
+        ? this.addImageURLs(message, categorizedAttachments.images)
+        : Promise.resolve([]),
+      categorizedAttachments.documents.length > 0
+        ? this.addDocuments(message, categorizedAttachments.documents)
+        : Promise.resolve([]),
+      categorizedAttachments.videos.length > 0
+        ? this.addVideos(message, categorizedAttachments.videos)
+        : Promise.resolve([]),
+      categorizedAttachments.audios.length > 0
+        ? this.addAudios(message, categorizedAttachments.audios)
+        : Promise.resolve([]),
+    ]);
+
+    const allFiles = [...imageFiles, ...documentFiles, ...videoFiles, ...audioFiles];
+    const seenFileIds = new Set();
+    const uniqueFiles = [];
+
+    for (const file of allFiles) {
+      if (file.file_id && !seenFileIds.has(file.file_id)) {
+        seenFileIds.add(file.file_id);
+        uniqueFiles.push(file);
+      } else if (!file.file_id) {
+        uniqueFiles.push(file);
+      }
+    }
+
+    return uniqueFiles;
+  }
+
   async buildMessages(
     messages,
     parentMessageId,
@@ -255,7 +423,7 @@ class AgentClient extends BaseClient {
         };
       }
 
-      const files = await this.addImageURLs(
+      const files = await this.processAttachments(
         orderedMessages[orderedMessages.length - 1],
         attachments,
       );
@@ -278,6 +446,47 @@ class AgentClient extends BaseClient {
         assistantName: this.options?.modelLabel,
       });
 
+      const hasFiles =
+        (message.documents && message.documents.length > 0) ||
+        (message.videos && message.videos.length > 0) ||
+        (message.audios && message.audios.length > 0) ||
+        (message.image_urls && message.image_urls.length > 0);
+
+      if (
+        hasFiles &&
+        message.isCreatedByUser &&
+        isDocumentSupportedEndpoint(this.options.agent.provider)
+      ) {
+        const contentParts = [];
+
+        if (message.documents && message.documents.length > 0) {
+          contentParts.push(...message.documents);
+        }
+
+        if (message.videos && message.videos.length > 0) {
+          contentParts.push(...message.videos);
+        }
+
+        if (message.audios && message.audios.length > 0) {
+          contentParts.push(...message.audios);
+        }
+
+        if (message.image_urls && message.image_urls.length > 0) {
+          contentParts.push(...message.image_urls);
+        }
+
+        if (typeof formattedMessage.content === 'string') {
+          contentParts.push({ type: 'text', text: formattedMessage.content });
+        } else {
+          const textPart = formattedMessage.content.find((part) => part.type === 'text');
+          if (textPart) {
+            contentParts.push(textPart);
+          }
+        }
+
+        formattedMessage.content = contentParts;
+      }
+
       if (message.ocr && i !== orderedMessages.length - 1) {
         if (typeof formattedMessage.content === 'string') {
           formattedMessage.content = message.ocr + '\n' + formattedMessage.content;
@@ -793,6 +1002,7 @@ class AgentClient extends BaseClient {
       };
 
       const toolSet = new Set((this.options.agent.tools ?? []).map((tool) => tool && tool.name));
+
       let { messages: initialMessages, indexTokenCountMap } = formatAgentMessages(
         payload,
         this.indexTokenCountMap,

api/server/services/Files/Local/crud.js

@@ -4,6 +4,7 @@ const axios = require('axios');
 const { logger } = require('@librechat/data-schemas');
 const { EModelEndpoint } = require('librechat-data-provider');
 const { generateShortLivedToken } = require('@librechat/api');
+const { resizeImageBuffer } = require('~/server/services/Files/images/resize');
 const { getBufferMetadata } = require('~/server/utils');
 const paths = require('~/config/paths');
 
@@ -286,7 +287,18 @@ async function uploadLocalFile({ req, file, file_id }) {
   await fs.promises.writeFile(newPath, inputBuffer);
   const filepath = path.posix.join('/', 'uploads', req.user.id, path.basename(newPath));
 
-  return { filepath, bytes };
+  let height, width;
+  if (file.mimetype && file.mimetype.startsWith('image/')) {
+    try {
+      const { width: imgWidth, height: imgHeight } = await resizeImageBuffer(inputBuffer, 'high');
+      height = imgHeight;
+      width = imgWidth;
+    } catch (error) {
+      logger.warn('[uploadLocalFile] Could not get image dimensions:', error.message);
+    }
+  }
+
+  return { filepath, bytes, height, width };
 }
 
 /**

api/server/services/Files/index.js

@@ -2,11 +2,13 @@ const { processCodeFile } = require('./Code/process');
 const { processFileUpload } = require('./process');
 const { uploadImageBuffer } = require('./images');
 const { hasAccessToFilesViaAgent, filterFilesByAgentAccess } = require('./permissions');
+const { getStrategyFunctions } = require('./strategies');
 
 module.exports = {
   processCodeFile,
   processFileUpload,
   uploadImageBuffer,
+  getStrategyFunctions,
   hasAccessToFilesViaAgent,
   filterFilesByAgentAccess,
 };

api/server/services/Files/process.js

@@ -419,11 +419,11 @@ const processFileUpload = async ({ req, res, metadata }) => {
   const isAssistantUpload = isAssistantsEndpoint(metadata.endpoint);
   const assistantSource =
     metadata.endpoint === EModelEndpoint.azureAssistants ? FileSources.azure : FileSources.openai;
+
   // Use the configured file strategy for regular file uploads (not vectordb)
   const source = isAssistantUpload ? assistantSource : appConfig.fileStrategy;
   const { handleFileUpload } = getStrategyFunctions(source);
   const { file_id, temp_file_id = null } = metadata;
-
   /** @type {OpenAI | undefined} */
   let openai;
   if (checkOpenAIStorage(source)) {

client/src/common/agents-types.ts

@@ -21,6 +21,7 @@ export type TAgentCapabilities = {
   [AgentCapabilities.execute_code]: boolean;
   [AgentCapabilities.end_after_tools]?: boolean;
   [AgentCapabilities.hide_sequential_outputs]?: boolean;
+  [AgentCapabilities.direct_attach]?: boolean;
 };
 
 export type AgentForm = {

client/src/components/Chat/Input/Files/AttachFileChat.tsx

@@ -36,6 +36,7 @@ function AttachFileChat({ disableInputs }: { disableInputs: boolean }) {
         disabled={disableInputs}
         conversationId={conversationId}
         endpointFileConfig={endpointFileConfig}
+        endpoint={endpoint}
       />
     );
   }