Skip to content

Commit 2df6eef

Browse files
rubentalstrajustinmdickey
rubentalstra
authored and
justinmdickey
committed
🚀 feat: Integrate Azure Blob Storage for file handling and image uploads (danny-avila#6153)
* 🚀 feat: Integrate Azure Blob Storage for file handling and image uploads * 🐼 refactor: Correct module import case for Azure in strategies.js * 🚀 feat: Add Azure support in SourceIcon component * 🚀 feat: Enhance Azure Blob Service initialization with Managed Identity support * 🐼 refactor: Remove unused Azure dependencies from package.json and package-lock.json * 🐼 refactor: Remove unused Azure dependencies from package.json and package-lock.json * 🐼 refactor: Remove unused Azure dependencies from package.json and package-lock.json * 🚀 feat: Add Azure SDK dependencies for identity and storage blob * 🔧 fix: Reorganize imports in strategies.js for better clarity * 🔧 fix: Correct comment formatting in strategies.js for consistency * 🔧 fix: Improve comment formatting in strategies.js for consistency
1 parent 7353737 commit 2df6eef

File tree

9 files changed

+800
-41
lines changed

9 files changed

+800
-41
lines changed

‎.env.example

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -485,6 +485,14 @@ AWS_SECRET_ACCESS_KEY=
485485
AWS_REGION=
486486
AWS_BUCKET_NAME=
487487

488+
#========================#
489+
# Azure Blob Storage #
490+
#========================#
491+
492+
AZURE_STORAGE_CONNECTION_STRING=
493+
AZURE_STORAGE_PUBLIC_ACCESS=false
494+
AZURE_CONTAINER_NAME=files
495+
488496
#========================#
489497
# Shared Links #
490498
#========================#

‎api/package.json

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,8 @@
3737
"@anthropic-ai/sdk": "^0.37.0",
3838
"@aws-sdk/client-s3": "^3.758.0",
3939
"@aws-sdk/s3-request-presigner": "^3.758.0",
40+
"@azure/identity": "^4.7.0",
41+
"@azure/storage-blob": "^12.26.0",
4042
"@azure/search-documents": "^12.0.0",
4143
"@google/generative-ai": "^0.23.0",
4244
"@googleapis/youtube": "^20.0.0",

‎api/server/services/AppService.js

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ const {
77
} = require('librechat-data-provider');
88
const { checkVariables, checkHealth, checkConfig, checkAzureVariables } = require('./start/checks');
99
const { azureAssistantsDefaults, assistantsConfigSetup } = require('./start/assistants');
10+
const { initializeAzureBlobService } = require('./Files/Azure/initialize');
1011
const { initializeFirebase } = require('./Files/Firebase/initialize');
1112
const { initializeS3 } = require('./Files/S3/initialize');
1213
const loadCustomConfig = require('./Config/loadCustomConfig');
@@ -45,6 +46,8 @@ const AppService = async (app) => {
4546

4647
if (fileStrategy === FileSources.firebase) {
4748
initializeFirebase();
49+
} else if (fileStrategy === FileSources.azure) {
50+
initializeAzureBlobService();
4851
} else if (fileStrategy === FileSources.s3) {
4952
initializeS3();
5053
}

‎api/server/services/Files/Azure/crud.js

Lines changed: 196 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,196 @@
1+
const fs = require('fs');
2+
const path = require('path');
3+
const axios = require('axios');
4+
const fetch = require('node-fetch');
5+
const { logger } = require('~/config');
6+
const { getAzureContainerClient } = require('./initialize');
7+
8+
const defaultBasePath = 'images';
9+
10+
/**
11+
* Uploads a buffer to Azure Blob Storage.
12+
*
13+
* Files will be stored at the path: {basePath}/{userId}/{fileName} within the container.
14+
*
15+
* @param {Object} params
16+
* @param {string} params.userId - The user's id.
17+
* @param {Buffer} params.buffer - The buffer to upload.
18+
* @param {string} params.fileName - The name of the file.
19+
* @param {string} [params.basePath='images'] - The base folder within the container.
20+
* @param {string} [params.containerName] - The Azure Blob container name.
21+
* @returns {Promise<string>} The URL of the uploaded blob.
22+
*/
23+
async function saveBufferToAzure({
24+
userId,
25+
buffer,
26+
fileName,
27+
basePath = defaultBasePath,
28+
containerName,
29+
}) {
30+
try {
31+
const containerClient = getAzureContainerClient(containerName);
32+
// Create the container if it doesn't exist. This is done per operation.
33+
await containerClient.createIfNotExists({
34+
access: process.env.AZURE_STORAGE_PUBLIC_ACCESS ? 'blob' : undefined,
35+
});
36+
const blobPath = `${basePath}/${userId}/${fileName}`;
37+
const blockBlobClient = containerClient.getBlockBlobClient(blobPath);
38+
await blockBlobClient.uploadData(buffer);
39+
return blockBlobClient.url;
40+
} catch (error) {
41+
logger.error('[saveBufferToAzure] Error uploading buffer:', error);
42+
throw error;
43+
}
44+
}
45+
46+
/**
47+
* Saves a file from a URL to Azure Blob Storage.
48+
*
49+
* @param {Object} params
50+
* @param {string} params.userId - The user's id.
51+
* @param {string} params.URL - The URL of the file.
52+
* @param {string} params.fileName - The name of the file.
53+
* @param {string} [params.basePath='images'] - The base folder within the container.
54+
* @param {string} [params.containerName] - The Azure Blob container name.
55+
* @returns {Promise<string>} The URL of the uploaded blob.
56+
*/
57+
async function saveURLToAzure({
58+
userId,
59+
URL,
60+
fileName,
61+
basePath = defaultBasePath,
62+
containerName,
63+
}) {
64+
try {
65+
const response = await fetch(URL);
66+
const buffer = await response.buffer();
67+
return await saveBufferToAzure({ userId, buffer, fileName, basePath, containerName });
68+
} catch (error) {
69+
logger.error('[saveURLToAzure] Error uploading file from URL:', error);
70+
throw error;
71+
}
72+
}
73+
74+
/**
75+
* Retrieves a blob URL from Azure Blob Storage.
76+
*
77+
* @param {Object} params
78+
* @param {string} params.fileName - The file name.
79+
* @param {string} [params.basePath='images'] - The base folder used during upload.
80+
* @param {string} [params.userId] - If files are stored in a user-specific directory.
81+
* @param {string} [params.containerName] - The Azure Blob container name.
82+
* @returns {Promise<string>} The blob's URL.
83+
*/
84+
async function getAzureURL({ fileName, basePath = defaultBasePath, userId, containerName }) {
85+
try {
86+
const containerClient = getAzureContainerClient(containerName);
87+
const blobPath = userId ? `${basePath}/${userId}/${fileName}` : `${basePath}/${fileName}`;
88+
const blockBlobClient = containerClient.getBlockBlobClient(blobPath);
89+
return blockBlobClient.url;
90+
} catch (error) {
91+
logger.error('[getAzureURL] Error retrieving blob URL:', error);
92+
throw error;
93+
}
94+
}
95+
96+
/**
97+
* Deletes a blob from Azure Blob Storage.
98+
*
99+
* @param {Object} params
100+
* @param {string} params.fileName - The name of the file.
101+
* @param {string} [params.basePath='images'] - The base folder where the file is stored.
102+
* @param {string} params.userId - The user's id.
103+
* @param {string} [params.containerName] - The Azure Blob container name.
104+
*/
105+
async function deleteFileFromAzure({
106+
fileName,
107+
basePath = defaultBasePath,
108+
userId,
109+
containerName,
110+
}) {
111+
try {
112+
const containerClient = getAzureContainerClient(containerName);
113+
const blobPath = `${basePath}/${userId}/${fileName}`;
114+
const blockBlobClient = containerClient.getBlockBlobClient(blobPath);
115+
await blockBlobClient.delete();
116+
logger.debug('[deleteFileFromAzure] Blob deleted successfully from Azure Blob Storage');
117+
} catch (error) {
118+
logger.error('[deleteFileFromAzure] Error deleting blob:', error.message);
119+
if (error.statusCode === 404) {
120+
return;
121+
}
122+
throw error;
123+
}
124+
}
125+
126+
/**
127+
* Uploads a file from the local file system to Azure Blob Storage.
128+
*
129+
* This function reads the file from disk and then uploads it to Azure Blob Storage
130+
* at the path: {basePath}/{userId}/{fileName}.
131+
*
132+
* @param {Object} params
133+
* @param {object} params.req - The Express request object.
134+
* @param {Express.Multer.File} params.file - The file object.
135+
* @param {string} params.file_id - The file id.
136+
* @param {string} [params.basePath='images'] - The base folder within the container.
137+
* @param {string} [params.containerName] - The Azure Blob container name.
138+
* @returns {Promise<{ filepath: string, bytes: number }>} An object containing the blob URL and its byte size.
139+
*/
140+
async function uploadFileToAzure({
141+
req,
142+
file,
143+
file_id,
144+
basePath = defaultBasePath,
145+
containerName,
146+
}) {
147+
try {
148+
const inputFilePath = file.path;
149+
const inputBuffer = await fs.promises.readFile(inputFilePath);
150+
const bytes = Buffer.byteLength(inputBuffer);
151+
const userId = req.user.id;
152+
const fileName = `${file_id}__${path.basename(inputFilePath)}`;
153+
const fileURL = await saveBufferToAzure({
154+
userId,
155+
buffer: inputBuffer,
156+
fileName,
157+
basePath,
158+
containerName,
159+
});
160+
await fs.promises.unlink(inputFilePath);
161+
return { filepath: fileURL, bytes };
162+
} catch (error) {
163+
logger.error('[uploadFileToAzure] Error uploading file:', error);
164+
throw error;
165+
}
166+
}
167+
168+
/**
169+
* Retrieves a readable stream for a blob from Azure Blob Storage.
170+
*
171+
* @param {object} _req - The Express request object.
172+
* @param {string} fileURL - The URL of the blob.
173+
* @returns {Promise<ReadableStream>} A readable stream of the blob.
174+
*/
175+
async function getAzureFileStream(_req, fileURL) {
176+
try {
177+
const response = await axios({
178+
method: 'get',
179+
url: fileURL,
180+
responseType: 'stream',
181+
});
182+
return response.data;
183+
} catch (error) {
184+
logger.error('[getAzureFileStream] Error getting blob stream:', error);
185+
throw error;
186+
}
187+
}
188+
189+
module.exports = {
190+
saveBufferToAzure,
191+
saveURLToAzure,
192+
getAzureURL,
193+
deleteFileFromAzure,
194+
uploadFileToAzure,
195+
getAzureFileStream,
196+
};

‎api/server/services/Files/Azure/images.js

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
const fs = require('fs');
2+
const path = require('path');
3+
const sharp = require('sharp');
4+
const { resizeImageBuffer } = require('../images/resize');
5+
const { updateUser } = require('~/models/userMethods');
6+
const { updateFile } = require('~/models/File');
7+
const { logger } = require('~/config');
8+
const { saveBufferToAzure } = require('./crud');
9+
10+
/**
11+
* Uploads an image file to Azure Blob Storage.
12+
* It resizes and converts the image similar to your Firebase implementation.
13+
*
14+
* @param {Object} params
15+
* @param {object} params.req - The Express request object.
16+
* @param {Express.Multer.File} params.file - The file object.
17+
* @param {string} params.file_id - The file id.
18+
* @param {EModelEndpoint} params.endpoint - The endpoint parameters.
19+
* @param {string} [params.resolution='high'] - The image resolution.
20+
* @param {string} [params.basePath='images'] - The base folder within the container.
21+
* @param {string} [params.containerName] - The Azure Blob container name.
22+
* @returns {Promise<{ filepath: string, bytes: number, width: number, height: number }>}
23+
*/
24+
async function uploadImageToAzure({
25+
req,
26+
file,
27+
file_id,
28+
endpoint,
29+
resolution = 'high',
30+
basePath = 'images',
31+
containerName,
32+
}) {
33+
try {
34+
const inputFilePath = file.path;
35+
const inputBuffer = await fs.promises.readFile(inputFilePath);
36+
const {
37+
buffer: resizedBuffer,
38+
width,
39+
height,
40+
} = await resizeImageBuffer(inputBuffer, resolution, endpoint);
41+
const extension = path.extname(inputFilePath);
42+
const userId = req.user.id;
43+
let webPBuffer;
44+
let fileName = `${file_id}__${path.basename(inputFilePath)}`;
45+
const targetExtension = `.${req.app.locals.imageOutputType}`;
46+
47+
if (extension.toLowerCase() === targetExtension) {
48+
webPBuffer = resizedBuffer;
49+
} else {
50+
webPBuffer = await sharp(resizedBuffer).toFormat(req.app.locals.imageOutputType).toBuffer();
51+
const extRegExp = new RegExp(path.extname(fileName) + '$');
52+
fileName = fileName.replace(extRegExp, targetExtension);
53+
if (!path.extname(fileName)) {
54+
fileName += targetExtension;
55+
}
56+
}
57+
const downloadURL = await saveBufferToAzure({
58+
userId,
59+
buffer: webPBuffer,
60+
fileName,
61+
basePath,
62+
containerName,
63+
});
64+
await fs.promises.unlink(inputFilePath);
65+
const bytes = Buffer.byteLength(webPBuffer);
66+
return { filepath: downloadURL, bytes, width, height };
67+
} catch (error) {
68+
logger.error('[uploadImageToAzure] Error uploading image:', error);
69+
throw error;
70+
}
71+
}
72+
73+
/**
74+
* Prepares the image URL and updates the file record.
75+
*
76+
* @param {object} req - The Express request object.
77+
* @param {MongoFile} file - The file object.
78+
* @returns {Promise<[MongoFile, string]>}
79+
*/
80+
async function prepareAzureImageURL(req, file) {
81+
const { filepath } = file;
82+
const promises = [];
83+
promises.push(updateFile({ file_id: file.file_id }));
84+
promises.push(filepath);
85+
return await Promise.all(promises);
86+
}
87+
88+
/**
89+
* Uploads and processes a user's avatar to Azure Blob Storage.
90+
*
91+
* @param {Object} params
92+
* @param {Buffer} params.buffer - The avatar image buffer.
93+
* @param {string} params.userId - The user's id.
94+
* @param {string} params.manual - Flag to indicate manual update.
95+
* @param {string} [params.basePath='images'] - The base folder within the container.
96+
* @param {string} [params.containerName] - The Azure Blob container name.
97+
* @returns {Promise<string>} The URL of the avatar.
98+
*/
99+
async function processAzureAvatar({ buffer, userId, manual, basePath = 'images', containerName }) {
100+
try {
101+
const downloadURL = await saveBufferToAzure({
102+
userId,
103+
buffer,
104+
fileName: 'avatar.png',
105+
basePath,
106+
containerName,
107+
});
108+
const isManual = manual === 'true';
109+
const url = `${downloadURL}?manual=${isManual}`;
110+
if (isManual) {
111+
await updateUser(userId, { avatar: url });
112+
}
113+
return url;
114+
} catch (error) {
115+
logger.error('[processAzureAvatar] Error uploading profile picture to Azure:', error);
116+
throw error;
117+
}
118+
}
119+
120+
module.exports = {
121+
uploadImageToAzure,
122+
prepareAzureImageURL,
123+
processAzureAvatar,
124+
};

‎api/server/services/Files/Azure/index.js

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
const crud = require('./crud');
2+
const images = require('./images');
3+
const initialize = require('./initialize');
4+
5+
module.exports = {
6+
...crud,
7+
...images,
8+
...initialize,
9+
};

0 commit comments

Comments
 (0)