Skip to content

Commit 7cd1c71

Browse files
shaunwarmanshaunwarman
committed
fix: move to counter approach and some query optimizations
1 parent 3bfbc2e commit 7cd1c71

File tree

5 files changed

+201
-61
lines changed

5 files changed

+201
-61
lines changed

‎app/controllers/web/admin/users.js‎

Lines changed: 9 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ const parser = require('mongodb-query-parser');
1212
const { boolean } = require('boolean');
1313
const _ = require('#helpers/lodash');
1414

15-
const { Users, Emails } = require('#models');
15+
const { Users } = require('#models');
1616
const config = require('#config');
1717

1818
const REGEX_BYTES = new RE2(/^((-|\+)?(\d+(?:\.\d+)?)) *(kb|mb|gb|tb|pb)$/i);
@@ -52,77 +52,25 @@ async function list(ctx) {
5252
}
5353
}
5454

55-
const now = new Date();
56-
const oneHourAgo = new Date(now - 60 * 60 * 1000);
57-
const twentyFourHoursAgo = new Date(now - 24 * 60 * 60 * 1000);
58-
const seventyTwoHoursAgo = new Date(now - 72 * 60 * 60 * 1000);
59-
60-
const [users, itemCount, emailCounts] = await Promise.all([
55+
const [users, itemCount] = await Promise.all([
6156
// eslint-disable-next-line unicorn/no-array-callback-reference
6257
Users.find(query)
6358
.limit(ctx.query.limit)
6459
.skip(ctx.paginate.skip)
6560
.lean()
6661
.sort(ctx.query.sort || '-created_at')
6762
.exec(),
68-
Users.countDocuments(query),
69-
// Get SMTP outbound email counts per user with time-based breakdowns
70-
Emails.aggregate([
71-
{
72-
$match: {
73-
// Only count delivered/sent emails (not failed/bounced)
74-
status: { $in: ['delivered', 'deferred', 'sent'] }
75-
}
76-
},
77-
{
78-
$group: {
79-
_id: '$user',
80-
totalEmails: { $sum: 1 },
81-
lastEmailAt: { $max: '$created_at' },
82-
// Count emails within time periods
83-
emailsLast1Hour: {
84-
$sum: {
85-
$cond: [{ $gte: ['$created_at', oneHourAgo] }, 1, 0]
86-
}
87-
},
88-
emailsLast24Hours: {
89-
$sum: {
90-
$cond: [{ $gte: ['$created_at', twentyFourHoursAgo] }, 1, 0]
91-
}
92-
},
93-
emailsLast72Hours: {
94-
$sum: {
95-
$cond: [{ $gte: ['$created_at', seventyTwoHoursAgo] }, 1, 0]
96-
}
97-
}
98-
}
99-
}
100-
])
63+
Users.countDocuments(query)
10164
]);
10265

103-
// Create a map for quick lookup of email counts
104-
const emailCountMap = new Map();
105-
for (const count of emailCounts) {
106-
emailCountMap.set(count._id.toString(), {
107-
totalEmails: count.totalEmails,
108-
lastEmailAt: count.lastEmailAt,
109-
emailsLast1Hour: count.emailsLast1Hour,
110-
emailsLast24Hours: count.emailsLast24Hours,
111-
emailsLast72Hours: count.emailsLast72Hours
112-
});
113-
}
114-
115-
// Add email counts to each user
66+
// Use the optimized user model counters instead of aggregation
11667
const usersWithEmailCounts = users.map((user) => ({
11768
...user,
118-
totalEmails: emailCountMap.get(user._id.toString())?.totalEmails || 0,
119-
lastEmailAt: emailCountMap.get(user._id.toString())?.lastEmailAt || null,
120-
emailsLast1Hour:
121-
emailCountMap.get(user._id.toString())?.emailsLast1Hour || 0,
122-
emailsLast24Hours:
123-
emailCountMap.get(user._id.toString())?.emailsLast24Hours || 0,
124-
emailsLast72Hours:
125-
emailCountMap.get(user._id.toString())?.emailsLast72Hours || 0
69+
totalEmails: user.smtp_emails_sent_total || 0,
70+
lastEmailAt: user.smtp_last_email_sent_at || null,
71+
emailsLast1Hour: user.smtp_emails_sent_1h || 0,
72+
emailsLast24Hours: user.smtp_emails_sent_24h || 0,
73+
emailsLast72Hours: user.smtp_emails_sent_72h || 0
12674
}));
12775

12876
const pageCount = Math.ceil(itemCount / ctx.query.limit);

‎app/models/emails.js‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1156,6 +1156,30 @@ Emails.post('save', async function (email, next) {
11561156
}
11571157
});
11581158

1159+
// Update user SMTP counters when email is successfully sent
1160+
Emails.post('save', async function (email) {
1161+
// Only update counters for new emails that are sent/delivered
1162+
if (!email._isNew || !['sent', 'delivered'].includes(email.status)) return;
1163+
1164+
try {
1165+
// Increment the user's SMTP counters
1166+
await Users.findByIdAndUpdate(email.user, {
1167+
$inc: {
1168+
smtp_emails_sent_1h: 1,
1169+
smtp_emails_sent_24h: 1,
1170+
smtp_emails_sent_72h: 1,
1171+
smtp_emails_sent_total: 1
1172+
},
1173+
$set: {
1174+
smtp_last_email_sent_at: new Date()
1175+
}
1176+
});
1177+
} catch (err) {
1178+
// Log error but don't fail the email save
1179+
logger.error(err, { email_id: email._id, user_id: email.user });
1180+
}
1181+
});
1182+
11591183
Emails.statics.getMessage = async function (obj, returnString = false) {
11601184
if (Buffer.isBuffer(obj)) {
11611185
if (returnString) return obj.toString();

‎app/models/users.js‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -206,6 +206,40 @@ object[config.userFields.smtpLimit] = {
206206
max: 100000
207207
};
208208

209+
// SMTP count tracking for admin dashboard
210+
object.smtp_emails_sent_1h = {
211+
type: Number,
212+
default: 0,
213+
min: 0,
214+
index: true
215+
};
216+
217+
object.smtp_emails_sent_24h = {
218+
type: Number,
219+
default: 0,
220+
min: 0,
221+
index: true
222+
};
223+
224+
object.smtp_emails_sent_72h = {
225+
type: Number,
226+
default: 0,
227+
min: 0,
228+
index: true
229+
};
230+
231+
object.smtp_emails_sent_total = {
232+
type: Number,
233+
default: 0,
234+
min: 0,
235+
index: true
236+
};
237+
238+
object.smtp_last_email_sent_at = {
239+
type: Date,
240+
index: true
241+
};
242+
209243
// Custom receipt email
210244
object[config.userFields.receiptEmail] = {
211245
type: String,

‎jobs/index.js‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -241,6 +241,12 @@ let jobs = [
241241
interval: '1d',
242242
timeout: 0
243243
},
244+
// update SMTP counters every hour
245+
{
246+
name: 'update-smtp-counters',
247+
interval: '1h',
248+
timeout: 0
249+
},
244250
// session management
245251
{
246252
name: 'session-management',

‎jobs/update-smtp-counters.js‎

Lines changed: 128 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
/**
2+
* Copyright (c) Forward Email LLC
3+
* SPDX-License-Identifier: BUSL-1.1
4+
*/
5+
6+
// eslint-disable-next-line import/no-unassigned-import
7+
require('#config/env');
8+
9+
const process = require('node:process');
10+
const { parentPort } = require('node:worker_threads');
11+
12+
// eslint-disable-next-line import/no-unassigned-import
13+
require('#config/mongoose');
14+
15+
const Graceful = require('@ladjs/graceful');
16+
const mongoose = require('mongoose');
17+
18+
const Emails = require('#models/emails');
19+
const Users = require('#models/users');
20+
const logger = require('#helpers/logger');
21+
const monitorServer = require('#helpers/monitor-server');
22+
const setupMongoose = require('#helpers/setup-mongoose');
23+
24+
monitorServer();
25+
26+
const graceful = new Graceful({
27+
mongooses: [mongoose],
28+
logger
29+
});
30+
31+
graceful.listen();
32+
33+
(async () => {
34+
await setupMongoose(logger);
35+
36+
try {
37+
const now = new Date();
38+
const oneHourAgo = new Date(now - 60 * 60 * 1000);
39+
const twentyFourHoursAgo = new Date(now - 24 * 60 * 60 * 1000);
40+
const seventyTwoHoursAgo = new Date(now - 72 * 60 * 60 * 1000);
41+
42+
logger.info('Starting SMTP counter update job');
43+
44+
// Get SMTP counts per user with time-based breakdowns
45+
const emailCounts = await Emails.aggregate([
46+
{
47+
$match: {
48+
// Only count delivered/sent emails (not failed/bounced)
49+
status: { $in: ['delivered', 'deferred', 'sent'] }
50+
}
51+
},
52+
{
53+
$group: {
54+
_id: '$user',
55+
totalEmails: { $sum: 1 },
56+
lastEmailAt: { $max: '$created_at' },
57+
// Count emails within time periods
58+
emailsLast1Hour: {
59+
$sum: {
60+
$cond: [{ $gte: ['$created_at', oneHourAgo] }, 1, 0]
61+
}
62+
},
63+
emailsLast24Hours: {
64+
$sum: {
65+
$cond: [{ $gte: ['$created_at', twentyFourHoursAgo] }, 1, 0]
66+
}
67+
},
68+
emailsLast72Hours: {
69+
$sum: {
70+
$cond: [{ $gte: ['$created_at', seventyTwoHoursAgo] }, 1, 0]
71+
}
72+
}
73+
}
74+
}
75+
]);
76+
77+
logger.info(`Found email counts for ${emailCounts.length} users`);
78+
79+
// Create bulk operations to update all users
80+
const bulkOperations = [];
81+
82+
// First, reset all counters to 0 for all users
83+
bulkOperations.push({
84+
updateMany: {
85+
filter: {},
86+
update: {
87+
$set: {
88+
smtp_emails_sent_1h: 0,
89+
smtp_emails_sent_24h: 0,
90+
smtp_emails_sent_72h: 0,
91+
smtp_emails_sent_total: 0,
92+
smtp_last_email_sent_at: null
93+
}
94+
}
95+
}
96+
});
97+
98+
// Then update users who have email counts
99+
for (const count of emailCounts) {
100+
bulkOperations.push({
101+
updateOne: {
102+
filter: { _id: count._id },
103+
update: {
104+
$set: {
105+
smtp_emails_sent_1h: count.emailsLast1Hour,
106+
smtp_emails_sent_24h: count.emailsLast24Hours,
107+
smtp_emails_sent_72h: count.emailsLast72Hours,
108+
smtp_emails_sent_total: count.totalEmails,
109+
smtp_last_email_sent_at: count.lastEmailAt
110+
}
111+
}
112+
}
113+
});
114+
}
115+
116+
if (bulkOperations.length > 0) {
117+
await Users.bulkWrite(bulkOperations, { ordered: false });
118+
logger.info(`Updated SMTP counters for all users`);
119+
}
120+
121+
logger.info('SMTP counter update job completed successfully');
122+
} catch (err) {
123+
await logger.error(err);
124+
}
125+
126+
if (parentPort) parentPort.postMessage('done');
127+
else process.exit(0);
128+
})();

0 commit comments

Comments
 (0)