Skip to content

Fix query performance by reordering property names #2588

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Oct 22, 2025
Merged

Fix query performance by reordering property names #2588

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1a06980
Fix query performance by reordering property names
jrahman Aug 26, 2025
f19556e
Update changelog
jrahman Aug 26, 2025
dfb6732
Update changelog, move to Upcoming Release section
jrahman Aug 28, 2025
c11ae6d
Merge branch 'main' into fix_query_perf
EmmaZhu Oct 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ChangeLog.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,10 @@

## Upcoming Release

General:

- Performance improvements for internal metadata access using in-memory metadata store

## 2025.07 Version 3.35.0

General:
Expand Down
28 changes: 14 additions & 14 deletions src/blob/persistence/LokiBlobMetadataStore.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -383,8 +383,8 @@ export default class LokiBlobMetadataStore
): Promise<ContainerModel> {
const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
const doc = coll.findOne({
accountName: container.accountName,
name: container.name
name: container.name,
accountName: container.accountName
});

if (doc) {
Expand Down Expand Up @@ -817,7 +817,7 @@ export default class LokiBlobMetadataStore
container: string
): Promise<void> {
const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
const doc = coll.findOne({ accountName: account, name: container });
const doc = coll.findOne({ name: container, accountName: account });
if (!doc) {
const requestId = context ? context.contextId : undefined;
throw StorageErrorFactory.getContainerNotFound(requestId);
Expand Down Expand Up @@ -1032,9 +1032,9 @@ export default class LokiBlobMetadataStore
);
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const blobDoc = coll.findOne({
name: blob.name,
accountName: blob.accountName,
containerName: blob.containerName,
name: blob.name,
snapshot: blob.snapshot
});

Expand Down Expand Up @@ -1219,9 +1219,9 @@ export default class LokiBlobMetadataStore
): Promise<BlobModel | undefined> {
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const blobDoc = coll.findOne({
name: blob,
accountName: account,
containerName: container,
name: blob,
snapshot
});

Expand Down Expand Up @@ -1799,9 +1799,9 @@ export default class LokiBlobMetadataStore

const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const doc = coll.findOne({
name: blob,
accountName: account,
containerName: container,
name: blob,
snapshot
});

Expand Down Expand Up @@ -1833,9 +1833,9 @@ export default class LokiBlobMetadataStore
> {
const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const doc = coll.findOne({
name: blob,
accountName: account,
containerName: container,
name: blob,
snapshot
});
if (!doc) {
Expand Down Expand Up @@ -2323,9 +2323,9 @@ export default class LokiBlobMetadataStore

const blobColl = this.db.getCollection(this.BLOBS_COLLECTION);
const blobDoc = blobColl.findOne({
name: block.blobName,
accountName: block.accountName,
containerName: block.containerName,
name: block.blobName
containerName: block.containerName
});

let blobExist = false;
Expand Down Expand Up @@ -2364,9 +2364,9 @@ export default class LokiBlobMetadataStore
// If the new block ID does not have same length with before uncommitted block ID, return failure.
if (blobExist) {
const existBlockDoc = coll.findOne({
blobName: block.blobName,
accountName: block.accountName,
containerName: block.containerName,
blobName: block.blobName
});
if (existBlockDoc) {
if (
Expand All @@ -2379,10 +2379,10 @@ export default class LokiBlobMetadataStore
}

const blockDoc = coll.findOne({
name: block.name,
accountName: block.accountName,
containerName: block.containerName,
blobName: block.blobName,
name: block.name,
isCommitted: block.isCommitted
});

Expand Down Expand Up @@ -3207,7 +3207,7 @@ export default class LokiBlobMetadataStore
forceExist?: boolean
): Promise<ContainerModel | undefined> {
const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
const doc = coll.findOne({ accountName: account, name: container });
const doc = coll.findOne({ name: container, accountName: account });

if (forceExist === undefined || forceExist === true) {
if (!doc) {
Expand Down Expand Up @@ -3271,7 +3271,7 @@ export default class LokiBlobMetadataStore
forceExist?: boolean
): Promise<ContainerModel | undefined> {
const coll = this.db.getCollection(this.CONTAINERS_COLLECTION);
const doc = coll.findOne({ accountName: account, name: container });
const doc = coll.findOne({ name: container, accountName: account });

if (!doc) {
if (forceExist) {
Expand Down Expand Up @@ -3347,9 +3347,9 @@ export default class LokiBlobMetadataStore

const coll = this.db.getCollection(this.BLOBS_COLLECTION);
const doc = coll.findOne({
name: blob,
accountName: account,
containerName: container,
name: blob,
snapshot
});

Expand Down
32 changes: 16 additions & 16 deletions src/queue/persistence/LokiQueueMetadataStore.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,10 +236,10 @@ export default class LokiQueueMetadataStore implements IQueueMetadataStore {
prefix === ""
? { $loki: { $gt: marker }, accountName: account }
: {
name: { $regex: `^${this.escapeRegex(prefix)}` },
$loki: { $gt: marker },
accountName: account
};
name: { $regex: `^${this.escapeRegex(prefix)}` },
$loki: { $gt: marker },
accountName: account
};

// Get one more item to help check if the query reach the tail of the collection.
const docs = coll
Expand Down Expand Up @@ -283,7 +283,7 @@ export default class LokiQueueMetadataStore implements IQueueMetadataStore {
context?: Context
): Promise<QueueModel> {
const coll = this.db.getCollection(this.QUEUES_COLLECTION);
const doc = coll.findOne({ accountName: account, name: queue });
const doc = coll.findOne({ name: queue, accountName: account });
if (!doc) {
const requestId = context ? context.contextID : undefined;
throw StorageErrorFactory.getQueueNotFound(requestId);
Expand All @@ -307,8 +307,8 @@ export default class LokiQueueMetadataStore implements IQueueMetadataStore {
): Promise<QUEUE_STATUSCODE> {
const coll = this.db.getCollection(this.QUEUES_COLLECTION);
const doc = coll.findOne({
accountName: queue.accountName,
name: queue.name
name: queue.name,
accountName: queue.accountName
});

// Check whether a conflict exists if there exist a queue with the given name.
Expand Down Expand Up @@ -386,7 +386,7 @@ export default class LokiQueueMetadataStore implements IQueueMetadataStore {
context?: Context
): Promise<void> {
const coll = this.db.getCollection(this.QUEUES_COLLECTION);
const doc = coll.findOne({ accountName: account, name: queue });
const doc = coll.findOne({ name: queue, accountName: account });

if (!doc) {
const requestId = context ? context.contextID : undefined;
Expand Down Expand Up @@ -418,7 +418,7 @@ export default class LokiQueueMetadataStore implements IQueueMetadataStore {
context?: Context
): Promise<void> {
const coll = this.db.getCollection(this.QUEUES_COLLECTION);
const doc = coll.findOne({ accountName: account, name: queue });
const doc = coll.findOne({ name: queue, accountName: account });

if (!doc) {
const requestId = context ? context.contextID : undefined;
Expand Down Expand Up @@ -446,7 +446,7 @@ export default class LokiQueueMetadataStore implements IQueueMetadataStore {
context?: Context
): Promise<void> {
const coll = this.db.getCollection(this.QUEUES_COLLECTION);
const doc = coll.findOne({ accountName: account, name: queue });
const doc = coll.findOne({ name: queue, accountName: account });

if (!doc) {
const requestId = context ? context.contextID : undefined;
Expand All @@ -472,7 +472,7 @@ export default class LokiQueueMetadataStore implements IQueueMetadataStore {
context?: Context
): Promise<number> {
const queueColl = this.db.getCollection(this.QUEUES_COLLECTION);
const doc = queueColl.findOne({ accountName: account, name: queue });
const doc = queueColl.findOne({ name: queue, accountName: account });
if (!doc) {
const requestId = context ? context.contextID : undefined;
throw StorageErrorFactory.getQueueNotFound(requestId);
Expand Down Expand Up @@ -649,9 +649,9 @@ export default class LokiQueueMetadataStore implements IQueueMetadataStore {

const coll = this.db.getCollection(this.MESSAGES_COLLECTION);
const doc = coll.findOne({
messageId,
accountName: account,
queueName: queue,
messageId
queueName: queue
});

if (!doc) {
Expand Down Expand Up @@ -687,9 +687,9 @@ export default class LokiQueueMetadataStore implements IQueueMetadataStore {

const coll = this.db.getCollection(this.MESSAGES_COLLECTION);
const doc = coll.findOne({
messageId: message.messageId,
accountName: message.accountName,
queueName: message.queueName,
messageId: message.messageId
queueName: message.queueName
});

if (!doc) {
Expand Down Expand Up @@ -783,7 +783,7 @@ export default class LokiQueueMetadataStore implements IQueueMetadataStore {
context?: Context
): void {
const queueColl = this.db.getCollection(this.QUEUES_COLLECTION);
const queueDoc = queueColl.findOne({ accountName: account, name: queue });
const queueDoc = queueColl.findOne({ name: queue, accountName: account });
if (!queueDoc) {
const requestId = context ? context.contextID : undefined;
throw StorageErrorFactory.getQueueNotFound(requestId);
Expand Down
24 changes: 12 additions & 12 deletions src/table/persistence/LokiTableMetadataStore.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,8 +265,8 @@ export default class LokiTableMetadataStore implements ITableMetadataStore {
);

const doc = tableEntityCollection.findOne({
PartitionKey: entity.PartitionKey,
RowKey: entity.RowKey
RowKey: entity.RowKey,
PartitionKey: entity.PartitionKey
});
if (doc) {
throw StorageErrorFactory.getEntityAlreadyExist(context);
Expand Down Expand Up @@ -392,8 +392,8 @@ export default class LokiTableMetadataStore implements ITableMetadataStore {

if (partitionKey !== undefined && rowKey !== undefined) {
const doc = tableEntityCollection.findOne({
PartitionKey: partitionKey,
RowKey: rowKey
RowKey: rowKey,
PartitionKey: partitionKey
}) as Entity;

this.checkForMissingEntity(doc, context);
Expand Down Expand Up @@ -503,8 +503,8 @@ export default class LokiTableMetadataStore implements ITableMetadataStore {
): Promise<Entity | undefined> {
const entityCollection = this.getEntityCollection(account, table, context);
const requestedDoc = entityCollection.findOne({
PartitionKey: partitionKey,
RowKey: rowKey
RowKey: rowKey,
PartitionKey: partitionKey
}) as Entity;

return requestedDoc;
Expand Down Expand Up @@ -919,8 +919,8 @@ export default class LokiTableMetadataStore implements ITableMetadataStore {
);

const doc = tableEntityCollection.findOne({
PartitionKey: entity.PartitionKey,
RowKey: entity.RowKey
RowKey: entity.RowKey,
PartitionKey: entity.PartitionKey
}) as Entity;

if (!doc) {
Expand Down Expand Up @@ -968,8 +968,8 @@ export default class LokiTableMetadataStore implements ITableMetadataStore {
);

const doc = tableEntityCollection.findOne({
PartitionKey: entity.PartitionKey,
RowKey: entity.RowKey
RowKey: entity.RowKey,
PartitionKey: entity.PartitionKey
}) as Entity;

if (!doc) {
Expand Down Expand Up @@ -1062,8 +1062,8 @@ export default class LokiTableMetadataStore implements ITableMetadataStore {
};
// lokijs applies this insert as an upsert
const doc = tableBatchCollection.findOne({
PartitionKey: entity.PartitionKey,
RowKey: entity.RowKey
RowKey: entity.RowKey,
PartitionKey: entity.PartitionKey
});
// we can't rely on upsert behavior if documents already exist
if (doc) {
Expand Down
Loading