Skip to content

Reduce memory usage during index construction. #8834

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
isoos merged 2 commits into from
Jun 27, 2025
Merged

Reduce memory usage during index construction. #8834

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7abfe99
Expose memory allocations in search benchmarks.
isoos Jun 25, 2025
1fbd8a1
Reduce memory usage during index construction.
isoos Jun 27, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 7 additions & 2 deletions app/bin/tools/isolate_search_benchmark.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:io';
import 'dart:math';

import 'package:_pub_shared/search/search_form.dart';
Expand All @@ -23,13 +24,15 @@ final queries = [
];

Future<void> main(List<String> args) async {
print('Loading...');
print('Started. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
final primaryRunner = await startSearchIsolate(snapshot: args.first);
final reducedRunner = await startSearchIsolate(
snapshot: args.first,
removeTextContent: true,
);
print('Loaded.');
print('Loaded. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');

for (var i = 0; i < 5; i++) {
await _benchmark(primaryRunner, primaryRunner);
Expand All @@ -39,6 +42,8 @@ Future<void> main(List<String> args) async {

await primaryRunner.close();
await reducedRunner.close();
print('Done. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
}

Future<void> _benchmark(IsolateRunner primary, IsolateRunner reduced) async {
Expand Down
8 changes: 8 additions & 0 deletions app/bin/tools/sdk_search_benchmark.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,17 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:io';

import 'package:pub_dev/search/sdk_mem_index.dart';

/// Loads a Dart SDK search snapshot and executes queries on it, benchmarking their total time to complete.
Future<void> main() async {
print('Started. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
final index = await createSdkMemIndex();
print('Loaded. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');

// NOTE: please add more queries to this list, especially if there is a performance bottleneck.
final queries = [
Expand All @@ -25,4 +31,6 @@ Future<void> main() async {
}
sw.stop();
print('${(sw.elapsedMilliseconds / count).toStringAsFixed(2)} ms/request');
print('Done. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
}
9 changes: 9 additions & 0 deletions app/bin/tools/search_benchmark.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,21 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:io';

import 'package:_pub_shared/search/search_form.dart';
import 'package:pub_dev/search/search_service.dart';
import 'package:pub_dev/search/updater.dart';

/// Loads a search snapshot and executes queries on it, benchmarking their total time to complete.
Future<void> main(List<String> args) async {
print('Started. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
// Assumes that the first argument is a search snapshot file.
final index = await loadInMemoryPackageIndexFromFile(args.first);
print('Loaded. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');

// NOTE: please add more queries to this list, especially if there is a performance bottleneck.
final queries = [
Expand All @@ -34,4 +41,6 @@ Future<void> main(List<String> args) async {
}
sw.stop();
print('${(sw.elapsedMilliseconds / count).toStringAsFixed(2)} ms/request');
print('Done. Current memory: ${ProcessInfo.currentRss ~/ 1024} KiB, '
'max memory: ${ProcessInfo.maxRss ~/ 1024} KiB');
}
42 changes: 12 additions & 30 deletions app/lib/search/mem_index.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import 'package:collection/collection.dart';
import 'package:logging/logging.dart';
import 'package:meta/meta.dart';
import 'package:pub_dev/search/heap.dart';
import 'package:pub_dev/service/topics/models.dart';
import 'package:pub_dev/third_party/bit_array/bit_array.dart';

import 'models.dart';
Expand All @@ -23,7 +22,6 @@ final _textSearchTimeout = Duration(milliseconds: 500);

class InMemoryPackageIndex {
final List<PackageDocument> _documents;
final _documentsByName = <String, PackageDocument>{};
final _nameToIndex = <String, int>{};
late final PackageNameIndex _packageNameIndex;
late final TokenIndex<String> _descrIndex;
Expand All @@ -47,23 +45,15 @@ class InMemoryPackageIndex {
late final List<IndexedPackageHit> _pointsOrderedHits;
late final List<IndexedPackageHit> _trendingOrderedHits;

// Contains all of the topics the index had seen so far.
// TODO: consider moving this into a separate index
// TODO: get the list of topics from the bucket
final _topics = <String>{
...canonicalTopics.aliasToCanonicalMap.values,
};

late final DateTime _lastUpdated;

InMemoryPackageIndex({
required Iterable<PackageDocument> documents,
}) : _documents = [...documents] {
final apiDocPageKeys = <IndexedApiDocPage>[];
final apiDocPageValues = <String>[];
final apiDocPageValues = <List<String>>[];
for (var i = 0; i < _documents.length; i++) {
final doc = _documents[i];
_documentsByName[doc.package] = doc;
_nameToIndex[doc.package] = i;

// transform tags into numberical IDs
Expand All @@ -78,16 +68,10 @@ class InMemoryPackageIndex {
for (final page in apiDocPages) {
if (page.symbols != null && page.symbols!.isNotEmpty) {
apiDocPageKeys.add(IndexedApiDocPage(i, page));
apiDocPageValues.add(page.symbols!.join(' '));
apiDocPageValues.add(page.symbols!);
}
}
}

// Note: we are not removing topics from this set, only adding them, no
// need for tracking the current topic count.
_topics.addAll(doc.tags
.where((t) => t.startsWith('topic:'))
.map((t) => t.split('topic:').last));
}

final packageKeys = _documents.map((d) => d.package).toList();
Expand All @@ -101,7 +85,7 @@ class InMemoryPackageIndex {
packageKeys,
_documents.map((d) => d.readme).toList(),
);
_apiSymbolIndex = TokenIndex(apiDocPageKeys, apiDocPageValues);
_apiSymbolIndex = TokenIndex.fromValues(apiDocPageKeys, apiDocPageValues);

// update download scores only if they were not set (should happen on old runtime's snapshot and local tests)
if (_documents.any((e) => e.downloadScore == null)) {
Expand Down Expand Up @@ -131,7 +115,7 @@ class InMemoryPackageIndex {
IndexInfo indexInfo() {
return IndexInfo(
isReady: true,
packageCount: _documentsByName.length,
packageCount: _documents.length,
lastUpdated: _lastUpdated,
);
}
Expand Down Expand Up @@ -363,7 +347,7 @@ class InMemoryPackageIndex {
}

// exact package name
if (_documentsByName.containsKey(parsedQueryText)) {
if (_nameToIndex.containsKey(parsedQueryText)) {
return parsedQueryText;
}

Expand All @@ -377,12 +361,9 @@ class InMemoryPackageIndex {
}
// Note: to keep it simple, we select the most downloaded one from competing matches.
return matches.reduce((a, b) {
if (_documentsByName[a]!.downloadCount >
_documentsByName[b]!.downloadCount) {
return a;
} else {
return b;
}
final aDoc = _documents[_nameToIndex[a]!];
final bDoc = _documents[_nameToIndex[b]!];
return aDoc.downloadCount > bDoc.downloadCount ? a : b;
});
}

Expand Down Expand Up @@ -487,7 +468,7 @@ class InMemoryPackageIndex {
packageScores.setValueMaxOf(doc.index, value);

// add the page and re-sort the current results
pages.add(MapEntry(doc.page.relativePath, value));
pages.add(MapEntry(doc.relativePath, value));
if (pages.length > 1) {
pages.sort((a, b) => -a.value.compareTo(b.value));
}
Expand Down Expand Up @@ -750,7 +731,8 @@ class IndexedPackageHit {

class IndexedApiDocPage {
final int index;
final ApiDocPage page;
final String relativePath;

IndexedApiDocPage(this.index, this.page);
IndexedApiDocPage(this.index, ApiDocPage page)
: relativePath = page.relativePath;
}
40 changes: 31 additions & 9 deletions app/lib/search/token_index.dart
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,21 +51,43 @@ class TokenIndex<K> {
if (text == null) {
continue;
}
final tokens = tokenize(text);
if (tokens == null || tokens.isEmpty) {
_build(i, text, skipDocumentWeight);
}
}

TokenIndex.fromValues(
List<K> ids,
List<List<String>?> values, {
bool skipDocumentWeight = false,
}) : _ids = ids {
assert(ids.length == values.length);
final length = values.length;
for (var i = 0; i < length; i++) {
final parts = values[i];

if (parts == null || parts.isEmpty) {
continue;
}
// Document weight is a highly scaled-down proxy of the length.
final dw =
skipDocumentWeight ? 1.0 : 1 + math.log(1 + tokens.length) / 100;
for (final e in tokens.entries) {
final token = e.key;
final weights = _inverseIds.putIfAbsent(token, () => {});
weights[i] = math.max(weights[i] ?? 0.0, e.value / dw);
for (final text in parts) {
_build(i, text, skipDocumentWeight);
}
}
}

void _build(int i, String text, bool skipDocumentWeight) {
final tokens = tokenize(text);
if (tokens == null || tokens.isEmpty) {
return;
}
// Document weight is a highly scaled-down proxy of the length.
final dw = skipDocumentWeight ? 1.0 : 1 + math.log(1 + tokens.length) / 100;
for (final e in tokens.entries) {
final token = e.key;
final weights = _inverseIds.putIfAbsent(token, () => {});
weights[i] = math.max(weights[i] ?? 0.0, e.value / dw);
}
}

factory TokenIndex.fromMap(Map<K, String> map) {
final keys = map.keys.toList();
final values = map.values.toList();
Expand Down