Chore: Sync fuzzer: Add new possible actions: Adding and syncing a new temporary client on an existing account

packages/tools/fuzzer/ActionTracker.ts

@@ -56,10 +56,13 @@ class ActionTracker {
 
 	public track(client: { email: string }) {
 		const clientId = client.email;
-		this.tree_.set(clientId, {
-			childIds: [],
-			sharedFolderIds: [],
-		});
+		// If the client's remote account already exists, continue using it:
+		if (!this.tree_.has(clientId)) {
+			this.tree_.set(clientId, {
+				childIds: [],
+				sharedFolderIds: [],
+			});
+		}
 
 		const getChildIds = (itemId: ItemId) => {
 			const item = this.idToItem_.get(itemId);

packages/tools/fuzzer/Client.ts

@@ -1,5 +1,5 @@
 import uuid, { createSecureRandom } from '@joplin/lib/uuid';
-import { ActionableClient, FolderMetadata, FuzzContext, HttpMethod, ItemId, Json, NoteData, RandomFolderOptions, UserData } from './types';
+import { ActionableClient, FolderMetadata, FuzzContext, HttpMethod, ItemId, Json, NoteData, RandomFolderOptions } from './types';
 import { join } from 'path';
 import { mkdir, remove } from 'fs-extra';
 import getStringProperty from './utils/getStringProperty';
@@ -12,6 +12,7 @@ import { commandToString } from '@joplin/utils';
 import { quotePath } from '@joplin/utils/path';
 import getNumberProperty from './utils/getNumberProperty';
 import retryWithCount from './utils/retryWithCount';
+import resolvePathWithinDir from '@joplin/lib/utils/resolvePathWithinDir';
 import { msleep, Second } from '@joplin/utils/time';
 import shim from '@joplin/lib/shim';
 import { spawn } from 'child_process';
@@ -21,6 +22,62 @@ import Stream = require('stream');
 
 const logger = Logger.create('Client');
 
+type AccountData = Readonly<{
+	email: string;
+	password: string;
+	serverId: string;
+	e2eePassword: string;
+	associatedClientCount: number;
+	onClientConnected: ()=> void;
+	onClientDisconnected: ()=> Promise<void>;
+}>;
+
+const createNewAccount = async (email: string, context: FuzzContext): Promise<AccountData> => {
+	const password = createSecureRandom();
+	const apiOutput = await context.execApi('POST', 'api/users', {
+		email,
+	});
+	const serverId = getStringProperty(apiOutput, 'id');
+
+	// The password needs to be set *after* creating the user.
+	const userRoute = `api/users/${encodeURIComponent(serverId)}`;
+	await context.execApi('PATCH', userRoute, {
+		email,
+		password,
+		email_confirmed: 1,
+	});
+
+	const closeAccount = async () => {
+		await context.execApi('DELETE', userRoute, {});
+	};
+
+	let referenceCounter = 0;
+	return {
+		email,
+		password,
+		e2eePassword: createSecureRandom().replace(/^-/, '_'),
+		serverId,
+		get associatedClientCount() {
+			return referenceCounter;
+		},
+		onClientConnected: () => {
+			referenceCounter++;
+		},
+		onClientDisconnected: async () => {
+			referenceCounter --;
+			assert.ok(referenceCounter >= 0, 'reference counter should be non-negative');
+			if (referenceCounter === 0) {
+				await closeAccount();
+			}
+		},
+	};
+};
+
+type ApiData = Readonly<{
+	port: number;
+	token: string;
+}>;
+
 type OnCloseListener = ()=> void;
 
 type ChildProcessWrapper = {
@@ -33,76 +90,56 @@ type ChildProcessWrapper = {
 // Should match the prompt used by the CLI "batch" command.
 const cliProcessPromptString = 'command> ';
 
-
-
 class Client implements ActionableClient {
 	public readonly email: string;
 
 	public static async create(actionTracker: ActionTracker, context: FuzzContext) {
+		const account = await createNewAccount(`${uuid.create()}@localhost`, context);
+
+		try {
+			return await this.fromAccount(account, actionTracker, context);
+		} catch (error) {
+			logger.error('Error creating client:', error);
+			await account.onClientDisconnected();
+			throw error;
+		}
+	}
+
+	private static async fromAccount(account: AccountData, actionTracker: ActionTracker, context: FuzzContext) {
 		const id = uuid.create();
 		const profileDirectory = join(context.baseDir, id);
 		await mkdir(profileDirectory);
 
-		const email = `${id}@localhost`;
-		const password = createSecureRandom();
-		const apiOutput = await context.execApi('POST', 'api/users', {
-			email,
-		});
-		const serverId = getStringProperty(apiOutput, 'id');
-
-		// The password needs to be set *after* creating the user.
-		const userRoute = `api/users/${encodeURIComponent(serverId)}`;
-		await context.execApi('PATCH', userRoute, {
-			email,
-			password,
-			email_confirmed: 1,
-		});
-
-		const closeAccount = async () => {
-			await context.execApi('DELETE', userRoute, {});
+		const apiData: ApiData = {
+			token: createSecureRandom().replace(/[-]/g, '_'),
+			port: await ClipperServer.instance().findAvailablePort(),
 		};
 
-		try {
-			const userData = {
-				email: getStringProperty(apiOutput, 'email'),
-				password,
-			};
-
-			assert.equal(email, userData.email);
-
-			const apiToken = createSecureRandom().replace(/[-]/g, '_');
-			const apiPort = await ClipperServer.instance().findAvailablePort();
-
-			const client = new Client(
-				actionTracker.track({ email }),
-				userData,
-				profileDirectory,
-				apiPort,
-				apiToken,
-			);
-
-			client.onClose(closeAccount);
+		const client = new Client(
+			context,
+			actionTracker,
+			actionTracker.track({ email: account.email }),
+			account,
+			profileDirectory,
+			apiData,
+			`${account.email}${account.associatedClientCount ? ` (${account.associatedClientCount})` : ''}`,
+		);
 
-			// Joplin Server sync
-			await client.execCliCommand_('config', 'sync.target', '9');
-			await client.execCliCommand_('config', 'sync.9.path', context.serverUrl);
-			await client.execCliCommand_('config', 'sync.9.username', userData.email);
-			await client.execCliCommand_('config', 'sync.9.password', userData.password);
-			await client.execCliCommand_('config', 'api.token', apiToken);
-			await client.execCliCommand_('config', 'api.port', String(apiPort));
+		account.onClientConnected();
 
-			const e2eePassword = createSecureRandom().replace(/^-/, '_');
-			await client.execCliCommand_('e2ee', 'enable', '--password', e2eePassword);
-			logger.info('Created and configured client');
+		// Joplin Server sync
+		await client.execCliCommand_('config', 'sync.target', '9');
+		await client.execCliCommand_('config', 'sync.9.path', context.serverUrl);
+		await client.execCliCommand_('config', 'sync.9.username', account.email);
+		await client.execCliCommand_('config', 'sync.9.password', account.password);
+		await client.execCliCommand_('config', 'api.token', apiData.token);
+		await client.execCliCommand_('config', 'api.port', String(apiData.port));
 
-			await client.startClipperServer_();
+		await client.execCliCommand_('e2ee', 'enable', '--password', account.e2eePassword);
+		logger.info('Created and configured client');
 
-			await client.sync();
-			return client;
-		} catch (error) {
-			await closeAccount();
-			throw error;
-		}
+		await client.startClipperServer_();
+		return client;
 	}
 
 	private onCloseListeners_: OnCloseListener[] = [];
@@ -116,13 +153,15 @@ class Client implements ActionableClient {
 	private transcript_: string[] = [];
 
 	private constructor(
+		private readonly context_: FuzzContext,
+		private readonly globalActionTracker_: ActionTracker,
 		private readonly tracker_: ActionableClient,
-		userData: UserData,
+		private readonly account_: AccountData,
 		private readonly profileDirectory: string,
-		private readonly apiPort_: number,
-		private readonly apiToken_: string,
+		private readonly apiData_: ApiData,
+		private readonly clientLabel_: string,
 	) {
-		this.email = userData.email;
+		this.email = account_.email;
 
 		// Don't skip child process-related tasks.
 		this.childProcessQueue_.setCanSkipTaskHandler(() => false);
@@ -186,9 +225,11 @@ class Client implements ActionableClient {
 	public async close() {
 		assert.ok(!this.closed_, 'should not be closed');
 
+		await this.account_.onClientDisconnected();
+
 		// Before removing the profile directory, verify that the profile directory is in the
 		// expected location:
-		const profileDirectory = this.profileDirectory;
+		const profileDirectory = resolvePathWithinDir(this.context_.baseDir, this.profileDirectory);
 		assert.ok(profileDirectory, 'profile directory for client should be contained within the main temporary profiles directory (should be safe to delete)');
 		await remove(profileDirectory);
 
@@ -204,8 +245,16 @@ class Client implements ActionableClient {
 		this.onCloseListeners_.push(listener);
 	}
 
+	public async createClientOnSameAccount() {
+		return await Client.fromAccount(this.account_, this.globalActionTracker_, this.context_);
+	}
+
+	public hasSameAccount(other: Client) {
+		return other.account_ === this.account_;
+	}
+
 	public get label() {
-		return this.email;
+		return this.clientLabel_;
 	}
 
 	private get cliCommandArguments() {
@@ -318,8 +367,8 @@ class Client implements ActionableClient {
 	// eslint-disable-next-line no-dupe-class-members -- This is not a duplicate class member
 	private async execApiCommand_(method: HttpMethod, route: string, data: Json|null = null): Promise<string> {
 		route = route.replace(/^[/]/, '');
-		const url = new URL(`http://localhost:${this.apiPort_}/${route}`);
-		url.searchParams.append('token', this.apiToken_);
+		const url = new URL(`http://localhost:${this.apiData_.port}/${route}`);
+		url.searchParams.append('token', this.apiData_.token);
 
 		this.transcript_.push(`\n[[${method} ${url}; body: ${JSON.stringify(data)}]]\n`);
 

packages/tools/fuzzer/ClientPool.ts

@@ -45,6 +45,17 @@ export default class ClientPool {
 		];
 	}
 
+	public async newWithSameAccount(sourceClient: Client) {
+		const client = await sourceClient.createClientOnSameAccount();
+		this.listenForClientClose_(client);
+		this.clients_ = [...this.clients_, client];
+		return client;
+	}
+
+	public othersWithSameAccount(client: Client) {
+		return this.clients_.filter(other => other !== client && other.hasSameAccount(client));
+	}
+
 	public async checkState() {
 		for (const client of this.clients_) {
 			await client.checkState();

packages/tools/fuzzer/sync-fuzzer.ts

@@ -150,7 +150,7 @@ const doRandomAction = async (context: FuzzContext, client: Client, clientPool:
 			});
 			if (!target) return false;
 
-			const other = clientPool.randomClient(c => c !== client);
+			const other = clientPool.randomClient(c => !c.hasSameAccount(client));
 			await client.shareFolder(target.id, other);
 			return true;
 		},
@@ -191,6 +191,63 @@ const doRandomAction = async (context: FuzzContext, client: Client, clientPool:
 			await client.moveItem(target.id, newParent.id);
 			return true;
 		},
+		newClientOnSameAccount: async () => {
+			const welcomeNoteCount = context.randInt(0, 30);
+			logger.info(`Syncing a new client on the same account ${welcomeNoteCount > 0 ? `(with ${welcomeNoteCount} initial notes)` : ''}`);
+			const createClientInitialNotes = async (client: Client) => {
+				if (welcomeNoteCount === 0) return;
+
+				// Create a new folder. Usually, new clients have a default set of
+				// welcome notes when first syncing.
+				const testNotesFolderId = uuid.create();
+				await client.createFolder({
+					id: testNotesFolderId,
+					title: 'Test -- from secondary client',
+					parentId: '',
+				});
+
+				for (let i = 0; i < welcomeNoteCount; i++) {
+					await client.createNote({
+						parentId: testNotesFolderId,
+						id: uuid.create(),
+						title: `Test note ${i}/${welcomeNoteCount}`,
+						body: `Test note (in account ${client.email}), created ${Date.now()}.`,
+					});
+				}
+			};
+
+			await client.sync();
+
+			const other = await clientPool.newWithSameAccount(client);
+			await createClientInitialNotes(other);
+
+			// Sometimes, a delay is needed between client creation
+			// and initial sync. Retry the initial sync and the checkState
+			// on failure:
+			await retryWithCount(async () => {
+				await other.sync();
+				await other.checkState();
+			}, {
+				delayOnFailure: (count) => Second * count,
+				count: 3,
+				onFail: async (error) => {
+					logger.warn('other.sync/other.checkState failed with', error, 'retrying...');
+				},
+			});
+
+			await client.sync();
+			return true;
+		},
+		removeClientsOnSameAccount: async () => {
+			const others = clientPool.othersWithSameAccount(client);
+			if (others.length === 0) return false;
+
+			for (const otherClient of others) {
+				assert.notEqual(otherClient, client);
+				await otherClient.close();
+			}
+			return true;
+		},
 	};
 
 	const actionKeys = [...Object.keys(actions)] as (keyof typeof actions)[];