Chore: Sync fuzzer: Re-use the same CLI process for commands run on the same client (#12913)

Author: personalizedrefrigerator

.eslintignore

@@ -98,6 +98,7 @@ packages/app-cli/app/app.js
 packages/app-cli/app/base-command.js
 packages/app-cli/app/command-apidoc.js
 packages/app-cli/app/command-attach.js
+packages/app-cli/app/command-batch.js
 packages/app-cli/app/command-cat.js
 packages/app-cli/app/command-config.js
 packages/app-cli/app/command-cp.js
@@ -135,6 +136,7 @@ packages/app-cli/app/gui/StatusBarWidget.js
 packages/app-cli/app/services/plugins/PluginRunner.js
 packages/app-cli/app/setupCommand.js
 packages/app-cli/app/utils/initializeCommandService.js
+packages/app-cli/app/utils/iterateStdin.js
 packages/app-cli/app/utils/shimInitCli.js
 packages/app-cli/app/utils/testUtils.js
 packages/app-cli/tests/HtmlToMd.js
@@ -1747,6 +1749,7 @@ packages/tools/fuzzer/utils/SeededRandom.js
 packages/tools/fuzzer/utils/getNumberProperty.js
 packages/tools/fuzzer/utils/getProperty.js
 packages/tools/fuzzer/utils/getStringProperty.js
+packages/tools/fuzzer/utils/openDebugSession.js
 packages/tools/fuzzer/utils/retryWithCount.js
 packages/tools/generate-database-types.js
 packages/tools/generate-images.js

.gitignore

@@ -71,6 +71,7 @@ packages/app-cli/app/app.js
 packages/app-cli/app/base-command.js
 packages/app-cli/app/command-apidoc.js
 packages/app-cli/app/command-attach.js
+packages/app-cli/app/command-batch.js
 packages/app-cli/app/command-cat.js
 packages/app-cli/app/command-config.js
 packages/app-cli/app/command-cp.js
@@ -108,6 +109,7 @@ packages/app-cli/app/gui/StatusBarWidget.js
 packages/app-cli/app/services/plugins/PluginRunner.js
 packages/app-cli/app/setupCommand.js
 packages/app-cli/app/utils/initializeCommandService.js
+packages/app-cli/app/utils/iterateStdin.js
 packages/app-cli/app/utils/shimInitCli.js
 packages/app-cli/app/utils/testUtils.js
 packages/app-cli/tests/HtmlToMd.js
@@ -1720,6 +1722,7 @@ packages/tools/fuzzer/utils/SeededRandom.js
 packages/tools/fuzzer/utils/getNumberProperty.js
 packages/tools/fuzzer/utils/getProperty.js
 packages/tools/fuzzer/utils/getStringProperty.js
+packages/tools/fuzzer/utils/openDebugSession.js
 packages/tools/fuzzer/utils/retryWithCount.js
 packages/tools/generate-database-types.js
 packages/tools/generate-images.js

packages/app-cli/app/app.ts

@@ -9,7 +9,6 @@ import Tag from '@joplin/lib/models/Tag';
 import Setting, { Env } from '@joplin/lib/models/Setting';
 import { reg } from '@joplin/lib/registry.js';
 import { dirname, fileExtension } from '@joplin/lib/path-utils';
-import { splitCommandString } from '@joplin/utils';
 import { _ } from '@joplin/lib/locale';
 import { pathExists, readFile, readdirSync } from 'fs-extra';
 import RevisionService from '@joplin/lib/services/RevisionService';
@@ -19,7 +18,6 @@ import { FolderEntity, NoteEntity } from '@joplin/lib/services/database/types';
 import initializeCommandService from './utils/initializeCommandService';
 const { cliUtils } = require('./cli-utils.js');
 const Cache = require('@joplin/lib/Cache');
-const { splitCommandBatch } = require('@joplin/lib/string-utils');
 
 class Application extends BaseApplication {
 
@@ -381,22 +379,6 @@ class Application extends BaseApplication {
 		return output;
 	}
 
-	public async commandList(argv: string[]) {
-		if (argv.length && argv[0] === 'batch') {
-			const commands = [];
-			const commandLines = splitCommandBatch(await readFile(argv[1], 'utf-8'));
-
-			for (const commandLine of commandLines) {
-				if (!commandLine.trim()) continue;
-				const splitted = splitCommandString(commandLine.trim());
-				commands.push(splitted);
-			}
-			return commands;
-		} else {
-			return [argv];
-		}
-	}
-
 	// We need this special case here because by the time the `version` command
 	// runs, the keychain has already been setup.
 	public checkIfKeychainEnabled(argv: string[]) {
@@ -433,15 +415,10 @@ class Application extends BaseApplication {
 		if (argv.length) {
 			this.gui_ = this.dummyGui();
 
-			this.currentFolder_ = await Folder.load(Setting.value('activeFolderId'));
-
 			await this.applySettingsSideEffects();
-
+			await this.refreshCurrentFolder();
 			try {
-				const commands = await this.commandList(argv);
-				for (const command of commands) {
-					await this.execCommand(command);
-				}
+				await this.execCommand(argv);
 			} catch (error) {
 				if (this.showStackTraces_) {
 					console.error(error);

packages/app-cli/app/command-batch.js

@@ -0,0 +1,79 @@
+import { splitCommandBatch } from '@joplin/lib/string-utils';
+import BaseCommand from './base-command';
+import { _ } from '@joplin/lib/locale';
+import { splitCommandString } from '@joplin/utils';
+import iterateStdin from './utils/iterateStdin';
+import { readFile } from 'fs-extra';
+import app from './app';
+
+interface Options {
+	'file-path': string;
+	options: {
+		'continue-on-failure': boolean;
+	};
+}
+
+class Command extends BaseCommand {
+	public usage() {
+		return 'batch <file-path>';
+	}
+
+	public options() {
+		return [
+			// These are present mostly for testing purposes
+			['--continue-on-failure', 'Continue running commands when one command in the batch fails.'],
+		];
+	}
+
+	public description() {
+		return _('Runs the commands contained in the text file. There should be one command per line.');
+	}
+
+	private streamCommands_ = async function*(filePath: string) {
+		const processLines = function*(lines: string) {
+			const commandLines = splitCommandBatch(lines);
+
+			for (const command of commandLines) {
+				if (!command.trim()) continue;
+				yield splitCommandString(command.trim());
+			}
+		};
+
+		if (filePath === '-') { // stdin
+			// Iterating over standard input conflicts with the CLI app's GUI.
+			if (app().hasGui()) {
+				throw new Error(_('Reading commands from standard input is only available in CLI mode.'));
+			}
+
+			for await (const lines of iterateStdin('command> ')) {
+				yield* processLines(lines);
+			}
+		} else {
+			const data = await readFile(filePath, 'utf-8');
+			yield* processLines(data);
+		}
+	};
+
+	public async action(options: Options) {
+		let lastError;
+		for await (const command of this.streamCommands_(options['file-path'])) {
+			try {
+				await app().refreshCurrentFolder();
+				await app().execCommand(command);
+			} catch (error) {
+				if (options.options['continue-on-failure']) {
+					app().stdout(error.message);
+					lastError = error;
+				} else {
+					throw error;
+				}
+			}
+		}
+
+		if (lastError) {
+			throw lastError;
+		}
+	}
+}
+
+module.exports = Command;

packages/app-cli/app/command-batch.ts

@@ -14,17 +14,25 @@ class Command extends BaseCommand {
 		return `${_('Start, stop or check the API server. To specify on which port it should run, set the api.port config variable. Commands are (%s).', ['start', 'stop', 'status'].join('|'))} This is an experimental feature - use at your own risks! It is recommended that the server runs off its own separate profile so that no two CLI instances access that profile at the same time. Use --profile to specify the profile path.`;
 	}
 
+	options() {
+		return [
+			['--exit-early', 'Allow the command to exit while the server is still running. The server will still stop when the app exits. Valid only for the `start` subcommand.'],
+			['--quiet', 'Log less information to the console. More verbose logs will still be available through log-clipper.txt.'],
+		];
+	}
+
 	async action(args) {
 		const command = args.command;
 
 		const ClipperServer = require('@joplin/lib/ClipperServer').default;
 		ClipperServer.instance().initialize();
 		const stdoutFn = (...s) => this.stdout(s.join(' '));
+		const ignoreOutputFn = ()=>{};
 		const clipperLogger = new Logger();
 		clipperLogger.addTarget('file', { path: `${Setting.value('profileDir')}/log-clipper.txt` });
 		clipperLogger.addTarget('console', { console: {
-			info: stdoutFn,
-			warn: stdoutFn,
+			info: args.options.quiet ? ignoreOutputFn : stdoutFn,
+			warn: args.options.quiet ? ignoreOutputFn : stdoutFn,
 			error: stdoutFn,
 		} });
 		ClipperServer.instance().setDispatch(() => {});
@@ -38,7 +46,11 @@ class Command extends BaseCommand {
 				this.stdout(_('Server is already running on port %d', runningOnPort));
 			} else {
 				await shim.fsDriver().writeFile(pidPath, process.pid.toString(), 'utf-8');
-				await ClipperServer.instance().start(); // Never exit
+				const promise = ClipperServer.instance().start();
+
+				if (!args.options['exit-early']) {
+					await promise; // Never exit
+				}
 			}
 		} else if (command === 'status') {
 			this.stdout(runningOnPort ? _('Server is running on port %d', runningOnPort) : _('Server is not running.'));

packages/app-cli/app/command-server.js

@@ -0,0 +1,54 @@
+import { createInterface } from 'readline/promises';
+
+const iterateStdin = async function*(prompt: string) {
+	let nextLineListeners: (()=> void)[] = [];
+	const dispatchAllListeners = () => {
+		const listeners = nextLineListeners;
+		nextLineListeners = [];
+		for (const listener of listeners) {
+			listener();
+		}
+	};
+
+	const rl = createInterface({
+		input: process.stdin,
+		output: process.stdout,
+	});
+	rl.setPrompt(prompt);
+
+	let buffer: string[] = [];
+	rl.on('line', (line) => {
+		buffer.push(line);
+		dispatchAllListeners();
+	});
+
+	let done = false;
+	rl.on('close', () => {
+		done = true;
+		dispatchAllListeners();
+	});
+
+	const readNextLines = () => {
+		return new Promise<string|null>(resolve => {
+			if (done) {
+				resolve(null);
+			} else if (buffer.length > 0) {
+				resolve(buffer.join('\n'));
+				buffer = [];
+			} else {
+				nextLineListeners.push(() => {
+					resolve(buffer.join('\n'));
+					buffer = [];
+				});
+			}
+		});
+	};
+
+	while (!done) {
+		rl.prompt();
+		const lines = await readNextLines();
+		yield lines;
+	}
+};
+
+export default iterateStdin;

packages/app-cli/app/utils/iterateStdin.ts

@@ -395,17 +395,18 @@ export default class BaseApplication {
 			// - All the calls below are cheap or do nothing if there's nothing
 			//   to do.
 			'syncInfoCache': async () => {
-				if (this.hasGui()) {
-					appLogger.info('"syncInfoCache" was changed - setting up encryption related code');
+				appLogger.info('"syncInfoCache" was changed - setting up encryption related code');
 
-					await loadMasterKeysFromSettings(EncryptionService.instance());
-					void DecryptionWorker.instance().scheduleStart();
-					const loadedMasterKeyIds = EncryptionService.instance().loadedMasterKeyIds();
+				await loadMasterKeysFromSettings(EncryptionService.instance());
+				const loadedMasterKeyIds = EncryptionService.instance().loadedMasterKeyIds();
+
+				this.dispatch({
+					type: 'MASTERKEY_REMOVE_NOT_LOADED',
+					ids: loadedMasterKeyIds,
+				});
 
-					this.dispatch({
-						type: 'MASTERKEY_REMOVE_NOT_LOADED',
-						ids: loadedMasterKeyIds,
-					});
+				if (this.hasGui()) {
+					void DecryptionWorker.instance().scheduleStart();
 
 					// Schedule a sync operation so that items that need to be encrypted
 					// are sent to sync target.