Improve ChordPro soft line break support (#1312)

* Sort scripts and fix double code generate

* Add support for parser tracing in test environment

Trace parsing like below:

```
const song = trace(chordSheet, (tracer) => (
  parser.parse(chordSheet, { softLineBreaks: true, tracer })
));
```

* Fix parsing soft line break following a bracket

Related to bettermusic#754

* Disable Jest's console to remove verbose logging

* `debug:chordpro` debug task using peggyjs editor

Running `yarn debug:chordpro` will open up `peggyjs.org/online.html` with the full chordpro grammar and JS compilation of the helpers included.

Author: martijnversluis

jest.config.ts

@@ -1,5 +1,6 @@
 /** @type {import('ts-jest/dist/types').InitialOptionsTsJest} */
 export default {
   preset: 'ts-jest',
+  setupFilesAfterEnv: ['<rootDir>/test/setup.ts'],
   testEnvironment: 'node',
 };

package.json

@@ -51,8 +51,10 @@
     "parcel": "^2.4.1",
     "parcel-transformer-hbs": "^1.0.4",
     "peggy": "^4.0.2",
+    "pegjs-backtrace": "^0.2.1",
     "pinst": "^3.0.0",
     "print": "^1.2.0",
+    "puppeteer": "^23.1.0",
     "shx": "^0.3.4",
     "theredoc": "^1.0.0",
     "ts-jest": "^29.2.3",
@@ -64,38 +66,39 @@
     "typescript-eslint": "^8.0.1"
   },
   "scripts": {
-    "build:suffix-normalize": "shx rm -rf src/normalize_mappings/suffix-normalize-mapping.ts && tsx src/normalize_mappings/generate-suffix-normalize-mapping.ts",
+    "build": "yarn build:code-generate && yarn build:sources && yarn build:bundle && yarn build:check-types",
+    "build:bundle": "yarn build:bundle:default && yarn build:bundle:min",
+    "build:bundle:default": "esbuild lib/index.js --outfile=lib/bundle.js --bundle --global-name=ChordSheetJS",
+    "build:bundle:min": "esbuild lib/index.js --outfile=lib/bundle.min.js --bundle --global-name=ChordSheetJS --minify-whitespace --minify-identifiers --minify-syntax",
+    "build:check-types": "tsc lib/main.d.ts",
     "build:chord-suffix-grammar": "yarn tsx script/generate_chord_suffix_grammar.ts",
+    "build:code-generate": "yarn build:suffix-normalize && yarn build:chord-suffix-grammar && yarn build:pegjs && yarn build:scales",
+    "build:pegjs": "yarn build:pegjs:chord && yarn build:pegjs:chordpro && yarn build:pegjs:chords-over-words",
+    "build:pegjs:chord": "tsx script/combine_files.ts src/parser/chord/base_grammar.pegjs src/parser/chord/suffix_grammar.pegjs src/parser/chord/combined_grammer.pegjs && peggy --plugin ts-pegjs -o src/parser/chord/peg_parser.ts src/parser/chord/combined_grammer.pegjs",
     "build:pegjs:chordpro": "tsx script/generate_parser.ts chord_pro --skip-chord-grammar",
     "build:pegjs:chords-over-words": "tsx script/generate_parser.ts chords_over_words",
-    "build:pegjs:chord": "tsx script/combine_files.ts src/parser/chord/base_grammar.pegjs src/parser/chord/suffix_grammar.pegjs src/parser/chord/combined_grammer.pegjs && peggy --plugin ts-pegjs -o src/parser/chord/peg_parser.ts src/parser/chord/combined_grammer.pegjs",
-    "build:pegjs": "yarn build:pegjs:chord && yarn build:pegjs:chordpro && yarn build:pegjs:chords-over-words",
     "build:scales": "tsx script/generate_scales.ts && yarn linter:fix src/scales.ts",
-    "build:code-generate": "yarn build:suffix-normalize && yarn build:chord-suffix-grammar && yarn build:pegjs && yarn build:scales",
     "build:sources": "parcel build",
-    "build:bundle": "yarn build:bundle:default && yarn build:bundle:min",
-    "build:bundle:default": "esbuild lib/index.js --outfile=lib/bundle.js --bundle --global-name=ChordSheetJS",
-    "build:bundle:min": "esbuild lib/index.js --outfile=lib/bundle.min.js --bundle --global-name=ChordSheetJS --minify-whitespace --minify-identifiers --minify-syntax",
-    "build:check-types": "tsc lib/main.d.ts",
-    "build": "yarn build:code-generate && yarn build:sources && yarn build:bundle && yarn build:check-types",
+    "build:suffix-normalize": "shx rm -rf src/normalize_mappings/suffix-normalize-mapping.ts && tsx src/normalize_mappings/generate-suffix-normalize-mapping.ts",
+    "ci": "yarn install && yarn lint && yarn test && yarn build && yarn readme",
+    "clean": "shx rm -rf node_modules && shx rm -rf lib",
+    "debug:chordpro": "tsx script/debug_parser.ts chord_pro --skip-chord-grammar",
     "dev": "parcel watch --no-cache",
-    "test": "yarn pretest && yarn lint && yarn jest:run",
-    "jest:watch": "jest --watch",
     "jest:debug": "bin/open_inspector && node --nolazy --inspect-brk ./node_modules/jest/bin/jest.js --runInBand --colors --verbose",
-    "jest:run:exp": "node --experimental-vm-modules node_modules/.bin/jest",
     "jest:run": "jest",
-    "linter:run": "yarn eslint",
-    "linter:fix": "yarn linter:run --fix",
-    "prelint": "yarn build:code-generate",
+    "jest:run:exp": "node --experimental-vm-modules node_modules/.bin/jest",
+    "jest:watch": "jest --watch",
     "lint": "yarn prelint && yarn linter:run .",
     "lint:fix": "yarn prelint && yarn linter:fix .",
-    "clean": "shx rm -rf node_modules && shx rm -rf lib",
-    "readme": "node_modules/.bin/jsdoc2md -f src/**/*.ts -f src/*.ts --configure ./jsdoc2md.json --template doc/README.hbs > README.md",
-    "prepublish": "pinst --disable && yarn install && yarn test && yarn build",
+    "linter:fix": "yarn linter:run --fix",
+    "linter:run": "yarn eslint",
     "postpublish": "pinst --enable",
-    "pretest": "yarn build:code-generate",
+    "prelint": "yarn pretest",
     "prepare": "husky install",
-    "ci": "yarn install && yarn lint && yarn test && yarn build && yarn readme"
+    "prepublish": "pinst --disable && yarn install && yarn test && yarn build",
+    "pretest": "NODE_ENV=test yarn build:code-generate",
+    "readme": "node_modules/.bin/jsdoc2md -f src/**/*.ts -f src/*.ts --configure ./jsdoc2md.json --template doc/README.hbs > README.md",
+    "test": "yarn pretest && yarn linter:run && yarn jest:run"
   },
   "dependencies": {
     "lodash.get": "^4.4.2"

script/debug_parser.ts

@@ -0,0 +1,73 @@
+import fs from 'fs';
+import puppeteer from 'puppeteer';
+import esbuild from 'esbuild';
+
+const parserName = process.argv[2];
+const args = process.argv.slice(3);
+const skipChordGrammar = args.includes('--skip-chord-grammar');
+
+const parserFolder = `./src/parser/${parserName}`;
+const grammarFile = `${parserFolder}/grammar.pegjs`;
+const helpersFile = `${parserFolder}/helpers.ts`;
+const chordGrammarFile = './src/parser/chord/base_grammar.pegjs';
+const chordSuffixGrammarFile = './src/parser/chord/suffix_grammar.pegjs';
+
+const parserGrammar = fs.readFileSync(grammarFile, 'utf8');
+const chordGrammar = skipChordGrammar ? '' : fs.readFileSync(chordGrammarFile);
+const chordSuffixGrammar = fs.readFileSync(chordSuffixGrammarFile);
+
+const result = esbuild.buildSync({
+  bundle: true,
+  entryPoints: [helpersFile],
+  globalName: 'helpers',
+  write: false,
+});
+
+const transpiledHelpers = result.outputFiles[0].text;
+
+const parserSource = [
+  `{\n${transpiledHelpers}\n}`,
+  parserGrammar,
+  chordGrammar,
+  chordSuffixGrammar,
+].join('\n\n');
+
+async function run() {
+  const browser = await puppeteer.launch({
+    args:['--start-maximized'],
+    defaultViewport: null,
+    headless: false,
+  });
+
+  async function shutdownHandler() {
+    await browser.close();
+  }
+
+  for (const event of ['exit', 'SIGINT', 'SIGUSR1', 'SIGUSR2', 'uncaughtException', 'SIGTERM']) {
+    process.on(event, shutdownHandler);
+  }
+
+  const [page] = await browser.pages();
+  await page.setViewport({ width: 0, height: 0 });
+  await page.goto('https://peggyjs.org/online.html');
+
+  await page.evaluate((grammar) => {
+    const textarea = document.getElementById('grammar');
+    if (!textarea) return;
+
+    const editorNode = textarea.nextSibling;
+    if (!editorNode) return;
+
+    // @ts-ignore
+    const editor = editorNode.CodeMirror;
+    editor.setValue(grammar);
+  }, parserSource);
+
+  while (true) {
+    // Loop forever to allow for interactive debugging with the online Peggy parser
+  }
+}
+
+run()
+  .then(() => console.log('Done'))
+  .catch(e => console.error(e));

script/generate_parser.ts

@@ -7,6 +7,9 @@ import fs from 'fs';
 const parserName = process.argv[2];
 const args = process.argv.slice(3);
 const skipChordGrammar = args.includes('--skip-chord-grammar');
+const enableTracing = process.env.NODE_ENV === 'test';
+
+console.warn('\x1b[34m', `👷 Building ${parserName} parser with${enableTracing ? '' : 'out'} tracing`);
 
 const parserFolder = `./src/parser/${parserName}`;
 const grammarFile = `${parserFolder}/grammar.pegjs`;
@@ -24,6 +27,9 @@ const source = peggy.generate(input, {
   grammarSource: grammarFile,
   output: 'source',
   format: 'commonjs',
+  trace: enableTracing,
 });
 
 fs.writeFileSync(outputFile, `import * as helpers from './helpers';\n\n${source}`);
+
+console.warn('\x1b[32m', `✨ Successfully built ${parserName} parser at ${outputFile}`);

src/parser/chord_pro/grammar.pegjs

@@ -99,18 +99,10 @@ Line
 Token
   = Tag
   / AnnotationLyricsPair
-  / ChordLyricsPair
+  / chordLyricsPair:ChordLyricsPair
   / MetaTernary
   / lyrics:Lyrics {
-      return lyrics.split('\xa0').flatMap((lyric, index) => ([
-          index == 0 ? null : { type: 'softLineBreak' },
-          {
-            type: 'chordLyricsPair',
-            chords: '',
-            lyrics: lyric,
-          },
-        ].filter(x => x)
-      ));
+      return helpers.applySoftLineBreaks(lyrics);
     }
 
 Comment
@@ -128,16 +120,13 @@ AnnotationLyricsPair
     }
 
 ChordLyricsPair
-  = chords:Chord lyrics:$(LyricsChar*) space:$(Space*) {
-      return {
-        type: 'chordLyricsPair',
-        chords: chords || '',
-        lyrics: lyrics + (space || ''),
-      };
+  = chords:Chord lyrics:LyricsChar* space:$(Space*) {
+      const mergedLyrics = lyrics.map(c => c.char || c).join('') + (space || '');
+      return helpers.breakChordLyricsPairOnSoftLineBreak(chords || '', mergedLyrics);
     }
 
 Lyrics
-  = lyrics: LyricsCharOrSpace+ {
+  = lyrics:LyricsCharOrSpace+ {
       return lyrics.map(c => c.char || c).join('');
     }
 

src/parser/chord_pro/helpers.ts

@@ -1,6 +1,8 @@
 import {
+  SerializedChordLyricsPair,
   SerializedItem,
   SerializedLine,
+  SerializedSoftLineBreak,
   SerializedTag,
 } from '../../serialized_types';
 
@@ -35,3 +37,55 @@ export function buildTag(name: string, value: string | null, location: FileRange
     location: location.start,
   };
 }
+
+export function stringSplitReplace(
+  string: string,
+  search: string,
+  replaceMatch: (subString: string) => any,
+  replaceRest: (subString: string) => any = (subString) => subString,
+): any[] {
+  const regExp = new RegExp(search, 'g');
+  const occurrences = Array.from(string.matchAll(regExp));
+  const result: string[] = [];
+  let index = 0;
+
+  occurrences.forEach((match) => {
+    const before = string.slice(index, match.index);
+    if (before !== '') result.push(replaceRest(before));
+    result.push(replaceMatch(match[0]));
+    index = match.index + match[0].length;
+  });
+
+  const rest = string.slice(index);
+  if (rest !== '') result.push(replaceRest(rest));
+
+  return result;
+}
+
+export function applySoftLineBreaks(lyrics: string): SerializedChordLyricsPair[] {
+  return stringSplitReplace(
+    lyrics,
+    '\xa0',
+    () => ({ type: 'softLineBreak' }),
+    (lyric) => ({ type: 'chordLyricsPair', chords: '', lyrics: lyric }),
+  ) as SerializedChordLyricsPair[];
+}
+
+export function breakChordLyricsPairOnSoftLineBreak(
+  chords: string,
+  lyrics: string,
+): Array<SerializedChordLyricsPair | SerializedSoftLineBreak> {
+  const pairs =  applySoftLineBreaks(lyrics || '');
+  let [first, ...rest] = pairs as Array<SerializedChordLyricsPair | SerializedSoftLineBreak>;
+  let addedLeadingChord: SerializedChordLyricsPair | null = null;
+
+  if (chords !== '') {
+    if (!first  || first.type === 'softLineBreak') {
+      addedLeadingChord = { type: 'chordLyricsPair', chords, lyrics: '' };
+    } else {
+      first = { ...first, chords };
+    }
+  }
+
+  return [addedLeadingChord, first || null, ...rest].filter((item) => item !== null);
+}

src/parser/chord_pro_parser.ts

@@ -3,6 +3,7 @@ import Song from '../chord_sheet/song';
 import ParserWarning from './parser_warning';
 import { normalizeLineEndings } from '../utilities';
 import ChordSheetSerializer from '../chord_sheet_serializer';
+import NullTracer from './null_tracer';
 
 export type ChordProParserOptions = ParseOptions & {
   softLineBreaks?: boolean;
@@ -33,7 +34,11 @@ class ChordProParser {
    * @returns {Song} The parsed song
    */
   parse(chordSheet: string, options?: ChordProParserOptions): Song {
-    const ast = parse(normalizeLineEndings(chordSheet), options);
+    const ast = parse(
+      normalizeLineEndings(chordSheet),
+      { tracer: new NullTracer(), ...options },
+    );
+
     this.song = new ChordSheetSerializer().deserialize(ast);
     return this.song;
   }

src/parser/chords_over_words_parser.ts

@@ -3,6 +3,7 @@ import ParserWarning from './parser_warning';
 import { parse, ParseOptions } from './chords_over_words/peg_parser';
 import { normalizeLineEndings } from '../utilities';
 import ChordSheetSerializer from '../chord_sheet_serializer';
+import NullTracer from './null_tracer';
 
 export type ChordsOverWordsParserOptions = ParseOptions & {
   softLineBreaks?: boolean;
@@ -68,7 +69,11 @@ class ChordsOverWordsParser {
    * @returns {Song} The parsed song
    */
   parse(chordSheet: string, options?: ChordsOverWordsParserOptions): Song {
-    const ast = parse(normalizeLineEndings(chordSheet), options);
+    const ast = parse(
+      normalizeLineEndings(chordSheet),
+      { tracer: new NullTracer(), ...options },
+    );
+
     this.song = new ChordSheetSerializer().deserialize(ast);
     return this.song;
   }

src/parser/null_tracer.ts

@@ -0,0 +1,5 @@
+class NullTracer {
+  trace() {}
+}
+
+export default NullTracer;