Generate identical atn simulation output across targets to compare parsing functionality

doc/creating-a-language-target.md

@@ -45,3 +45,26 @@ $ mvn compile
 ```
 
 That should proceed with success. See [Building ANTLR](building-antlr.md) for more details.
+
+## Comparing your target's parsing decisionmaking with Java's
+
+ANTLR's power comes from it's dynamic parsing strategy, but that means each target
+must implement that complicated algorithm. You should compare your target's debug
+output for ParserATNSimulator with Java's.
+
+Run this so we get right jars before trying this script:
+
+```
+cd ANTLR-ROOT-DIR
+mvn install -DskipTests=true
+cd runtime-tests 
+mvn install -DskipTests=true # yes do it again
+```
+
+Run the script from `runtime-tests` dir with
+
+```
+../scripts/traceatn.sh /tmp/JSON.g4 json -target Go /tmp/foo.json
+```
+
+or whatever your test grammar, start rule, target, test input are.

runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.cpp.stg

@@ -33,7 +33,7 @@ int main(int argc, const char* argv[]) {
   CommonTokenStream tokens(&lexer);
 <if(parserName)>
   <parserName> parser(&tokens);
-<if(debug)>
+<if(showDiagnosticErrors)>
   DiagnosticErrorListener errorListener;
   parser.addErrorListener(&errorListener);
 <endif>

runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.cs.stg

@@ -1,5 +1,6 @@
 using System;
 using Antlr4.Runtime;
+using Antlr4.Runtime.Atn;
 using Antlr4.Runtime.Tree;
 using System.Text;
 
@@ -12,8 +13,11 @@ public class Test {
 		var tokens = new CommonTokenStream(lex);
 <if(parserName)>
 		var parser = new <parserName>(tokens);
-<if(debug)>
+<if(showDiagnosticErrors)>
 		parser.AddErrorListener(new DiagnosticErrorListener());
+<endif>
+<if(traceATN)>
+	    ParserATNSimulator.trace_atn_sim = true;
 <endif>
 		parser.BuildParseTree = true;
 		var tree = parser.<parserStartRuleName>();

runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.dart.stg

@@ -12,7 +12,7 @@ void main(List\<String> args) async {
   final tokens = CommonTokenStream(lex);
 <if(parserName)>
   final parser = <parserName>(tokens);
-<if(debug)>
+<if(showDiagnosticErrors)>
   parser.addErrorListener(new DiagnosticErrorListener());
 <endif>
 <if(profile)>

runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.go.stg

@@ -27,6 +27,9 @@ func (this *TreeShapeListener) EnterEveryRule(ctx antlr.ParserRuleContext) {
 <endif>
 
 func main() {
+<if(traceATN)>
+    antlr.ParserATNSimulatorTraceATNSim = true
+<endif>
 	input, err := antlr.NewFileStream(os.Args[1])
 		if err != nil {
 		fmt.Printf("Failed to find file: %v", err)
@@ -36,7 +39,7 @@ func main() {
 	stream := antlr.NewCommonTokenStream(lexer,0)
 <if(parserName)>
 	p := parser.New<parserName>(stream)
-<if(debug)>
+<if(showDiagnosticErrors)>
 	p.AddErrorListener(antlr.NewDiagnosticErrorListener(true))
 <endif>
 	p.BuildParseTrees = true

runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.java.stg

@@ -29,8 +29,11 @@ public class Test {
 		parser.setOutStream(outStream);
 		parser.removeErrorListeners();
 		parser.addErrorListener(errorListener);
-<if(debug)>
+<if(showDiagnosticErrors)>
 		parser.addErrorListener(new DiagnosticErrorListener());
+<endif>
+<if(traceATN)>
+		ParserATNSimulator.trace_atn_sim = true;
 <endif>
 		parser.setBuildParseTree(true);
 <if(profile)>

runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.js.stg

@@ -28,7 +28,7 @@ function main(argv) {
     var stream = new antlr4.CommonTokenStream(lexer);
 <if(parserName)>
     var parser = new <parserName>(stream);
-<if(debug)>
+<if(showDiagnosticErrors)>
     parser.addErrorListener(new antlr4.error.DiagnosticErrorListener());
 <endif>
     parser.buildParseTrees = true;
@@ -38,6 +38,10 @@ function main(argv) {
         return this;
     };
     parser.printer = new printer();
+<if(traceATN)>
+	parser._interp.trace_atn_sim = true;
+	antlr4.context.PredictionContext.trace_atn_sim = true;
+<endif>
     var tree = parser.<parserStartRuleName>();
     antlr4.tree.ParseTreeWalker.DEFAULT.walk(new TreeShapeListener(), tree);
 <else>

runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.php.stg

@@ -48,7 +48,7 @@ $lexer->addErrorListener(new ConsoleErrorListener());
 $tokens = new CommonTokenStream($lexer);
 <if(parserName)>
 $parser = new <parserName>($tokens);
-<if(debug)>
+<if(showDiagnosticErrors)>
 $parser->addErrorListener(new DiagnosticErrorListener());
 <endif>
 $parser->addErrorListener(new ConsoleErrorListener());

runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.py.stg

@@ -27,12 +27,16 @@ class TreeShapeListener(ParseTreeListener):
 <endif>
 
 def main(argv):
+<if(traceATN)>
+    ParserATNSimulator.trace_atn_sim = True
+    PredictionContext._trace_atn_sim = True
+<endif>
     input = FileStream(argv[1], encoding='utf-8', errors='replace')
     lexer = <lexerName>(input)
     stream = CommonTokenStream(lexer)
 <if(parserName)>
     parser = <parserName>(stream)
-<if(debug)>
+<if(showDiagnosticErrors)>
     parser.addErrorListener(DiagnosticErrorListener())
 <endif>
     parser.buildParseTrees = True

runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/main.swift.stg

@@ -23,7 +23,7 @@ let lex = <lexerName>(input)
 let tokens = CommonTokenStream(lex)
 <if(parserName)>
 let parser = try <parserName>(tokens)
-<if(debug)>
+<if(showDiagnosticErrors)>
 parser.addErrorListener(DiagnosticErrorListener())
 <endif>
 parser.setBuildParseTree(true)

runtime-testsuite/test/org/antlr/v4/test/runtime/CustomDescriptors.java

@@ -50,7 +50,7 @@ private static RuntimeTestDescriptor getLineSeparatorLfDescriptor() {
 				"lexer grammar L;\n" +
 						"T: ~'\\n'+;\n" +
 						"SEPARATOR: '\\n';",
-				null, false, false, null, uri);
+				null, false, false, false, null, uri);
 	}
 
 	private static RuntimeTestDescriptor getLineSeparatorCrLfDescriptor() {
@@ -71,7 +71,7 @@ private static RuntimeTestDescriptor getLineSeparatorCrLfDescriptor() {
 				"lexer grammar L;\n" +
 						"T: ~'\\r'+;\n" +
 						"SEPARATOR: '\\r\\n';",
-				null, false, false, null, uri);
+				null, false, false, false, null, uri);
 	}
 
 	private static RuntimeTestDescriptor getLargeLexerDescriptor() {
@@ -98,7 +98,7 @@ private static RuntimeTestDescriptor getLargeLexerDescriptor() {
 				"",
 				grammarName,
 				grammar.toString(),
-				null, false, false, null, uri);
+				null, false, false, false, null, uri);
 	}
 
 	private static RuntimeTestDescriptor getAtnStatesSizeMoreThan65535Descriptor() {
@@ -147,7 +147,7 @@ private static RuntimeTestDescriptor getAtnStatesSizeMoreThan65535Descriptor() {
 				"",
 				grammarName,
 				grammar.toString(),
-				null, false, false,
+				null, false, false, false,
 				new String[] {"CSharp", "Python2", "Python3", "Go", "PHP", "Swift", "JavaScript", "Dart"}, uri);
 	}
 
@@ -192,6 +192,6 @@ private static RuntimeTestDescriptor getMultiTokenAlternativeDescriptor() {
 				"r",
 				"P",
 				grammar,
-				null, false, false, null, uri);
+				null, false, false, false, null, uri);
 	}
 }

runtime-testsuite/test/org/antlr/v4/test/runtime/Processor.java

@@ -61,7 +61,7 @@ public ProcessorResult start() throws InterruptedException, IOException {
 		String output = stdoutReader.toString();
 		String errors = stderrReader.toString();
 		if (throwOnNonZeroErrorCode && process.exitValue() != 0) {
-			throw new InterruptedException(joinLines(output, errors));
+			throw new InterruptedException("Exit code "+process.exitValue()+" with output:\n"+joinLines(output, errors));
 		}
 		return new ProcessorResult(process.exitValue(), output, errors);
 	}

runtime-testsuite/test/org/antlr/v4/test/runtime/RunOptions.java

@@ -18,6 +18,7 @@ public class RunOptions {
 	public final String input;
 	public final boolean profile;
 	public final boolean showDiagnosticErrors;
+	public final boolean traceATN;
 	public final boolean showDFA;
 	public final Stage endStage;
 	public final boolean returnObject;
@@ -26,7 +27,7 @@ public class RunOptions {
 	public RunOptions(String grammarFileName, String grammarStr, String parserName, String lexerName,
 					  boolean useListener, boolean useVisitor, String startRuleName,
 					  String input, boolean profile, boolean showDiagnosticErrors,
-					  boolean showDFA, Stage endStage, boolean returnObject,
+					  boolean traceATN, boolean showDFA, Stage endStage, boolean returnObject,
 					  String language, String superClass) {
 		this.grammarFileName = grammarFileName;
 		this.grammarStr = grammarStr;
@@ -61,6 +62,7 @@ else if (lexerName != null) {
 		this.input = input;
 		this.profile = profile;
 		this.showDiagnosticErrors = showDiagnosticErrors;
+		this.traceATN = traceATN;
 		this.showDFA = showDFA;
 		this.endStage = endStage;
 		this.returnObject = returnObject;

runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeRunner.java

@@ -22,6 +22,14 @@
 import static org.antlr.v4.test.runtime.RuntimeTestUtils.*;
 
 public abstract class RuntimeRunner implements AutoCloseable {
+	/** Turn this on to see output like:
+	 *  RUNNING cmake . -DCMAKE_BUILD_TYPE=Release in /Users/parrt/antlr/code/antlr4/runtime/Cpp
+	 *  RUNNING make -j 20 in /Users/parrt/antlr/code/antlr4/runtime/Cpp
+	 *  RUNNING ln -s /Users/parrt/antlr/code/antlr4/runtime/Cpp/dist/libantlr4-runtime.dylib in /var/folders/w1/_nr4stn13lq0rvjdkwh7q8cc0000gn/T/CppRunner-ForkJoinPool-1-worker-23-1668284191961
+	 *  RUNNING clang++ -std=c++17 -I /Users/parrt/antlr/code/antlr4/runtime/Cpp/runtime/src -L. -lantlr4-runtime -pthread -o Test.out Test.cpp TLexer.cpp TParser.cpp TListener.cpp TBaseListener.cpp TVisitor.cpp TBaseVisitor.cpp in /var/folders/w1/_nr4stn13lq0rvjdkwh7q8cc0000gn/T/CppRunner-ForkJoinPool-1-worker-23-1668284191961
+	 */
+	public static final boolean WATCH_COMMANDS_EXEC = false;
+
 	public abstract String getLanguage();
 
 	protected String getExtension() { return getLanguage().toLowerCase(); }
@@ -170,7 +178,7 @@ public State run(RunOptions runOptions) {
 			return generatedState;
 		}
 
-		if (!initAntlrRuntimeIfRequired()) {
+		if (!initAntlrRuntimeIfRequired(runOptions)) {
 			// Do not repeat ANTLR runtime initialization error
 			return new CompiledState(generatedState, new Exception(getTitleName() + " ANTLR runtime is not initialized"));
 		}
@@ -223,7 +231,8 @@ protected void writeRecognizerFile(RunOptions runOptions) {
 		outputFileST.add("lexerName", runOptions.lexerName);
 		outputFileST.add("parserName", runOptions.parserName);
 		outputFileST.add("parserStartRuleName", grammarParseRuleToRecognizerName(runOptions.startRuleName));
-		outputFileST.add("debug", runOptions.showDiagnosticErrors);
+		outputFileST.add("showDiagnosticErrors", runOptions.showDiagnosticErrors);
+		outputFileST.add("traceATN", runOptions.traceATN);
 		outputFileST.add("profile", runOptions.profile);
 		outputFileST.add("showDFA", runOptions.showDFA);
 		outputFileST.add("useListener", runOptions.useListener);
@@ -238,7 +247,7 @@ protected String grammarParseRuleToRecognizerName(String startRuleName) {
 
 	protected void addExtraRecognizerParameters(ST template) {}
 
-	private boolean initAntlrRuntimeIfRequired() {
+	private boolean initAntlrRuntimeIfRequired(RunOptions runOptions) {
 		String language = getLanguage();
 		InitializationStatus status;
 
@@ -260,7 +269,7 @@ private boolean initAntlrRuntimeIfRequired() {
 			if (status.isInitialized == null) {
 				Exception exception = null;
 				try {
-					initRuntime();
+					initRuntime(runOptions);
 				} catch (Exception e) {
 					exception = e;
 					e.printStackTrace();
@@ -272,7 +281,7 @@ private boolean initAntlrRuntimeIfRequired() {
 		return status.isInitialized;
 	}
 
-	protected void initRuntime() throws Exception {
+	protected void initRuntime(RunOptions runOptions) throws Exception {
 	}
 
 	protected CompiledState compile(RunOptions runOptions, GeneratedState generatedState) {
@@ -298,7 +307,8 @@ protected ExecutedState execute(RunOptions runOptions, CompiledState compiledSta
 			ProcessorResult result = Processor.run(args.toArray(new String[0]), getTempDirPath(), getExecEnvironment());
 			output = result.output;
 			errors = result.errors;
-		} catch (InterruptedException | IOException e) {
+		}
+		catch (InterruptedException | IOException e) {
 			exception = e;
 		}
 		return new ExecutedState(compiledState, output, errors, exception);
@@ -309,10 +319,19 @@ protected ProcessorResult runCommand(String[] command, String workPath) throws E
 	}
 
 	protected ProcessorResult runCommand(String[] command, String workPath, String description) throws Exception {
+		String cmd = String.join(" ", command);
+		if ( WATCH_COMMANDS_EXEC ) {
+			System.out.println("RUNNING "+cmd+" in "+workPath);
+		}
 		try {
 			return Processor.run(command, workPath);
-		} catch (InterruptedException | IOException e) {
-			throw description != null ? new Exception("can't " + description, e) : e;
+		}
+		catch (InterruptedException | IOException e) {
+			String msg = "command \""+cmd+"\"\n  in "+workPath+" failed";
+			if ( description != null ) {
+				msg += ":\n  can't "+description;
+			}
+			throw new Exception(msg, e);
 		}
 	}
 
@@ -322,7 +341,8 @@ private void removeTempTestDirIfRequired() {
 			if (dirFile.exists()) {
 				try {
 					deleteDirectory(dirFile);
-				} catch (IOException e) {
+				}
+				catch (IOException e) {
 					e.printStackTrace();
 				}
 			}

runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestDescriptor.java

@@ -51,6 +51,8 @@ public class RuntimeTestDescriptor {
 	/** For parsing, engage the DiagnosticErrorListener, dumping results to stderr */
 	public final boolean showDiagnosticErrors;
 
+	public final boolean traceATN;
+
 	public final String[] skipTargets;
 
 	public final URI uri;
@@ -59,7 +61,7 @@ public RuntimeTestDescriptor(GrammarType testType, String name, String notes,
 								 String input, String output, String errors,
 								 String startRule,
 								 String grammarName, String grammar, List<Pair<String, String>> slaveGrammars,
-								 boolean showDFA, boolean showDiagnosticErrors, String[] skipTargets,
+								 boolean showDiagnosticErrors, boolean traceATN, boolean showDFA, String[] skipTargets,
 								 URI uri) {
 		this.testType = testType;
 		this.name = name;
@@ -73,6 +75,7 @@ public RuntimeTestDescriptor(GrammarType testType, String name, String notes,
 		this.slaveGrammars = slaveGrammars;
 		this.showDFA = showDFA;
 		this.showDiagnosticErrors = showDiagnosticErrors;
+		this.traceATN = traceATN;
 		this.skipTargets = skipTargets != null ? skipTargets : new String[0];
 		this.uri = uri;
 	}

runtime-testsuite/test/org/antlr/v4/test/runtime/RuntimeTestDescriptorParser.java

@@ -103,6 +103,7 @@ public static RuntimeTestDescriptor parse(String name, String text, URI uri) thr
 		String errors = "";
 		boolean showDFA = false;
 		boolean showDiagnosticErrors = false;
+		boolean traceATN = false;
 		String[] skipTargets = new String[0];
 		for (Pair<String,String> p : pairs) {
 			String section = p.a;
@@ -152,6 +153,9 @@ else if ( value.indexOf('\n')>=0 ) {
 							case "showDiagnosticErrors":
 								showDiagnosticErrors = true;
 								break;
+							case "traceATN":
+								traceATN = true;
+								break;
 						}
 					}
 					break;
@@ -163,7 +167,7 @@ else if ( value.indexOf('\n')>=0 ) {
 			}
 		}
 		return new RuntimeTestDescriptor(testType, name, notes, input, output, errors, startRule, grammarName, grammar,
-				slaveGrammars, showDFA, showDiagnosticErrors, skipTargets, uri);
+				slaveGrammars, showDiagnosticErrors, traceATN, showDFA, skipTargets, uri);
 	}
 
 	/** Get A, B, or C from: