Merge branch 'master' into support_intel_compiler

Author: parrt

.travis/before-install-osx-dotnet.sh

@@ -3,9 +3,9 @@
 set -euo pipefail
 
 cache_dir="$HOME/Library/Caches/Antlr4"
-dotnet_url='https://download.microsoft.com/download/B/9/F/B9F1AF57-C14A-4670-9973-CDF47209B5BF/dotnet-dev-osx-x64.1.0.4.pkg'
+dotnet_url='https://download.microsoft.com/download/F/4/F/F4FCB6EC-5F05-4DF8-822C-FF013DF1B17F/dotnet-dev-osx-x64.1.1.4.pkg'
 dotnet_file=$(basename "$dotnet_url")
-dotnet_shasum='63b5d99028cd8b2454736076106c96ba7d05f0fc'
+dotnet_shasum='dc46d93716db8bea8cc3c668088cc9e39384b5a4'
 
 thisdir=$(dirname "$0")
 

appveyor.yml

@@ -2,10 +2,16 @@ version: '4.7.1-SNAPSHOT+AppVeyor.{build}'
 cache:
   - '%USERPROFILE%\.m2'
   - '%USERPROFILE%\.nuget\packages -> **\project.json'
+image: Visual Studio 2017  
 build: off
 build_script:
   - mvn -DskipTests install --batch-mode
-  - msbuild runtime/CSharp/runtime/CSharp/Antlr4.vs2013.sln /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" /verbosity:detailed
+  - msbuild /target:restore /target:rebuild /property:Configuration=Release /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" /verbosity:detailed runtime/CSharp/runtime/CSharp/Antlr4.dotnet.sln
   - msbuild ./runtime-testsuite/target/classes/CSharp/runtime/CSharp/Antlr4.vs2013.sln /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" /verbosity:detailed
+after_build: 
+  - msbuild /target:pack /property:Configuration=Release /verbosity:detailed runtime/CSharp/runtime/CSharp/Antlr4.dotnet.sln
 test_script:
   - mvn install -Dantlr-python2-python="C:\Python27\python.exe" -Dantlr-python3-python="C:\Python35\python.exe" -Dantlr-javascript-nodejs="C:\Program Files (x86)\nodejs\node.exe" --batch-mode
+artifacts:
+- path: 'runtime\**\*.nupkg'  
+  name: NuGet

contributors.txt

@@ -165,10 +165,12 @@ YYYY/MM/DD, github id, Full name, email
 2017/08/29, Eddy Reyes, [email protected]
 2017/09/09, brauliobz, Bráulio Bezerra, [email protected]
 2017/09/11, sachinjain024, Sachin Jain, [email protected]
+2017/09/25, kaedvann, Rostislav Listerenko, [email protected]
 2017/10/06, bramp, Andrew Brampton, [email protected]
 2017/10/15, simkimsia, Sim Kim Sia, [email protected]
 2017/10/27, Griffon26, Maurice van der Pot, [email protected]
 2017/05/29, rlfnb, Ralf Neeb, [email protected]
 2017/10/29, gendalph, Максим Прохоренко, Maxim\dotProhorenko@[email protected]
 2017/11/02, jasonmoo, Jason Mooberry, [email protected]
 2017/11/05, ajaypanyala, Ajay Panyala, [email protected]
+2017/11/24, zqlu.cn, Zhiqiang Lu, [email protected]

doc/interpreters.md

@@ -1,8 +1,14 @@
-# Parser and lexer interpreters
+# Parser and Lexer Interpreters
 
 *Since ANTLR 4.2*
 
-For small parsing tasks it is sometimes convenient to use ANTLR in interpreted mode, rather than generating a parser in a particular target, compiling it and running it as part of your application. Here's some sample code that creates lexer and parser Grammar objects and then creates interpreters. Once we have a ParserInterpreter, we can use it to parse starting in any rule we like, given a rule index (which the Grammar can provide).
+For small parsing tasks it is sometimes convenient to use ANTLR in interpreted mode, rather than generating a parser in a particular target, compiling it and running it as part of your application. Here's some sample code that creates lexer and parser Grammar objects and then creates interpreters. Once we have a ParserInterpreter, we can use it to parse starting in any rule we like, given a rule index (which the grammar + the parser can provide).
+
+## Action Code
+
+Since interpreters don't use generated parsers + lexers they cannot execute any action code (including predicates). That means the interpreter runs as if there were no predicates at all. If your grammar requires action code in order to parse correctly you will not be able to test it using this approach.
+
+## Java Target Interpreter Setup
 
 ```java
 LexerGrammar lg = new LexerGrammar(
@@ -76,4 +82,49 @@ ParseTree t = parse(fileName, XMLLexerGrammar, XMLParserGrammar, "document");
 
 This is also how we will integrate instantaneous parsing into ANTLRWorks2 and development environment plug-ins.
 
-See [TestParserInterpreter.java](https://github.com/antlr/antlr4/blob/master/tool-testsuite/test/org/antlr/v4/test/tool/TestParserInterpreter.java).
+See [TestParserInterpreter.java](../tool-testsuite/test/org/antlr/v4/test/tool/TestParserInterpreter.java).
+
+## Non-Java Target Interpreter Setup
+The ANTLR4 runtimes do not contain any grammar parsing classes (they are in the ANTLR4 tool  jar). Hence we cannot use `LexerGrammar` and `Grammar` to parse grammars for the interpreter. Instead we directly instantiate `LexerInterpreter` and `ParserInterpreter` objects. They require some data (namely symbol information and the ATNs) which only the ANTLR4 tool can give us. However, on each generation run ANTLR not only produces your parser + lexer files but also interpreter data files (*.interp) which contain all you need to feed the interpreters.
+
+A support class (`InterpreterDataReader`) is used to load the data for your convenience, which makes this very easy to use. Btw. even the Java target go this route instead of using the non-runtime classes `Grammar` and `LexerGrammar`. Sometimes it might not be feasible to use the tool jar for whatever reason.
+
+Here's how the setup looks like (C++ example):
+
+```cpp
+/**
+ * sourceFileName - name of the file with content to parse
+ * lexerName - the name of your lexer (arbitrary, that's what is used in error messages)
+ * parserName - ditto for the parser
+ * lexerDataFileName - the lexer interpeter data file name (e.g. `<path>/ExprLexer.interp`)
+ * parserDataFileName - ditto for the parser (e.g. `<path>/Expr.interp`)
+ * startRule - the name of the rule to start parsing at
+ */
+void parse(std::string const& sourceFileName,
+  std::string const& lexerName, std::string const& parserName,
+  std::string const& lexerDataFileName, std::string const& parserDataFileName,
+  std::string const& startRule) {
+  
+    InterpreterData lexerData = InterpreterDataReader::parseFile(lexerDataFileName);
+    InterpreterData parserData = InterpreterDataReader::parseFile(parserDataFileName);
+
+    ANTLRFileStream input(sourceFileName);
+    LexerInterpreter lexEngine(lexerName, lexerData.vocabulary, lexerData.ruleNames,
+      lexerData.channels, lexerData.modes, lexerData.atn, &input);
+    CommonTokenStream tokens(&lexEngine);
+
+    /* Remove comment to print the tokens.
+    tokens.fill();
+    std::cout << "INPUT:" << std::endl;
+    for (auto token : tokens.getTokens()) {
+      std::cout << token->toString() << std::endl;
+    }
+    */
+
+    ParserInterpreter parser(parserName, parserData.vocabulary, parserData.ruleNames,
+      parserData.atn, &tokens);
+    tree::ParseTree *tree = parser.parse(parser.getRuleIndex(startRule));
+
+    std::cout << "parse tree: " << tree->toStringTree(&parser) << std::endl;
+}
+```

doc/releasing-antlr.md

@@ -32,8 +32,7 @@ Edit the repository looking for 4.5 or whatever and update it. Bump version in t
  * runtime/Python3/setup.py
  * runtime/Python3/src/antlr4/Recognizer.py
  * runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Properties/AssemblyInfo.cs
- * runtime/CSharp/build/version.ps1
- * runtime/CSharp/runtime/CSharp/Package.nuspec
+ * runtime/CSharp/runtime/CSharp/Antlr4.Runtime/Antlr4.Runtime.dotnet.csproj
  * runtime/JavaScript/src/antlr4/package.json
  * runtime/JavaScript/src/antlr4/Recognizer.js
  * runtime/Cpp/VERSION
@@ -245,75 +244,55 @@ popd
 
 ### CSharp
 
-*Publishing to Nuget from Linux/MacOSX*
+*Publishing to Nuget from Windows*
 
 **Install the pre-requisites**
 
 Of course you need Mono and `nuget` to be installed. On mac:
 
-- mono - on mac, `brew install mono`
-- nuget - on mac, `brew install nuget` or you can [download nuget.exe](https://dist.nuget.org/win-x86-commandline/latest/nuget.exe)
+- .NET build tools - can be loaded from [here](https://www.visualstudio.com/downloads/)
+- nuget - download [nuget.exe](https://www.nuget.org/downloads)
 - dotnet - follow [the instructions here](https://www.microsoft.com/net/core)
 
-From the shell on mac, you can check all is ok by typing
+Alternatively, you can install Visual Studio 2017 and make sure to check boxes with .NET Core SDK.
 
-```bash
-nuget
+You also need to enable .NET Framework 3.5 support in Windows "Programs and Features".
+
+If everything is ok, the following command will restore nuget packages, build Antlr for .NET Standard and .NET 3.5 and create nuget package:
+
+```PS
+msbuild /target:restore /target:rebuild /target:pack /property:Configuration=Release .\Antlr4.dotnet.sln /verbosity:minimal
 ```
 
-This should display the nuget help. 
+This should display something like this: 
 
 **Creating and packaging the assembly**
 
-```bash
-$ cd runtime/CSharp/runtime/CSharp
-$ ./build-nuget-package.sh
-...
-Build succeeded.
-    0 Warning(s)
-    0 Error(s)
-Attempting to build package from 'Package.nuspec'.
-Successfully created package '/path/to/antlr/.../Antlr4.Runtime.Standard.4.7.0.nupkg'.
 ```
+Microsoft (R) Build Engine version 15.4.8.50001 for .NET Framework
+Copyright (C) Microsoft Corporation. All rights reserved.
 
-This should display: Successfully created package *<package-path>*
-
-Alternately, you may want to build ANTLR using Xamarin Studio Community (free).
+  Restoring packages for C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\Antlr4.Runtime.dotnet.csproj...
+  Generating MSBuild file C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\obj\Antlr4.Runtime.dotnet.csproj.nuget.g.props.
+  Generating MSBuild file C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\obj\Antlr4.Runtime.dotnet.csproj.nuget.g.targets.
+  Restore completed in 427.62 ms for C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\Antlr4.Runtime.dotnet.csproj.
+  Antlr4.Runtime.dotnet -> C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\netstandard1.3\Antlr4.Runtime.Standard.dll
+  Antlr4.Runtime.dotnet -> C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\net35\Antlr4.Runtime.Standard.dll
+  Successfully created package 'C:\Code\antlr4-fork\runtime\CSharp\runtime\CSharp\Antlr4.Runtime\lib\Release\Antlr4.Runtime.Standard.4.7.2.nupkg'.
+```
 
 **Publishing to NuGet**
 
 You need to be a NuGet owner for "ANTLR 4 Standard Runtime"
-As a registered NuGet user, you can then manually upload the package spec here (`runtime/CSharp/runtime/CSharp/Package.nuspec`): [https://www.nuget.org/packages/manage/upload](https://www.nuget.org/packages/manage/upload)
+As a registered NuGet user, you can then manually upload the package here: [https://www.nuget.org/packages/manage/upload](https://www.nuget.org/packages/manage/upload)
 
 Alternately, you can publish from the cmd line. You need to get your NuGet key from [https://www.nuget.org/account#](https://www.nuget.org/account#) and then from the cmd line, you can then type:
 
-```bash
+```cmd
 nuget push Antlr4.Runtime.Standard.<version>.nupkg <your-key> -Source https://www.nuget.org/api/v2/package
 ```
 
-**Creating DLLs**
-
-```bash
-cd ~/antlr/code/antlr4/runtime/CSharp/runtime/CSharp
-# kill previous ones manually as "xbuild /t:Clean" didn't seem to do it
-rm Antlr4.Runtime/bin/net20/Release/Antlr4.Runtime.dll
-rm Antlr4.Runtime/obj/net20/Release/Antlr4.Runtime.dll
-# build
-xbuild /p:Configuration=Release Antlr4.Runtime/Antlr4.Runtime.mono.csproj
-# zip it up to get a version number on zip filename
-zip --junk-paths /tmp/antlr-csharp-runtime-4.7.zip Antlr4.Runtime/obj/net20/Release/Antlr4.Runtime.Standard.dll
-cp /tmp/antlr-csharp-runtime-4.7.zip ~/antlr/sites/website-antlr4/download
-```
-
-Move target to website
-
-```bash
-pushd ~/antlr/sites/website-antlr4/download
-git add antlr-csharp-runtime-4.7.zip
-git commit -a -m 'update C# runtime'
-git push origin gh-pages
-popd
-```
+Nuget packages are also accessible as artifacts of [AppVeyor builds](https://ci.appveyor.com/project/parrt/antlr4/build/artifacts). 
 
 ### Python
 

runtime-testsuite/test/org/antlr/v4/test/runtime/csharp/BaseCSharpTest.java

@@ -10,6 +10,7 @@
 import org.antlr.v4.runtime.Token;
 import org.antlr.v4.runtime.TokenSource;
 import org.antlr.v4.runtime.WritableToken;
+import org.antlr.v4.runtime.misc.Utils;
 import org.antlr.v4.test.runtime.ErrorQueue;
 import org.antlr.v4.test.runtime.RuntimeTestSupport;
 import org.antlr.v4.test.runtime.StreamVacuum;
@@ -337,7 +338,9 @@ protected String rawExecRecognizer(String parserName,
 	}
 
 	public String execRecognizer() {
-		compile();
+		boolean success = compile();
+		assertTrue(success);
+
 		String output = execTest();
 		if ( output!=null && output.length()==0 ) {
 			output = null;
@@ -354,6 +357,7 @@ public boolean compile() {
                     return false;
                 return true;
             } catch(Exception e) {
+                e.printStackTrace(System.err);
                 return false;
             }
         }
@@ -362,6 +366,7 @@ public boolean compile() {
             try {
                 return buildDotnetProject();
             } catch(Exception e) {
+                e.printStackTrace(System.err);
                 return false;
             }
         }
@@ -390,10 +395,15 @@ private boolean buildProject() throws Exception {
 		stdoutVacuum.join();
 		stderrVacuum.join();
 		// xbuild sends errors to output, so check exit code
-		boolean success = process.exitValue()==0;
+		int exitValue = process.exitValue();
+		boolean success = (exitValue == 0);
 		if ( !success ) {
 			this.stderrDuringParse = stdoutVacuum.toString();
-			System.err.println("buildProject stderrVacuum: "+ this.stderrDuringParse);
+			String stderrString = stderrVacuum.toString();
+			System.err.println("buildProject command: " + Utils.join(args, " "));
+			System.err.println("buildProject exitValue: " + exitValue);
+			System.err.println("buildProject stdout: " + stderrDuringParse);
+			System.err.println("buildProject stderr: " + stderrString);
 		}
 		return success;
 	}
@@ -505,6 +515,7 @@ public boolean buildDotnetProject() {
                 runtimeProjPath
             };
             boolean success = runProcess(args, tmpdir);
+            assertTrue(success);
 
             // restore project
             args = new String[] {
@@ -514,6 +525,7 @@ public boolean buildDotnetProject() {
                 "--no-dependencies"
             };
             success = runProcess(args, tmpdir);
+            assertTrue(success);
 
             // build test
             args = new String[] {
@@ -525,6 +537,7 @@ public boolean buildDotnetProject() {
                 "--no-dependencies"
             };
             success = runProcess(args, tmpdir);
+            assertTrue(success);
         }
         catch(Exception e) {
             e.printStackTrace(System.err);
@@ -535,6 +548,10 @@ public boolean buildDotnetProject() {
     }
 
     private boolean runProcess(String[] args, String path) throws Exception {
+        return runProcess(args, path, 0);
+    }
+
+    private boolean runProcess(String[] args, String path, int retries) throws Exception {
         ProcessBuilder pb = new ProcessBuilder(args);
         pb.directory(new File(path));
         Process process = pb.start();
@@ -545,10 +562,28 @@ private boolean runProcess(String[] args, String path) throws Exception {
         process.waitFor();
         stdoutVacuum.join();
         stderrVacuum.join();
-        boolean success = process.exitValue()==0;
+        int exitValue = process.exitValue();
+        boolean success = (exitValue == 0);
         if ( !success ) {
             this.stderrDuringParse = stderrVacuum.toString();
-            System.err.println("runProcess stderrVacuum: "+ this.stderrDuringParse);
+            System.err.println("runProcess command: " + Utils.join(args, " "));
+            System.err.println("runProcess exitValue: " + exitValue);
+            System.err.println("runProcess stdoutVacuum: " + stdoutVacuum.toString());
+            System.err.println("runProcess stderrVacuum: " + stderrDuringParse);
+        }
+        if (exitValue == 132) {
+            // Retry after SIGILL.  We are seeing this intermittently on
+            // macOS (issue #2078).
+            if (retries < 3) {
+                System.err.println("runProcess retrying; " + retries +
+                                   " retries so far");
+                 return runProcess(args, path, retries + 1);
+            }
+            else {
+                System.err.println("runProcess giving up after " + retries +
+                                   " retries");
+                return false;
+            }
         }
         return success;
     }
@@ -577,9 +612,28 @@ public String execTest() {
 			ProcessBuilder pb = new ProcessBuilder(args);
 			pb.directory(tmpdirFile);
 			Process process = pb.start();
+			StreamVacuum stdoutVacuum = new StreamVacuum(process.getInputStream());
+			StreamVacuum stderrVacuum = new StreamVacuum(process.getErrorStream());
+			stdoutVacuum.start();
+			stderrVacuum.start();
 			process.waitFor();
+			stdoutVacuum.join();
+			stderrVacuum.join();
 			String writtenOutput = TestOutputReading.read(output);
 			this.stderrDuringParse = TestOutputReading.read(errorOutput);
+			int exitValue = process.exitValue();
+			String stdoutString = stdoutVacuum.toString().trim();
+			String stderrString = stderrVacuum.toString().trim();
+			if (exitValue != 0) {
+				System.err.println("execTest command: " + Utils.join(args, " "));
+				System.err.println("execTest exitValue: " + exitValue);
+			}
+			if (!stdoutString.isEmpty()) {
+				System.err.println("execTest stdoutVacuum: " + stdoutString);
+			}
+			if (!stderrString.isEmpty()) {
+				System.err.println("execTest stderrVacuum: " + stderrString);
+			}
 			return writtenOutput;
 		}
 		catch (Exception e) {

runtime-testsuite/test/org/antlr/v4/test/runtime/swift/BaseSwiftTest.java

@@ -281,7 +281,7 @@ private void writeParserTestFile(String parserName,
 						"\n" +
 						"do {\n" +
 						"let args = CommandLine.arguments\n" +
-						"let input = ANTLRFileStream(args[1])\n" +
+						"let input = try ANTLRFileStream(args[1])\n" +
 						"let lex = <lexerName>(input)\n" +
 						"let tokens = CommonTokenStream(lex)\n" +
 						"<createParser>\n" +
@@ -327,7 +327,7 @@ private void writeLexerTestFile(String lexerName, boolean showDFA) {
 
 						"setbuf(stdout, nil)\n" +
 						"let args = CommandLine.arguments\n" +
-						"let input = ANTLRFileStream(args[1])\n" +
+						"let input = try ANTLRFileStream(args[1])\n" +
 						"let lex = <lexerName>(input)\n" +
 						"let tokens = CommonTokenStream(lex)\n" +