Fix JVM crash when running java (fixes #216) (#265)

Author: gnodet

src/main/java/org/fusesource/jansi/AnsiMain.java

@@ -95,6 +95,7 @@ public static void main(String... args) throws IOException {
         System.out.println("java.version= " + System.getProperty("java.version") + ", "
                 + "java.vendor= " + System.getProperty("java.vendor") + ","
                 + " java.home= " + System.getProperty("java.home"));
+        System.out.println("Console: " + System.console());
 
         System.out.println();
 

src/main/native/jansi_isatty.c

@@ -57,8 +57,8 @@ JNIEXPORT jint JNICALL CLibrary_NATIVE(isatty)
 
 	/* check if fd is a pipe */
 	HANDLE h = (HANDLE) _get_osfhandle(arg0);
-	DWORD t = GetFileType(h);
-	if (t == FILE_TYPE_CHAR) {
+	DWORD t = h != NULL ? GetFileType(h) : 0;
+	if (h != NULL && t == FILE_TYPE_CHAR) {
 	    // check that this is a real tty because the /dev/null
 	    // and /dev/zero streams are also of type FILE_TYPE_CHAR
 		rc = GetConsoleMode(h, &mode) != 0;
@@ -84,20 +84,25 @@ JNIEXPORT jint JNICALL CLibrary_NATIVE(isatty)
 			else {
 
 				name = nameinfo->Name.Buffer;
-				name[nameinfo->Name.Length / 2] = 0;
-
-				//fprintf( stderr, "Standard stream %d: pipe name: %S\n", arg0, name);
-
-				/*
-				 * Check if this could be a MSYS2 pty pipe ('msys-XXXX-ptyN-XX')
-				 * or a cygwin pty pipe ('cygwin-XXXX-ptyN-XX')
-				 */
-				if ((wcsstr(name, L"msys-") || wcsstr(name, L"cygwin-")) && wcsstr(name, L"-pty")) {
-					rc = 1;
-				} else {
-					// This is definitely not a tty
-					rc = 0;
+				if (name == NULL) {
+				    rc = 0;
 				}
+				else {
+                    name[nameinfo->Name.Length / 2] = 0;
+
+                    //fprintf( stderr, "Standard stream %d: pipe name: %S\n", arg0, name);
+
+                    /*
+                     * Check if this could be a MSYS2 pty pipe ('msys-XXXX-ptyN-XX')
+                     * or a cygwin pty pipe ('cygwin-XXXX-ptyN-XX')
+                     */
+                    if ((wcsstr(name, L"msys-") || wcsstr(name, L"cygwin-")) && wcsstr(name, L"-pty")) {
+                        rc = 1;
+                    } else {
+                        // This is definitely not a tty
+                        rc = 0;
+                    }
+                }
 			}
 		}
 	}

src/main/resources/org/fusesource/jansi/internal/native/Windows/arm64/libjansi.so

@@ -15,8 +15,16 @@
  */
 package org.fusesource.jansi;
 
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
 import org.fusesource.jansi.Ansi.Color;
+import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.EnabledOnOs;
+import org.junit.jupiter.api.condition.OS;
 import org.junit.jupiter.params.ParameterizedTest;
 import org.junit.jupiter.params.provider.CsvSource;
 
@@ -181,6 +189,34 @@ public void testColorDisabled() {
         }
     }
 
+    @Test
+    @EnabledOnOs(OS.WINDOWS)
+    @Disabled("Does not really fail: launch `javaw -jar jansi-xxx.jar` directly instead")
+    public void testAnsiMainWithNoConsole() throws Exception {
+        Path javaHome = Paths.get(System.getProperty("java.home"));
+        Path java = javaHome.resolve("bin\\javaw.exe");
+        String cp = System.getProperty("java.class.path");
+
+        Process process = new ProcessBuilder()
+                .command(java.toString(), "-cp", cp, AnsiMain.class.getName())
+                .start();
+
+        ByteArrayOutputStream baos = new ByteArrayOutputStream();
+        try (InputStream in = process.getInputStream()) {
+            byte[] buffer = new byte[8192];
+            while (true) {
+                int nb = in.read(buffer);
+                if (nb > 0) {
+                    baos.write(buffer, 0, nb);
+                } else {
+                    break;
+                }
+            }
+        }
+
+        assertTrue(baos.toString().contains("test on System.out"), baos.toString());
+    }
+
     private static void assertAnsi(String expected, Ansi actual) {
         assertEquals(expected.replace("ESC", "\033"), actual.toString());
     }