Merge pull request #3025 from niw/fix_swift_input_stream

Fix Swift runtime ANTLRInputStream that can’t read Unicode scalars

Author: parrt

contributors.txt

@@ -286,6 +286,7 @@ YYYY/MM/DD, github id, Full name, email
 2020/11/26, mr-c, Michael R. Crusoe, [email protected]
 2020/12/01, maxence-lefebvre, Maxence Lefebvre, [email protected]
 2020/12/03, electrum, David Phillips, [email protected]
+2021/01/03, niw, Yoshimasa Niwa, [email protected]
 2021/01/25, l215884529, Qiheng Liu, [email protected]
 2021/02/02, tsotnikov, Taras Sotnikov, [email protected]
 2021/02/10, jirislaby, Jiri Slaby, [email protected]

runtime/Swift/Sources/Antlr4/ANTLRFileStream.swift

@@ -13,10 +13,9 @@ public class ANTLRFileStream: ANTLRInputStream {
 
     public init(_ fileName: String, _ encoding: String.Encoding? = nil) throws {
         self.fileName = fileName
-        super.init()
         let fileContents = try String(contentsOfFile: fileName, encoding: encoding ?? .utf8)
-        data = Array(fileContents)
-        n = data.count
+        let data = Array(fileContents.unicodeScalars)
+        super.init(data, data.count)
     }
 
     override

runtime/Swift/Sources/Antlr4/ANTLRInputStream.swift

@@ -15,10 +15,10 @@ public class ANTLRInputStream: CharStream {
     ///
     /// The data being scanned
     /// 
-    internal var data: [Character]
+    internal let data: [UnicodeScalar]
 
     /// 
-    /// How many characters are actually in the buffer
+    /// How many unicode scalars are actually in the buffer
     /// 
     internal var n: Int
 
@@ -34,23 +34,33 @@ public class ANTLRInputStream: CharStream {
 
     public init() {
         n = 0
-        data = [Character]()
+        data = []
     }
 
     /// 
     /// Copy data in string to a local char array
     /// 
     public init(_ input: String) {
-        self.data = Array(input)
+        self.data = Array(input.unicodeScalars)
         self.n = data.count
     }
 
     /// 
     /// This is the preferred constructor for strings as no data is copied
     /// 
-    public init(_ data: [Character], _ numberOfActualCharsInArray: Int) {
+    public init(_ data: [UnicodeScalar], _ numberOfActualUnicodeScalarsInArray: Int) {
         self.data = data
-        self.n = numberOfActualCharsInArray
+        self.n = numberOfActualUnicodeScalarsInArray
+    }
+
+    ///
+    /// This is only for backward compatibility that accepts array of `Character`.
+    /// Use `init(_ data: [UnicodeScalar], _ numberOfActualUnicodeScalarsInArray: Int)` instead.
+    ///
+    public init(_ data: [Character], _ numberOfActualUnicodeScalarsInArray: Int) {
+        let string = String(data)
+        self.data = Array(string.unicodeScalars)
+        self.n = numberOfActualUnicodeScalarsInArray
     }
 
     public func reset() {
@@ -90,7 +100,7 @@ public class ANTLRInputStream: CharStream {
         }
         //print("char LA("+i+")="+(char)data[p+i-1]+"; p="+p);
         //print("LA("+i+"); p="+p+" n="+n+" data.length="+data.length);
-        return data[p + i - 1].unicodeValue
+        return Int(data[p + i - 1].value)
     }
 
     public func LT(_ i: Int) -> Int {
@@ -145,14 +155,19 @@ public class ANTLRInputStream: CharStream {
             return ""
         }
         let stop = min(n, interval.b + 1)
-        return String(data[start ..< stop])
+
+        var unicodeScalarView = String.UnicodeScalarView()
+        unicodeScalarView.append(contentsOf: data[start ..< stop])
+        return String(unicodeScalarView)
     }
 
     public func getSourceName() -> String {
         return name ?? ANTLRInputStream.UNKNOWN_SOURCE_NAME
     }
 
     public func toString() -> String {
-        return String(data)
+        var unicodeScalarView = String.UnicodeScalarView()
+        unicodeScalarView.append(contentsOf: data)
+        return String(unicodeScalarView)
     }
 }

runtime/Swift/Tests/Antlr4Tests/ANTLRInputStreamTests.swift

@@ -0,0 +1,45 @@
+/// Copyright (c) 2012-2017 The ANTLR Project. All rights reserved.
+/// Use of this file is governed by the BSD 3-clause license that
+/// can be found in the LICENSE.txt file in the project root.
+
+import Foundation
+
+import XCTest
+import Antlr4
+
+class ANTLRInputStreamTests: XCTestCase {
+    func testASCIICharactersString() {
+        let inputStream = ANTLRInputStream("Cat")
+        XCTAssertEqual(inputStream.LA(1), 0x0043)
+        XCTAssertEqual(inputStream.LA(2), 0x0061)
+        XCTAssertEqual(inputStream.LA(3), 0x0074)
+    }
+
+    func testBasicMultilingualPlaneCharactersString() {
+        // Three Japanese hiragana characters.
+        let inputStream = ANTLRInputStream("\u{3053}\u{306D}\u{3053}")
+        XCTAssertEqual(inputStream.LA(1), 0x3053)
+        XCTAssertEqual(inputStream.LA(2), 0x306D)
+        XCTAssertEqual(inputStream.LA(3), 0x3053)
+    }
+
+    func testSupplementaryMultilingualPlaneCharactersString() {
+        // Three "Cat", "Cat Face", and "Grinning Cat with Smiling Eyes" emojis
+        let inputStream = ANTLRInputStream("\u{1F408}\u{1F431}\u{1F638}")
+        XCTAssertEqual(inputStream.LA(1), 0x1F408)
+        XCTAssertEqual(inputStream.LA(2), 0x1F431)
+        XCTAssertEqual(inputStream.LA(3), 0x1F638)
+    }
+
+    func testGraphemeCharactersString() {
+        // One "Family (Man, Woman, Girl, Boy)" emoji
+        let inputStream = ANTLRInputStream("\u{1F468}\u{200D}\u{1F469}\u{200D}\u{1F467}\u{200D}\u{1F466}")
+        XCTAssertEqual(inputStream.LA(1), 0x1F468)
+        XCTAssertEqual(inputStream.LA(2), 0x200D)
+        XCTAssertEqual(inputStream.LA(3), 0x1F469)
+        XCTAssertEqual(inputStream.LA(4), 0x200D)
+        XCTAssertEqual(inputStream.LA(5), 0x1F467)
+        XCTAssertEqual(inputStream.LA(6), 0x200D)
+        XCTAssertEqual(inputStream.LA(7), 0x1F466)
+    }
+}