[Strings] Represent string values as WTF-16 internally

CHANGELOG.md

@@ -21,6 +21,7 @@ Current Trunk
  - Add a new `BinaryenModuleReadWithFeatures` function to the C API that allows
    to configure which features to enable in the parser.
  - The build-time option to use legacy WasmGC opcodes is removed.
+ - The strings in `string.const` instructions must now be valid WTF-8.
 
 v117
 ----

scripts/fuzz_opt.py

@@ -333,9 +333,6 @@ def is_git_repo():
     'exception-handling.wast',
     'translate-to-new-eh.wast',
     'rse-eh.wast',
-    # Non-UTF8 strings trap in V8, and have limitations in our interpreter
-    'string-lowering.wast',
-    'precompute-strings.wast',
 ]
 
 

src/binaryen-c.cpp

@@ -26,6 +26,7 @@
 #include "pass.h"
 #include "shell-interface.h"
 #include "support/colors.h"
+#include "support/string.h"
 #include "wasm-binary.h"
 #include "wasm-builder.h"
 #include "wasm-interpreter.h"
@@ -1895,8 +1896,13 @@ BinaryenExpressionRef BinaryenStringNew(BinaryenModuleRef module,
 }
 BinaryenExpressionRef BinaryenStringConst(BinaryenModuleRef module,
                                           const char* name) {
+  // Re-encode from WTF-8 to WTF-16.
+  std::stringstream wtf16;
+  [[maybe_unused]] bool valid = String::convertWTF8ToWTF16(wtf16, name);
+  assert(valid);
+  // TODO: Use wtf16.view() once we have C++20.
   return static_cast<Expression*>(
-    Builder(*(Module*)module).makeStringConst(name));
+    Builder(*(Module*)module).makeStringConst(wtf16.str()));
 }
 BinaryenExpressionRef BinaryenStringMeasure(BinaryenModuleRef module,
                                             BinaryenOp op,

src/literal.h

@@ -85,7 +85,7 @@ class Literal {
     assert(type.isSignature());
   }
   explicit Literal(std::shared_ptr<GCData> gcData, HeapType type);
-  explicit Literal(std::string string);
+  explicit Literal(std::string_view string);
   Literal(const Literal& other);
   Literal& operator=(const Literal& other);
   ~Literal();

src/parser/contexts.h

@@ -22,6 +22,7 @@
 #include "lexer.h"
 #include "support/name.h"
 #include "support/result.h"
+#include "support/string.h"
 #include "wasm-builder.h"
 #include "wasm-ir-builder.h"
 #include "wasm.h"
@@ -2491,7 +2492,13 @@ struct ParseDefsCtx : TypeParserCtx<ParseDefsCtx> {
   Result<> makeStringConst(Index pos,
                            const std::vector<Annotation>& annotations,
                            std::string_view str) {
-    return withLoc(pos, irBuilder.makeStringConst(Name(str)));
+    // Re-encode from WTF-8 to WTF-16.
+    std::stringstream wtf16;
+    if (!String::convertWTF8ToWTF16(wtf16, str)) {
+      return in.err(pos, "invalid string constant");
+    }
+    // TODO: Use wtf16.view() once we have C++20.
+    return withLoc(pos, irBuilder.makeStringConst(wtf16.str()));
   }
 
   Result<> makeStringMeasure(Index pos,

src/parser/lexer.cpp

@@ -23,6 +23,7 @@
 #include <variant>
 
 #include "lexer.h"
+#include "support/string.h"
 
 using namespace std::string_view_literals;
 
@@ -308,25 +309,7 @@ struct LexStrCtx : LexCtx {
     if ((0xd800 <= u && u < 0xe000) || 0x110000 <= u) {
       return false;
     }
-    if (u < 0x80) {
-      // 0xxxxxxx
-      *escapeBuilder << uint8_t(u);
-    } else if (u < 0x800) {
-      // 110xxxxx 10xxxxxx
-      *escapeBuilder << uint8_t(0b11000000 | ((u >> 6) & 0b00011111));
-      *escapeBuilder << uint8_t(0b10000000 | ((u >> 0) & 0b00111111));
-    } else if (u < 0x10000) {
-      // 1110xxxx 10xxxxxx 10xxxxxx
-      *escapeBuilder << uint8_t(0b11100000 | ((u >> 12) & 0b00001111));
-      *escapeBuilder << uint8_t(0b10000000 | ((u >> 6) & 0b00111111));
-      *escapeBuilder << uint8_t(0b10000000 | ((u >> 0) & 0b00111111));
-    } else {
-      // 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
-      *escapeBuilder << uint8_t(0b11110000 | ((u >> 18) & 0b00000111));
-      *escapeBuilder << uint8_t(0b10000000 | ((u >> 12) & 0b00111111));
-      *escapeBuilder << uint8_t(0b10000000 | ((u >> 6) & 0b00111111));
-      *escapeBuilder << uint8_t(0b10000000 | ((u >> 0) & 0b00111111));
-    }
+    String::writeWTF8CodePoint(*escapeBuilder, u);
     return true;
   }
 };

src/passes/Print.cpp

@@ -2232,7 +2232,13 @@ struct PrintExpressionContents
   }
   void visitStringConst(StringConst* curr) {
     printMedium(o, "string.const ");
-    String::printEscaped(o, curr->string.str);
+    // Re-encode from WTF-16 to WTF-8.
+    std::stringstream wtf8;
+    [[maybe_unused]] bool valid =
+      String::convertWTF16ToWTF8(wtf8, curr->string.str);
+    assert(valid);
+    // TODO: Use wtf8.view() once we have C++20.
+    String::printEscaped(o, wtf8.str());
   }
   void visitStringMeasure(StringMeasure* curr) {
     switch (curr->op) {

src/passes/StringLowering.cpp

@@ -147,8 +147,14 @@ struct StringGathering : public Pass {
       }
 
       auto& string = strings[i];
+      // Re-encode from WTF-16 to WTF-8 to make the name easier to read.
+      std::stringstream wtf8;
+      [[maybe_unused]] bool valid =
+        String::convertWTF16ToWTF8(wtf8, string.str);
+      assert(valid);
+      // TODO: Use wtf8.view() once we have C++20.
       auto name = Names::getValidGlobalName(
-        *module, std::string("string.const_") + std::string(string.str));
+        *module, std::string("string.const_") + std::string(wtf8.str()));
       globalName = name;
       newNames.insert(name);
       auto* stringConst = builder.makeStringConst(string);

src/support/json.cpp

@@ -21,7 +21,12 @@ namespace json {
 
 void Value::stringify(std::ostream& os, bool pretty) {
   if (isString()) {
-    wasm::String::printEscapedJSON(os, getCString());
+    std::stringstream wtf16;
+    [[maybe_unused]] bool valid =
+      wasm::String::convertWTF8ToWTF16(wtf16, getIString().str);
+    assert(valid);
+    // TODO: Use wtf16.view() once we have C++20.
+    wasm::String::printEscapedJSON(os, wtf16.str());
   } else if (isArray()) {
     os << '[';
     auto first = true;