Revert "src: support UTF-8 in compiled-in JS source files"

This reverts commit 3380cd5.

We should fix Electron's build to not revert this commit in future.

Author: zcbenz

node.gyp

@@ -149,7 +149,7 @@
         'src',
         'tools/msvs/genfiles',
         'deps/uv/src/ares',
-        '<(SHARED_INTERMEDIATE_DIR)',
+        '<(SHARED_INTERMEDIATE_DIR)', # for node_natives.h
       ],
 
       'sources': [
@@ -174,6 +174,7 @@
         'src/node_debug_options.cc',
         'src/node_file.cc',
         'src/node_http_parser.cc',
+        'src/node_javascript.cc',
         'src/node_main.cc',
         'src/node_os.cc',
         'src/node_revert.cc',
@@ -245,11 +246,11 @@
         'deps/http_parser/http_parser.h',
         'deps/v8/include/v8.h',
         'deps/v8/include/v8-debug.h',
+        '<(SHARED_INTERMEDIATE_DIR)/node_natives.h',
         # javascript files to make for an even more pleasant IDE experience
         '<@(library_files)',
         # node.gyp is added to the project by default.
         'common.gypi',
-        '<(SHARED_INTERMEDIATE_DIR)/node_javascript.cc',
       ],
 
       'defines': [
@@ -722,13 +723,12 @@
       'actions': [
         {
           'action_name': 'node_js2c',
-          'process_outputs_as_sources': 1,
           'inputs': [
             '<@(library_files)',
             './config.gypi',
           ],
           'outputs': [
-            '<(SHARED_INTERMEDIATE_DIR)/node_javascript.cc',
+            '<(SHARED_INTERMEDIATE_DIR)/node_natives.h',
           ],
           'conditions': [
             [ 'node_use_dtrace=="false" and node_use_etw=="false"', {

src/node_javascript.cc

@@ -0,0 +1,51 @@
+#include "node.h"
+#include "node_natives.h"
+#include "v8.h"
+#include "env.h"
+#include "env-inl.h"
+
+namespace node {
+
+using v8::Local;
+using v8::NewStringType;
+using v8::Object;
+using v8::String;
+
+// id##_data is defined in node_natives.h.
+#define V(id)                                                                 \
+  static struct : public String::ExternalOneByteStringResource {              \
+    const char* data() const override {                                       \
+      return reinterpret_cast<const char*>(id##_data);                        \
+    }                                                                         \
+    size_t length() const override { return sizeof(id##_data); }              \
+    void Dispose() override { /* Default calls `delete this`. */ }            \
+  } id##_external_data;
+NODE_NATIVES_MAP(V)
+#undef V
+
+Local<String> MainSource(Environment* env) {
+  auto maybe_string =
+      String::NewExternalOneByte(
+          env->isolate(),
+          &internal_bootstrap_node_external_data);
+  return maybe_string.ToLocalChecked();
+}
+
+void DefineJavaScript(Environment* env, Local<Object> target) {
+  auto context = env->context();
+#define V(id)                                                                 \
+  do {                                                                        \
+    auto key =                                                                \
+        String::NewFromOneByte(                                               \
+            env->isolate(), id##_name, NewStringType::kNormal,                \
+            sizeof(id##_name)).ToLocalChecked();                              \
+    auto value =                                                              \
+        String::NewExternalOneByte(                                           \
+            env->isolate(), &id##_external_data).ToLocalChecked();            \
+    CHECK(target->Set(context, key, value).FromJust());                       \
+  } while (0);
+  NODE_NATIVES_MAP(V)
+#undef V
+}
+
+}  // namespace node

tools/js2c.py

@@ -37,14 +37,11 @@
 import string
 
 
-def ToCArray(elements, step=10):
-  slices = (elements[i:i+step] for i in xrange(0, len(elements), step))
-  slices = map(lambda s: ','.join(str(x) for x in s), slices)
-  return ',\n'.join(slices)
-
-
 def ToCString(contents):
-  return ToCArray(map(ord, contents), step=20)
+  step = 20
+  slices = (contents[i:i+step] for i in xrange(0, len(contents), step))
+  slices = map(lambda s: ','.join(str(ord(c)) for c in s), slices)
+  return ',\n'.join(slices)
 
 
 def ReadFile(filename):
@@ -164,72 +161,34 @@ def ReadMacros(lines):
   return (constants, macros)
 
 
-TEMPLATE = """
-#include "node.h"
-#include "node_javascript.h"
-#include "v8.h"
-#include "env.h"
-#include "env-inl.h"
-
-namespace node {{
-
-{definitions}
+HEADER_TEMPLATE = """\
+#ifndef NODE_NATIVES_H_
+#define NODE_NATIVES_H_
 
-v8::Local<v8::String> MainSource(Environment* env) {{
-  return internal_bootstrap_node_value.ToStringChecked(env->isolate());
-}}
+#include <stdint.h>
 
-void DefineJavaScript(Environment* env, v8::Local<v8::Object> target) {{
-  {initializers}
-}}
+#define NODE_NATIVES_MAP(V) \\
+{node_natives_map}
 
+namespace node {{
+{sources}
 }}  // namespace node
-"""
 
-ONE_BYTE_STRING = """
-static const uint8_t raw_{var}[] = {{ {data} }};
-static struct : public v8::String::ExternalOneByteStringResource {{
-  const char* data() const override {{
-    return reinterpret_cast<const char*>(raw_{var});
-  }}
-  size_t length() const override {{ return arraysize(raw_{var}); }}
-  void Dispose() override {{ /* Default calls `delete this`. */ }}
-  v8::Local<v8::String> ToStringChecked(v8::Isolate* isolate) {{
-    return v8::String::NewExternalOneByte(isolate, this).ToLocalChecked();
-  }}
-}} {var};
+#endif  // NODE_NATIVES_H_
 """
 
-TWO_BYTE_STRING = """
-static const uint16_t raw_{var}[] = {{ {data} }};
-static struct : public v8::String::ExternalStringResource {{
-  const uint16_t* data() const override {{ return raw_{var}; }}
-  size_t length() const override {{ return arraysize(raw_{var}); }}
-  void Dispose() override {{ /* Default calls `delete this`. */ }}
-  v8::Local<v8::String> ToStringChecked(v8::Isolate* isolate) {{
-    return v8::String::NewExternalTwoByte(isolate, this).ToLocalChecked();
-  }}
-}} {var};
-"""
 
-INITIALIZER = """\
-CHECK(target->Set(env->context(),
-                  {key}.ToStringChecked(env->isolate()),
-                  {value}.ToStringChecked(env->isolate())).FromJust());
+NODE_NATIVES_MAP = """\
+  V({escaped_id}) \\
 """
 
 
-def Render(var, data):
-  # Treat non-ASCII as UTF-8 and convert it to UTF-16.
-  if any(ord(c) > 127 for c in data):
-    template = TWO_BYTE_STRING
-    data = map(ord, data.decode('utf-8').encode('utf-16be'))
-    data = [data[i] * 256 + data[i+1] for i in xrange(0, len(data), 2)]
-    data = ToCArray(data)
-  else:
-    template = ONE_BYTE_STRING
-    data = ToCString(data)
-  return template.format(var=var, data=data)
+SOURCES = """\
+static const uint8_t {escaped_id}_name[] = {{
+{name}}};
+static const uint8_t {escaped_id}_data[] = {{
+{data}}};
+"""
 
 
 def JS2C(source, target):
@@ -248,32 +207,36 @@ def JS2C(source, target):
   (consts, macros) = ReadMacros(macro_lines)
 
   # Build source code lines
-  definitions = []
-  initializers = []
+  node_natives_map = []
+  sources = []
 
-  for name in modules:
-    lines = ReadFile(str(name))
+  for s in modules:
+    lines = ReadFile(str(s))
     lines = ExpandConstants(lines, consts)
     lines = ExpandMacros(lines, macros)
+    data = ToCString(lines)
 
     # On Windows, "./foo.bar" in the .gyp file is passed as "foo.bar"
     # so don't assume there is always a slash in the file path.
-    if '/' in name or '\\' in name:
-      name = '/'.join(re.split('/|\\\\', name)[1:])
+    if '/' in s or '\\' in s:
+      id = '/'.join(re.split('/|\\\\', s)[1:])
+    else:
+      id = s
+
+    if '.' in id:
+      id = id.split('.', 1)[0]
 
-    name = name.split('.', 1)[0]
-    var = name.replace('-', '_').replace('/', '_')
-    key = '%s_key' % var
-    value = '%s_value' % var
+    name = ToCString(id)
+    escaped_id = id.replace('-', '_').replace('/', '_')
+    node_natives_map.append(NODE_NATIVES_MAP.format(**locals()))
+    sources.append(SOURCES.format(**locals()))
 
-    definitions.append(Render(key, name))
-    definitions.append(Render(value, lines))
-    initializers.append(INITIALIZER.format(key=key, value=value))
+  node_natives_map = ''.join(node_natives_map)
+  sources = ''.join(sources)
 
   # Emit result
   output = open(str(target[0]), "w")
-  output.write(TEMPLATE.format(definitions=''.join(definitions),
-                               initializers=''.join(initializers)))
+  output.write(HEADER_TEMPLATE.format(**locals()))
   output.close()
 
 def main():