Self-extracting WebAssembly binaries

cpp/perspective/build.js

@@ -13,36 +13,39 @@
 const { execSync } = require("child_process");
 const os = require("os");
 const path = require("path");
-const fflate = require("fflate");
-const fs = require("fs");
 
 const stdio = "inherit";
-const env = process.PSP_DEBUG ? "debug" : "release";
+const env = process.env.PSP_DEBUG ? "debug" : "release";
 const cwd = path.join(process.cwd(), "dist", env);
 
 delete process.env.NODE;
 
+function bootstrap(file) {
+    execSync(`cargo run -p perspective-bootstrap -- ${file}`, {
+        cwd: path.join(process.cwd(), "..", "..", "rust", "perspective-viewer"),
+        stdio,
+    });
+}
+
 try {
     execSync(`mkdirp ${cwd}`, { stdio });
     process.env.CLICOLOR_FORCE = 1;
     execSync(`emcmake cmake ${__dirname} -Wno-dev -DCMAKE_BUILD_TYPE=${env}`, {
         cwd,
         stdio,
     });
+
     execSync(`emmake make -j${process.env.PSP_NUM_CPUS || os.cpus().length}`, {
         cwd,
         stdio,
     });
+
     execSync(`cpy web/**/* ../web`, { cwd, stdio });
     execSync(`cpy node/**/* ../node`, { cwd, stdio });
-
-    const wasm = fs.readFileSync("dist/web/perspective.cpp.wasm");
-    const compressed = fflate.compressSync(wasm);
-    fs.writeFileSync("dist/web/perspective.cpp.wasm", compressed);
-
-    const wasm2 = fs.readFileSync("dist/node/perspective.cpp.wasm");
-    const compressed2 = fflate.compressSync(wasm2);
-    fs.writeFileSync("dist/node/perspective.cpp.wasm", compressed2);
+    if (!process.env.PSP_DEBUG) {
+        bootstrap(`../../cpp/perspective/dist/web/perspective.cpp.wasm`);
+        bootstrap(`../../cpp/perspective/dist/node/perspective.cpp.wasm`);
+    }
 } catch (e) {
     console.error(e);
     process.exit(1);

cpp/perspective/package.json

@@ -12,8 +12,5 @@
     "scripts": {
         "build": "node ../../tools/perspective-scripts/run_emsdk.mjs node ./build.js",
         "clean": "rimraf dist"
-    },
-    "devDependencies": {
-        "fflate": "^0.7.4"
     }
 }

cpp/perspective/src/include/perspective/scalar.h

@@ -200,49 +200,49 @@ struct PERSPECTIVE_EXPORT t_tscalar {
     bool m_inplace;
 };
 
-inline t_tscalar operator"" _ts(long double v) {
-    t_tscalar rv;
-    double tmp = v;
-    rv.set(tmp);
-    return rv;
-}
-
-inline t_tscalar operator"" _ts(unsigned long long int v) {
-    t_tscalar rv;
-    std::int64_t tmp = v;
-    rv.set(tmp);
-    return rv;
-}
-
-inline t_tscalar operator"" _ts(const char* v, std::size_t len) {
-    t_tscalar rv;
-    rv.set(v);
-    return rv;
-}
-
-inline t_tscalar operator"" _ns(long double v) {
-    t_tscalar rv;
-    rv.m_data.m_uint64 = 0;
-    rv.m_type = DTYPE_FLOAT64;
-    rv.m_status = STATUS_INVALID;
-    return rv;
-}
-
-inline t_tscalar operator"" _ns(unsigned long long int v) {
-    t_tscalar rv;
-    rv.m_data.m_uint64 = 0;
-    rv.m_type = DTYPE_INT64;
-    rv.m_status = STATUS_INVALID;
-    return rv;
-}
-
-inline t_tscalar operator"" _ns(const char* v, std::size_t len) {
-    t_tscalar rv;
-    rv.m_data.m_uint64 = 0;
-    rv.m_type = DTYPE_STR;
-    rv.m_status = STATUS_INVALID;
-    return rv;
-}
+// inline t_tscalar operator"" _ts(long double v) {
+//     t_tscalar rv;
+//     double tmp = v;
+//     rv.set(tmp);
+//     return rv;
+// }
+
+// inline t_tscalar operator"" _ts(unsigned long long int v) {
+//     t_tscalar rv;
+//     std::int64_t tmp = v;
+//     rv.set(tmp);
+//     return rv;
+// }
+
+// inline t_tscalar operator"" _ts(const char* v, std::size_t len) {
+//     t_tscalar rv;
+//     rv.set(v);
+//     return rv;
+// }
+
+// inline t_tscalar operator"" _ns(long double v) {
+//     t_tscalar rv;
+//     rv.m_data.m_uint64 = 0;
+//     rv.m_type = DTYPE_FLOAT64;
+//     rv.m_status = STATUS_INVALID;
+//     return rv;
+// }
+
+// inline t_tscalar operator"" _ns(unsigned long long int v) {
+//     t_tscalar rv;
+//     rv.m_data.m_uint64 = 0;
+//     rv.m_type = DTYPE_INT64;
+//     rv.m_status = STATUS_INVALID;
+//     return rv;
+// }
+
+// inline t_tscalar operator"" _ns(const char* v, std::size_t len) {
+//     t_tscalar rv;
+//     rv.m_data.m_uint64 = 0;
+//     rv.m_type = DTYPE_STR;
+//     rv.m_status = STATUS_INVALID;
+//     return rv;
+// }
 
 PERSPECTIVE_EXPORT t_tscalar mknone();
 PERSPECTIVE_EXPORT t_tscalar mknull(t_dtype dtype);

packages/perspective/package.json

@@ -42,7 +42,6 @@
         "clean": "rimraf dist"
     },
     "dependencies": {
-        "fflate": "^0.7.4",
         "stoppable": "1.1.0",
         "ws": "^6.1.2"
     },

packages/perspective/src/js/perspective.browser.js

@@ -14,19 +14,13 @@ import * as defaults from "./config/constants.js";
 import { get_config, get_type_config as _get_type_config } from "./config";
 import { Client } from "./api/client.js";
 import { WebSocketClient } from "./websocket/client";
-
 import { override_config } from "./config/index.js";
-import { Decompress } from "fflate";
 
 import wasm_worker from "../../src/js/perspective.worker.js";
 import wasm from "../../dist/pkg/web/perspective.cpp.wasm";
 
 let IS_INLINE = false;
 
-function is_gzip(buffer) {
-    return new Uint32Array(buffer.slice(0, 4))[0] == 559903;
-}
-
 /**
  * Singleton WASM file download cache.
  */
@@ -41,59 +35,22 @@ const _override = /* @__PURE__ */ (function () {
 
                 async wasm() {
                     let _wasm = await wasm;
-
-                    let parts = [];
-                    let length = 0;
-                    const decompressor = new Decompress((chunk) => {
-                        if (chunk) {
-                            length += chunk.byteLength;
-                            parts.push(chunk);
-                        }
-                    });
-
                     if (_wasm instanceof ArrayBuffer && !_wasm.buffer) {
                         _wasm = new Uint8Array(_wasm);
                     }
 
                     if (_wasm.buffer && _wasm.buffer instanceof ArrayBuffer) {
                         IS_INLINE = true;
-                        if (is_gzip(_wasm.buffer)) {
-                            decompressor.push(_wasm, true);
-                        } else {
-                            length = _wasm.byteLength;
-                            parts = [_wasm];
-                        }
+                        length = _wasm.byteLength;
+                        this._wasm = _wasm;
                     } else if (_wasm instanceof ArrayBuffer) {
                         length = _wasm.byteLength;
-                        parts = [new Uint8Array(_wasm)];
+                        this._wasm = new Uint8Array(_wasm);
                     } else {
                         const resp = await fetch(_wasm);
-                        const reader = resp.body.getReader();
-                        let state = 0;
-                        while (true) {
-                            const { value, done } = await reader.read();
-                            if (done) break;
-                            if (
-                                (state === 0 && is_gzip(value.buffer)) ||
-                                state === 1
-                            ) {
-                                state = 1;
-                                decompressor.push(value, done);
-                            } else {
-                                state = 2;
-                                length += value.byteLength;
-                                parts.push(value);
-                            }
-                        }
+                        this._wasm = await resp.arrayBuffer();
                     }
 
-                    let offset = 0;
-                    const buffer = new Uint8Array(length);
-                    for (const part of parts) {
-                        buffer.set(part, offset);
-                        offset += part.byteLength;
-                    }
-                    this._wasm = buffer.buffer;
                     return this._wasm;
                 }
             })();

packages/perspective/src/js/perspective.js

@@ -2195,20 +2195,24 @@ export default function (Module) {
          * @param {ArrayBuffer} buffer an ArrayBuffer or Buffer containing the
          * Perspective WASM code
          */
-        init(msg) {
-            if (typeof WebAssembly === "undefined") {
-                throw new Error("WebAssembly not supported");
-            } else {
-                __MODULE__({
-                    wasmBinary: msg.buffer,
-                    wasmJSMethod: "native-wasm",
-                    locateFile: (x) => x,
-                }).then((mod) => {
-                    __MODULE__ = mod;
-                    __MODULE__.init();
-                    super.init(msg);
-                });
-            }
+        async init(msg) {
+            let wasmBinary = msg.buffer;
+            try {
+                const mod = await WebAssembly.instantiate(wasmBinary);
+                const exports = mod.instance.exports;
+                const size = exports.size();
+                const offset = exports.offset();
+                const array = new Uint8Array(exports.memory.buffer);
+                wasmBinary = array.slice(offset, offset + size);
+            } catch {}
+            __MODULE__ = await __MODULE__({
+                wasmBinary,
+                wasmJSMethod: "native-wasm",
+                locateFile: (x) => x,
+            });
+
+            __MODULE__.init();
+            super.init(msg);
         }
     }
 

packages/perspective/src/js/perspective.node.js

@@ -24,52 +24,33 @@ const http = require("http");
 const WebSocket = require("ws");
 const process = require("process");
 const path = require("path");
-const { Decompress } = require("fflate");
 
 const load_perspective = require("../../dist/pkg/node/perspective.cpp.js");
 
 const LOCAL_PATH = path.join(process.cwd(), "node_modules");
 const buffer = require("../../dist/pkg/node/perspective.cpp.wasm").default;
 
-function deflate(buffer) {
-    let parts = [];
-    let length = 0;
-    const decompressor = new Decompress((chunk) => {
-        if (chunk) {
-            length += chunk.byteLength;
-            parts.push(chunk);
-        }
-    });
-
-    decompressor.push(buffer, true);
-    let offset = 0;
-    const buffer2 = new Uint8Array(length);
-    for (const part of parts) {
-        buffer2.set(part, offset);
-        offset += part.byteLength;
-    }
-
-    return buffer2.buffer;
-}
 const SYNC_SERVER = new (class extends Server {
     init(msg) {
-        let done;
-        this._loaded_promise = new Promise((x) => {
-            done = x;
-        });
-        buffer
-            .then((buffer) => {
-                return load_perspective({
-                    wasmBinary: deflate(buffer),
-                    wasmJSMethod: "native-wasm",
-                });
-            })
-            .then((core) => {
-                core.init();
-                this.perspective = perspective(core);
-                super.init(msg);
-                done();
+        this._loaded_promise = (async () => {
+            let wasmBinary = await buffer;
+            try {
+                const mod = await WebAssembly.instantiate(wasmBinary);
+                const exports = mod.instance.exports;
+                const size = exports.size();
+                const offset = exports.offset();
+                const array = new Uint8Array(exports.memory.buffer);
+                wasmBinary = array.slice(offset, offset + size);
+            } catch {}
+            const core = await load_perspective({
+                wasmBinary,
+                wasmJSMethod: "native-wasm",
             });
+
+            core.init();
+            this.perspective = perspective(core);
+            super.init(msg);
+        })();
     }
 
     post(msg) {