esm: add chaining to loaders

Author: JakobJingleheimer

doc/api/errors.md

@@ -1737,6 +1737,13 @@ An import assertion is not supported by this version of Node.js.
 An option pair is incompatible with each other and cannot be used at the same
 time.
 
+<a id="ERR_INCOMPLETE_LOADER_CHAIN"></a>
+
+### `ERR_INCOMPLETE_LOADER_CHAIN`
+
+An ESM loader hook returned without calling `next()` and without explicitly
+signally a short-circuit.
+
 <a id="ERR_INPUT_TYPE_NOT_ALLOWED"></a>
 
 ### `ERR_INPUT_TYPE_NOT_ALLOWED`

lib/internal/errors.js

@@ -1111,6 +1111,13 @@ E('ERR_IMPORT_ASSERTION_TYPE_UNSUPPORTED',
   'Import assertion type "%s" is unsupported', TypeError);
 E('ERR_INCOMPATIBLE_OPTION_PAIR',
   'Option "%s" cannot be used in combination with option "%s"', TypeError);
+E(
+  'ERR_INCOMPLETE_LOADER_CHAIN',
+  'The "%s" hook from %s did not call the next hook in its chain and did not' +
+  ' explicitly signal a short-circuit. If this is intentional, include' +
+  ' `shortCircuit: true` in the hook\'s return.',
+  Error
+);
 E('ERR_INPUT_TYPE_NOT_ALLOWED', '--input-type can only be used with string ' +
   'input via --eval, --print, or STDIN', Error);
 E('ERR_INSPECTOR_ALREADY_ACTIVATED',

lib/internal/modules/esm/loader.js

@@ -16,13 +16,15 @@ const {
   PromiseAll,
   RegExpPrototypeExec,
   SafeArrayIterator,
+  SafeMap,
   SafeWeakMap,
   StringPrototypeStartsWith,
   globalThis,
 } = primordials;
 const { MessageChannel } = require('internal/worker/io');
 
 const {
+  ERR_INCOMPLETE_LOADER_CHAIN,
   ERR_INTERNAL_ASSERTION,
   ERR_INVALID_ARG_TYPE,
   ERR_INVALID_ARG_VALUE,
@@ -70,28 +72,30 @@ class ESMLoader {
   /**
    * Prior to ESM loading. These are called once before any modules are started.
    * @private
-   * @property {Function[]} globalPreloaders First-in-first-out list of
-   * preload hooks.
+   * @property {Map<URL['href'], Function>} globalPreloaders Last-in-first-out
+   *  list of preload hooks.
    */
-  #globalPreloaders = [];
+  #globalPreloaders = new SafeMap();
 
   /**
    * Phase 2 of 2 in ESM loading.
    * @private
-   * @property {Function[]} loaders First-in-first-out list of loader hooks.
+   * @property {Map<URL['href'], Function>} loaders Last-in-first-out
+   *  collection of loader hooks.
    */
-  #loaders = [
-    defaultLoad,
-  ];
+  #loaders = new SafeMap([
+    ['node:esm/load.js', defaultLoad],
+  ]);
 
   /**
    * Phase 1 of 2 in ESM loading.
    * @private
-   * @property {Function[]} resolvers First-in-first-out list of resolver hooks
+   * @property {Map<URL['href'], Function>} resolvers Last-in-first-out
+   *  collection of resolver hooks.
    */
-  #resolvers = [
-    defaultResolve,
-  ];
+  #resolvers = new SafeMap([
+    ['node:esm/resolve.js', defaultResolve],
+  ]);
 
   #importMetaInitializer = initializeImportMeta;
 
@@ -115,7 +119,9 @@ class ESMLoader {
    */
   translators = translators;
 
-  constructor() {
+  constructor({ isInternal = false } = {}) {
+    this.isInternal = isInternal;
+
     if (getOptionValue('--experimental-loader')) {
       emitExperimentalWarning('Custom ESM Loaders');
     }
@@ -198,32 +204,46 @@ class ESMLoader {
    * user-defined loaders (as returned by ESMLoader.import()).
    */
   async addCustomLoaders(
-    customLoaders = [],
+    customLoaders = new SafeMap(),
   ) {
-    if (!ArrayIsArray(customLoaders)) customLoaders = [customLoaders];
-
-    for (let i = 0; i < customLoaders.length; i++) {
-      const exports = customLoaders[i];
+    // Maps are first-in-first-out, but hook chains are last-in-first-out,
+    // so create a new container for the incoming hooks (which have already
+    // been reversed).
+    const globalPreloaders = new SafeMap();
+    const resolvers = new SafeMap();
+    const loaders = new SafeMap();
+
+    for (const { 0: url, 1: exports } of customLoaders) {
       const {
         globalPreloader,
         resolver,
         loader,
       } = ESMLoader.pluckHooks(exports);
 
-      if (globalPreloader) ArrayPrototypePush(
-        this.#globalPreloaders,
+      if (globalPreloader) globalPreloaders.set(
+        url,
         FunctionPrototypeBind(globalPreloader, null), // [1]
       );
-      if (resolver) ArrayPrototypePush(
-        this.#resolvers,
+      if (resolver) resolvers.set(
+        url,
         FunctionPrototypeBind(resolver, null), // [1]
       );
-      if (loader) ArrayPrototypePush(
-        this.#loaders,
+      if (loader) loaders.set(
+        url,
         FunctionPrototypeBind(loader, null), // [1]
       );
     }
 
+    // Append the pre-existing hooks (the builtin/default ones)
+    for (const p of this.#globalPreloaders) globalPreloaders.set(p[0], p[1]);
+    for (const p of this.#resolvers) resolvers.set(p[0], p[1]);
+    for (const p of this.#loaders) loaders.set(p[0], p[1]);
+
+    // Replace the obsolete maps with the fully-loaded & properly sequenced one
+    this.#globalPreloaders = globalPreloaders;
+    this.#resolvers = resolvers;
+    this.#loaders = loaders;
+
     // [1] ensure hook function is not bound to ESMLoader instance
 
     this.preload();
@@ -308,14 +328,21 @@ class ESMLoader {
    */
   async getModuleJob(specifier, parentURL, importAssertions) {
     let importAssertionsForResolve;
-    if (this.#loaders.length !== 1) {
-      // We can skip cloning if there are no user provided loaders because
+
+    if (this.#loaders.size !== 1) {
+      // We can skip cloning if there are no user-provided loaders because
       // the Node.js default resolve hook does not use import assertions.
-      importAssertionsForResolve =
-        ObjectAssign(ObjectCreate(null), importAssertions);
+      importAssertionsForResolve = ObjectAssign(
+        ObjectCreate(null),
+        importAssertions,
+      );
     }
-    const { format, url } =
-      await this.resolve(specifier, parentURL, importAssertionsForResolve);
+
+    const { format, url } = await this.resolve(
+      specifier,
+      parentURL,
+      importAssertionsForResolve,
+    );
 
     let job = this.moduleMap.get(url, importAssertions.type);
 
@@ -408,9 +435,13 @@ class ESMLoader {
 
     const namespaces = await PromiseAll(new SafeArrayIterator(jobs));
 
-    return wasArr ?
-      namespaces :
-      namespaces[0];
+    if (!wasArr) return namespaces[0];
+
+    const namespaceMap = new SafeMap();
+
+    for (let i = 0; i < count; i++) namespaceMap.set(specifiers[i], namespaces[i]);
+
+    return namespaceMap;
   }
 
   /**
@@ -423,12 +454,33 @@ class ESMLoader {
    * @returns {object}
    */
   async load(url, context = {}) {
-    const defaultLoader = this.#loaders[0];
+    const loaders = this.#loaders.entries();
+    let {
+      0: loaderFilePath,
+      1: loader,
+    } = loaders.next().value;
+    let chainFinished = this.#loaders.size === 1;
+
+    function next(nextUrl) {
+      const {
+        done,
+        value,
+      } = loaders.next();
+      ({
+        0: loaderFilePath,
+        1: loader,
+      } = value);
+
+      if (done || loader === defaultLoad) chainFinished = true;
+
+      return loader(nextUrl, context, next);
+    }
 
-    const loader = this.#loaders.length === 1 ?
-      defaultLoader :
-      this.#loaders[1];
-    const loaded = await loader(url, context, defaultLoader);
+    const loaded = await loader(
+      url,
+      context,
+      next,
+    );
 
     if (typeof loaded !== 'object') {
       throw new ERR_INVALID_RETURN_VALUE(
@@ -440,9 +492,14 @@ class ESMLoader {
 
     const {
       format,
+      shortCircuit,
       source,
     } = loaded;
 
+    if (!chainFinished && !shortCircuit) {
+      throw new ERR_INCOMPLETE_LOADER_CHAIN('load', loaderFilePath);
+    }
+
     if (format == null) {
       const dataUrl = RegExpPrototypeExec(
         /^data:([^/]+\/[^;,]+)(?:[^,]*?)(;base64)?,/,
@@ -594,21 +651,38 @@ class ESMLoader {
       parentURL,
     );
 
-    const conditions = DEFAULT_CONDITIONS;
+    const resolvers = this.#resolvers.entries();
+    let {
+      0: resolverFilePath,
+      1: resolver,
+    } = resolvers.next().value;
+    let chainFinished = this.#resolvers.size === 1;
 
-    const defaultResolver = this.#resolvers[0];
+    const context = {
+      conditions: DEFAULT_CONDITIONS,
+      importAssertions,
+      parentURL,
+    };
+
+    function next(suppliedUrl) {
+      const {
+        done,
+        value,
+      } = resolvers.next();
+      ({
+        0: resolverFilePath,
+        1: resolver,
+      } = value);
+
+      if (done || resolver === defaultResolve) chainFinished = true;
+
+      return resolver(suppliedUrl, context, next);
+    }
 
-    const resolver = this.#resolvers.length === 1 ?
-      defaultResolver :
-      this.#resolvers[1];
     const resolution = await resolver(
       originalSpecifier,
-      {
-        conditions,
-        importAssertions,
-        parentURL,
-      },
-      defaultResolver,
+      context,
+      next,
     );
 
     if (typeof resolution !== 'object') {
@@ -619,7 +693,15 @@ class ESMLoader {
       );
     }
 
-    const { format, url } = resolution;
+    const {
+      format,
+      shortCircuit,
+      url,
+    } = resolution;
+
+    if (!chainFinished && !shortCircuit) {
+      throw new ERR_INCOMPLETE_LOADER_CHAIN('resolve', resolverFilePath);
+    }
 
     if (
       format != null &&

lib/internal/modules/run_main.js

@@ -48,11 +48,19 @@ function runMainESM(mainPath) {
   const { loadESM } = require('internal/process/esm_loader');
   const { pathToFileURL } = require('internal/url');
 
-  handleMainPromise(loadESM((esmLoader) => {
-    const main = path.isAbsolute(mainPath) ?
-      pathToFileURL(mainPath).href : mainPath;
-    return esmLoader.import(main, undefined, ObjectCreate(null));
-  }));
+  handleMainPromise(
+    loadESM((esmLoader) => {
+      const main = path.isAbsolute(mainPath) ?
+        pathToFileURL(mainPath).href :
+        mainPath;
+
+      return esmLoader.import(
+        main,
+        undefined,
+        ObjectCreate(null),
+      );
+    })
+  );
 }
 
 async function handleMainPromise(promise) {

lib/internal/process/esm_loader.js

@@ -49,8 +49,6 @@ exports.esmLoader = esmLoader;
  */
 async function initializeLoader() {
   const { getOptionValue } = require('internal/options');
-  // customLoaders CURRENTLY can be only 1 (a string)
-  // Once chaining is implemented, it will be string[]
   const customLoaders = getOptionValue('--experimental-loader');
 
   if (!customLoaders.length) return;
@@ -65,18 +63,18 @@ async function initializeLoader() {
   // A separate loader instance is necessary to avoid cross-contamination
   // between internal Node.js and userland. For example, a module with internal
   // state (such as a counter) should be independent.
-  const internalEsmLoader = new ESMLoader();
+  const internalEsmLoader = new ESMLoader({ isInternal: true });
 
   // Importation must be handled by internal loader to avoid poluting userland
-  const exports = await internalEsmLoader.import(
-    customLoaders,
+  const exportsMap = await internalEsmLoader.import(
+    customLoaders.reverse(), // last-in-first-out
     pathToFileURL(cwd).href,
     ObjectCreate(null),
   );
 
   // Hooks must then be added to external/public loader
   // (so they're triggered in userland)
-  await esmLoader.addCustomLoaders(exports);
+  await esmLoader.addCustomLoaders(exportsMap);
 }
 
 exports.loadESM = async function loadESM(callback) {

src/node_options.cc

@@ -328,7 +328,7 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
   AddOption("--experimental-json-modules", "", NoOp{}, kAllowedInEnvironment);
   AddOption("--experimental-loader",
             "use the specified module as a custom loader",
-            &EnvironmentOptions::userland_loader,
+            &EnvironmentOptions::userland_loaders,
             kAllowedInEnvironment);
   AddAlias("--loader", "--experimental-loader");
   AddOption("--experimental-modules", "", NoOp{}, kAllowedInEnvironment);

src/node_options.h

@@ -159,7 +159,7 @@ class EnvironmentOptions : public Options {
   bool trace_warnings = false;
   bool extra_info_on_fatal_exception = true;
   std::string unhandled_rejections;
-  std::string userland_loader;
+  std::vector<std::string> userland_loaders;
   bool verify_base_objects =
 #ifdef DEBUG
       true;