esm: refactor responseURL handling

PR-URL: #43164
Reviewed-By: Antoine du Hamel <[email protected]>
Reviewed-By: Minwoo Jung <[email protected]>
Reviewed-By: Jacob Smith <[email protected]>

Author: guybedford

lib/internal/modules/cjs/loader.js

@@ -1026,8 +1026,7 @@ function wrapSafe(filename, content, cjsModuleInstance) {
       displayErrors: true,
       importModuleDynamically: async (specifier, _, importAssertions) => {
         const loader = asyncESM.esmLoader;
-        return loader.import(specifier,
-                             loader.getBaseURL(normalizeReferrerURL(filename)),
+        return loader.import(specifier, normalizeReferrerURL(filename),
                              importAssertions);
       },
     });
@@ -1043,8 +1042,7 @@ function wrapSafe(filename, content, cjsModuleInstance) {
       filename,
       importModuleDynamically(specifier, _, importAssertions) {
         const loader = asyncESM.esmLoader;
-        return loader.import(specifier,
-                             loader.getBaseURL(normalizeReferrerURL(filename)),
+        return loader.import(specifier, normalizeReferrerURL(filename),
                              importAssertions);
       },
     });

lib/internal/modules/esm/get_source.js

@@ -26,15 +26,13 @@ function createImportMetaResolve(defaultParentUrl) {
  * @param {{url: string}} context
  */
 function initializeImportMeta(meta, context) {
-  let url = context.url;
+  const { url } = context;
 
   // Alphabetical
   if (experimentalImportMetaResolve) {
     meta.resolve = createImportMetaResolve(url);
   }
 
-  url = asyncESM.esmLoader.getBaseURL(url);
-
   meta.url = url;
 }
 

lib/internal/modules/esm/initialize_import_meta.js

@@ -1,8 +1,67 @@
 'use strict';
 
+const {
+  ArrayPrototypePush,
+  RegExpPrototypeExec,
+  decodeURIComponent,
+} = primordials;
+
 const { defaultGetFormat } = require('internal/modules/esm/get_format');
-const { defaultGetSource } = require('internal/modules/esm/get_source');
 const { validateAssertions } = require('internal/modules/esm/assert');
+const { getOptionValue } = require('internal/options');
+const { fetchModule } = require('internal/modules/esm/fetch_module');
+
+// Do not eagerly grab .manifest, it may be in TDZ
+const policy = getOptionValue('--experimental-policy') ?
+  require('internal/process/policy') :
+  null;
+const experimentalNetworkImports =
+  getOptionValue('--experimental-network-imports');
+
+const { Buffer: { from: BufferFrom } } = require('buffer');
+
+const { readFile: readFileAsync } = require('internal/fs/promises').exports;
+const { URL } = require('internal/url');
+const {
+  ERR_INVALID_URL,
+  ERR_UNSUPPORTED_ESM_URL_SCHEME,
+} = require('internal/errors').codes;
+
+const DATA_URL_PATTERN = /^[^/]+\/[^,;]+(?:[^,]*?)(;base64)?,([\s\S]*)$/;
+
+async function getSource(url, context) {
+  const parsed = new URL(url);
+  let responseURL = url;
+  let source;
+  if (parsed.protocol === 'file:') {
+    source = await readFileAsync(parsed);
+  } else if (parsed.protocol === 'data:') {
+    const match = RegExpPrototypeExec(DATA_URL_PATTERN, parsed.pathname);
+    if (!match) {
+      throw new ERR_INVALID_URL(url);
+    }
+    const { 1: base64, 2: body } = match;
+    source = BufferFrom(decodeURIComponent(body), base64 ? 'base64' : 'utf8');
+  } else if (experimentalNetworkImports && (
+    parsed.protocol === 'https:' ||
+    parsed.protocol === 'http:'
+  )) {
+    const res = await fetchModule(parsed, context);
+    source = await res.body;
+    responseURL = res.resolvedHREF;
+  } else {
+    const supportedSchemes = ['file', 'data'];
+    if (experimentalNetworkImports) {
+      ArrayPrototypePush(supportedSchemes, 'http', 'https');
+    }
+    throw new ERR_UNSUPPORTED_ESM_URL_SCHEME(parsed, supportedSchemes);
+  }
+  if (policy?.manifest) {
+    policy.manifest.assertIntegrity(parsed, source);
+  }
+  return { responseURL, source };
+}
+
 
 /**
  * Node.js default load hook.
@@ -11,6 +70,7 @@ const { validateAssertions } = require('internal/modules/esm/assert');
  * @returns {object}
  */
 async function defaultLoad(url, context) {
+  let responseURL = url;
   const { importAssertions } = context;
   let {
     format,
@@ -29,11 +89,12 @@ async function defaultLoad(url, context) {
   ) {
     source = null;
   } else if (source == null) {
-    source = await defaultGetSource(url, context);
+    ({ responseURL, source } = await getSource(url, context));
   }
 
   return {
     format,
+    responseURL,
     source,
   };
 }

lib/internal/modules/esm/load.js

@@ -17,13 +17,11 @@ const {
   RegExpPrototypeExec,
   SafeArrayIterator,
   SafeWeakMap,
-  StringPrototypeStartsWith,
   globalThis,
 } = primordials;
 const { MessageChannel } = require('internal/worker/io');
 
 const {
-  ERR_INTERNAL_ASSERTION,
   ERR_INVALID_ARG_TYPE,
   ERR_INVALID_ARG_VALUE,
   ERR_INVALID_RETURN_PROPERTY_VALUE,
@@ -50,10 +48,6 @@ const { defaultLoad } = require('internal/modules/esm/load');
 const { translators } = require(
   'internal/modules/esm/translators');
 const { getOptionValue } = require('internal/options');
-const {
-  fetchModule,
-} = require('internal/modules/esm/fetch_module');
-
 
 /**
  * Prevent the specifier resolution warning from being printed twice
@@ -238,9 +232,7 @@ class ESMLoader {
       const module = new ModuleWrap(url, undefined, source, 0, 0);
       callbackMap.set(module, {
         importModuleDynamically: (specifier, { url }, importAssertions) => {
-          return this.import(specifier,
-                             this.getBaseURL(url),
-                             importAssertions);
+          return this.import(specifier, url, importAssertions);
         }
       });
 
@@ -256,43 +248,6 @@ class ESMLoader {
     };
   }
 
-  /**
-   * Returns the url to use for the resolution of a given cache key url
-   * These are not guaranteed to be the same.
-   *
-   * In WHATWG HTTP spec for ESM the cache key is the non-I/O bound
-   * synchronous resolution using only string operations
-   *   ~= resolveImportMap(new URL(specifier, importerHREF))
-   *
-   * The url used for subsequent resolution is the response URL after
-   * all redirects have been resolved.
-   *
-   * https://example.com/foo redirecting to https://example.com/bar
-   * would have a cache key of https://example.com/foo and baseURL
-   * of https://example.com/bar
-   *
-   * MUST BE SYNCHRONOUS for import.meta initialization
-   * MUST BE CALLED AFTER receiving the url body due to I/O
-   * @param {string} url
-   * @returns {string}
-   */
-  getBaseURL(url) {
-    if (
-      StringPrototypeStartsWith(url, 'http:') ||
-      StringPrototypeStartsWith(url, 'https:')
-    ) {
-      // The request & response have already settled, so they are in
-      // fetchModule's cache, in which case, fetchModule returns
-      // immediately and synchronously
-      url = fetchModule(new URL(url), { parentURL: url }).resolvedHREF;
-      // This should only occur if the module hasn't been fetched yet
-      if (typeof url !== 'string') {
-        throw new ERR_INTERNAL_ASSERTION(`Base url for module ${url} not loaded.`);
-      }
-    }
-    return url;
-  }
-
   /**
    * Get a (possibly still pending) module job from the cache,
    * or create one and return its Promise.
@@ -346,6 +301,7 @@ class ESMLoader {
     const moduleProvider = async (url, isMain) => {
       const {
         format: finalFormat,
+        responseURL,
         source,
       } = await this.load(url, {
         format,
@@ -355,10 +311,10 @@ class ESMLoader {
       const translator = translators.get(finalFormat);
 
       if (!translator) {
-        throw new ERR_UNKNOWN_MODULE_FORMAT(finalFormat, url);
+        throw new ERR_UNKNOWN_MODULE_FORMAT(finalFormat, responseURL);
       }
 
-      return FunctionPrototypeCall(translator, this, url, source, isMain);
+      return FunctionPrototypeCall(translator, this, responseURL, source, isMain);
     };
 
     const inspectBrk = (
@@ -442,6 +398,29 @@ class ESMLoader {
       format,
       source,
     } = loaded;
+    let responseURL = loaded.responseURL;
+
+    if (responseURL === undefined) {
+      responseURL = url;
+    }
+
+    let responseURLObj;
+    if (typeof responseURL === 'string') {
+      try {
+        responseURLObj = new URL(responseURL);
+      } catch {
+        // responseURLObj not defined will throw in next branch.
+      }
+    }
+
+    if (responseURLObj?.href !== responseURL) {
+      throw new ERR_INVALID_RETURN_PROPERTY_VALUE(
+        'undefined or a fully resolved URL string',
+        hookErrIdentifier,
+        'responseURL',
+        responseURL,
+      );
+    }
 
     if (format == null) {
       const dataUrl = RegExpPrototypeExec(
@@ -477,6 +456,7 @@ class ESMLoader {
 
     return {
       format,
+      responseURL,
       source,
     };
   }

lib/internal/modules/esm/loader.js

@@ -76,8 +76,7 @@ class ModuleJob {
       // these `link` callbacks depending on each other.
       const dependencyJobs = [];
       const promises = this.module.link(async (specifier, assertions) => {
-        const baseURL = this.loader.getBaseURL(url);
-        const jobPromise = this.loader.getModuleJob(specifier, baseURL, assertions);
+        const jobPromise = this.loader.getModuleJob(specifier, url, assertions);
         ArrayPrototypePush(dependencyJobs, jobPromise);
         const job = await jobPromise;
         return job.modulePromise;

lib/internal/modules/esm/module_job.js

@@ -103,9 +103,7 @@ function errPath(url) {
 }
 
 async function importModuleDynamically(specifier, { url }, assertions) {
-  return asyncESM.esmLoader.import(specifier,
-                                   asyncESM.esmLoader.getBaseURL(url),
-                                   assertions);
+  return asyncESM.esmLoader.import(specifier, url, assertions);
 }
 
 // Strategy for loading a standard JavaScript module.
@@ -116,9 +114,7 @@ translators.set('module', async function moduleStrategy(url, source, isMain) {
   debug(`Translating StandardModule ${url}`);
   const module = new ModuleWrap(url, undefined, source, 0, 0);
   moduleWrap.callbackMap.set(module, {
-    initializeImportMeta: (meta, wrap) => this.importMetaInitialize(meta, {
-      url: wrap.url
-    }),
+    initializeImportMeta: (meta, wrap) => this.importMetaInitialize(meta, { url }),
     importModuleDynamically,
   });
   return module;

lib/internal/modules/esm/translators.js

@@ -78,7 +78,6 @@ const expectedModules = new Set([
   'NativeModule internal/modules/esm/fetch_module',
   'NativeModule internal/modules/esm/formats',
   'NativeModule internal/modules/esm/get_format',
-  'NativeModule internal/modules/esm/get_source',
   'NativeModule internal/modules/esm/handle_process_exit',
   'NativeModule internal/modules/esm/initialize_import_meta',
   'NativeModule internal/modules/esm/load',