fetch: refactor referrer policy util functions (#3706)

* improve setRequestReferrerPolicyOnRedirect, create

* fix bug

* adapt more

* improve isOriginPotentiallyTrustworthy

* comments and stuff

* add newline

* Update lib/web/fetch/util.js

Co-authored-by: Khafra <[email protected]>

* fix

* fix test

* add test

* fix

* revert

* tests

* fix

---------

Co-authored-by: Khafra <[email protected]>

Author: Uzlopak

lib/web/fetch/constants.js

@@ -22,10 +22,9 @@ const badPorts = /** @type {const} */ ([
 const badPortsSet = new Set(badPorts)
 
 /**
- * @see https://w3c.github.io/webappsec-referrer-policy/#referrer-policies
+ * @see https://w3c.github.io/webappsec-referrer-policy/#referrer-policy-header
  */
-const referrerPolicy = /** @type {const} */ ([
-  '',
+const referrerPolicyTokens = /** @type {const} */ ([
   'no-referrer',
   'no-referrer-when-downgrade',
   'same-origin',
@@ -35,7 +34,15 @@ const referrerPolicy = /** @type {const} */ ([
   'strict-origin-when-cross-origin',
   'unsafe-url'
 ])
-const referrerPolicySet = new Set(referrerPolicy)
+
+/**
+ * @see https://w3c.github.io/webappsec-referrer-policy/#referrer-policies
+ */
+const referrerPolicy = /** @type {const} */ ([
+  '',
+  ...referrerPolicyTokens
+])
+const referrerPolicyTokensSet = new Set(referrerPolicyTokens)
 
 const requestRedirect = /** @type {const} */ (['follow', 'manual', 'error'])
 
@@ -120,5 +127,5 @@ module.exports = {
   corsSafeListedMethodsSet,
   safeMethodsSet,
   forbiddenMethodsSet,
-  referrerPolicySet
+  referrerPolicyTokens: referrerPolicyTokensSet
 }

lib/web/fetch/util.js

@@ -2,7 +2,7 @@
 
 const { Transform } = require('node:stream')
 const zlib = require('node:zlib')
-const { redirectStatusSet, referrerPolicySet: referrerPolicyTokens, badPortsSet } = require('./constants')
+const { redirectStatusSet, referrerPolicyTokens, badPortsSet } = require('./constants')
 const { getGlobalOrigin } = require('./global')
 const { collectASequenceOfCodePoints, collectAnHTTPQuotedString, removeChars, parseMIMEType } = require('./data-url')
 const { performance } = require('node:perf_hooks')
@@ -170,29 +170,24 @@ function isValidHeaderValue (potentialValue) {
   ) === false
 }
 
-// https://w3c.github.io/webappsec-referrer-policy/#set-requests-referrer-policy-on-redirect
-function setRequestReferrerPolicyOnRedirect (request, actualResponse) {
-  //  Given a request request and a response actualResponse, this algorithm
-  //  updates request’s referrer policy according to the Referrer-Policy
-  //  header (if any) in actualResponse.
-
-  // 1. Let policy be the result of executing § 8.1 Parse a referrer policy
-  // from a Referrer-Policy header on actualResponse.
-
-  // 8.1 Parse a referrer policy from a Referrer-Policy header
+/**
+ * Parse a referrer policy from a Referrer-Policy header
+ * @see https://w3c.github.io/webappsec-referrer-policy/#parse-referrer-policy-from-header
+ */
+function parseReferrerPolicy (actualResponse) {
   // 1. Let policy-tokens be the result of extracting header list values given `Referrer-Policy` and response’s header list.
-  const { headersList } = actualResponse
+  const policyHeader = (actualResponse.headersList.get('referrer-policy', true) ?? '').split(',')
+
   // 2. Let policy be the empty string.
+  let policy = ''
+
   // 3. For each token in policy-tokens, if token is a referrer policy and token is not the empty string, then set policy to token.
-  // 4. Return policy.
-  const policyHeader = (headersList.get('referrer-policy', true) ?? '').split(',')
 
   // Note: As the referrer-policy can contain multiple policies
   // separated by comma, we need to loop through all of them
   // and pick the first valid one.
   // Ref: https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Referrer-Policy#specify_a_fallback_policy
-  let policy = ''
-  if (policyHeader.length > 0) {
+  if (policyHeader.length) {
     // The right-most policy takes precedence.
     // The left-most policy is the fallback.
     for (let i = policyHeader.length; i !== 0; i--) {
@@ -204,6 +199,23 @@ function setRequestReferrerPolicyOnRedirect (request, actualResponse) {
     }
   }
 
+  // 4. Return policy.
+  return policy
+}
+
+/**
+ * Given a request request and a response actualResponse, this algorithm
+ * updates request’s referrer policy according to the Referrer-Policy
+ * header (if any) in actualResponse.
+ * @see https://w3c.github.io/webappsec-referrer-policy/#set-requests-referrer-policy-on-redirect
+ * @param {import('./request').Request} request
+ * @param {import('./response').Response} actualResponse
+ */
+function setRequestReferrerPolicyOnRedirect (request, actualResponse) {
+  // 1. Let policy be the result of executing § 8.1 Parse a referrer policy
+  // from a Referrer-Policy header on actualResponse.
+  const policy = parseReferrerPolicy(actualResponse)
+
   // 2. If policy is not the empty string, then set request’s referrer policy to policy.
   if (policy !== '') {
     request.referrerPolicy = policy
@@ -374,8 +386,16 @@ function clonePolicyContainer (policyContainer) {
   }
 }
 
-// https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer
+/**
+ * Determine request’s Referrer
+ *
+ * @see https://w3c.github.io/webappsec-referrer-policy/#determine-requests-referrer
+ */
 function determineRequestsReferrer (request) {
+  // Given a request request, we can determine the correct referrer information
+  // to send by examining its referrer policy as detailed in the following
+  // steps, which return either no referrer or a URL:
+
   // 1. Let policy be request's referrer policy.
   const policy = request.referrerPolicy
 
@@ -387,6 +407,8 @@ function determineRequestsReferrer (request) {
   let referrerSource = null
 
   // 3. Switch on request’s referrer:
+
+  // "client"
   if (request.referrer === 'client') {
     // Note: node isn't a browser and doesn't implement document/iframes,
     // so we bypass this step and replace it with our own.
@@ -397,8 +419,9 @@ function determineRequestsReferrer (request) {
       return 'no-referrer'
     }
 
-    // note: we need to clone it as it's mutated
+    // Note: we need to clone it as it's mutated
     referrerSource = new URL(globalOrigin)
+  // a URL
   } else if (webidl.is.URL(request.referrer)) {
     // Let referrerSource be request’s referrer.
     referrerSource = request.referrer
@@ -500,18 +523,26 @@ function determineRequestsReferrer (request) {
 }
 
 /**
+ * Certain portions of URLs must not be included when sending a URL as the
+ * value of a `Referer` header: a URLs fragment, username, and password
+ * components must be stripped from the URL before it’s sent out. This
+ * algorithm accepts a origin-only flag, which defaults to false. If set to
+ * true, the algorithm will additionally remove the URL’s path and query
+ * components, leaving only the scheme, host, and port.
+ *
  * @see https://w3c.github.io/webappsec-referrer-policy/#strip-url
  * @param {URL} url
- * @param {boolean} [originOnly]
+ * @param {boolean} [originOnly=false]
  */
-function stripURLForReferrer (url, originOnly) {
+function stripURLForReferrer (url, originOnly = false) {
   // 1. Assert: url is a URL.
   assert(webidl.is.URL(url))
 
+  // Note: Create a new URL instance to avoid mutating the original URL.
   url = new URL(url)
 
   // 2. If url’s scheme is a local scheme, then return no referrer.
-  if (url.protocol === 'file:' || url.protocol === 'about:' || url.protocol === 'blank:') {
+  if (urlIsLocal(url)) {
     return 'no-referrer'
   }
 
@@ -525,7 +556,7 @@ function stripURLForReferrer (url, originOnly) {
   url.hash = ''
 
   // 6. If the origin-only flag is true, then:
-  if (originOnly) {
+  if (originOnly === true) {
     // 1. Set url’s path to « the empty string ».
     url.pathname = ''
 
@@ -537,45 +568,134 @@ function stripURLForReferrer (url, originOnly) {
   return url
 }
 
-function isURLPotentiallyTrustworthy (url) {
-  if (!webidl.is.URL(url)) {
+const potentialleTrustworthyIPv4RegExp = new RegExp('^(?:' +
+  '(?:127\\.)' +
+  '(?:(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[0-9])\\.){2}' +
+  '(?:25[0-5]|2[0-4][0-9]|1[0-9][0-9]|[1-9][0-9]|[1-9])' +
+')$')
+
+const potentialleTrustworthyIPv6RegExp = new RegExp('^(?:' +
+  '(?:(?:0{1,4}):){7}(?:(?:0{0,3}1))|' +
+  '(?:(?:0{1,4}):){1,6}(?::(?:0{0,3}1))|' +
+  '(?:::(?:0{0,3}1))|' +
+')$')
+
+/**
+ * Check if host matches one of the CIDR notations 127.0.0.0/8 or ::1/128.
+ *
+ * @param {string} origin
+ * @returns {boolean}
+ */
+function isOriginIPPotentiallyTrustworthy (origin) {
+  // IPv6
+  if (origin.includes(':')) {
+    // Remove brackets from IPv6 addresses
+    if (origin[0] === '[' && origin[origin.length - 1] === ']') {
+      origin = origin.slice(1, -1)
+    }
+    return potentialleTrustworthyIPv6RegExp.test(origin)
+  }
+
+  // IPv4
+  return potentialleTrustworthyIPv4RegExp.test(origin)
+}
+
+/**
+ * A potentially trustworthy origin is one which a user agent can generally
+ * trust as delivering data securely.
+ *
+ * Return value `true` means `Potentially Trustworthy`.
+ * Return value `false` means `Not Trustworthy`.
+ *
+ * @see https://w3c.github.io/webappsec-secure-contexts/#is-origin-trustworthy
+ * @param {string} origin
+ * @returns {boolean}
+ */
+function isOriginPotentiallyTrustworthy (origin) {
+  // 1. If origin is an opaque origin, return "Not Trustworthy".
+  if (origin == null || origin === 'null') {
     return false
   }
 
-  // If child of about, return true
-  if (url.href === 'about:blank' || url.href === 'about:srcdoc') {
+  // 2. Assert: origin is a tuple origin.
+  origin = new URL(origin)
+
+  // 3. If origin’s scheme is either "https" or "wss",
+  //    return "Potentially Trustworthy".
+  if (origin.protocol === 'https:' || origin.protocol === 'wss:') {
     return true
   }
 
-  // If scheme is data, return true
-  if (url.protocol === 'data:') return true
+  // 4. If origin’s host matches one of the CIDR notations 127.0.0.0/8 or
+  // ::1/128 [RFC4632], return "Potentially Trustworthy".
+  if (isOriginIPPotentiallyTrustworthy(origin.hostname)) {
+    return true
+  }
 
-  // If file, return true
-  if (url.protocol === 'file:') return true
+  // 5. If the user agent conforms to the name resolution rules in
+  //    [let-localhost-be-localhost] and one of the following is true:
 
-  return isOriginPotentiallyTrustworthy(url.origin)
+  //    origin’s host is "localhost" or "localhost."
+  if (origin.hostname === 'localhost' || origin.hostname === 'localhost.') {
+    return true
+  }
 
-  function isOriginPotentiallyTrustworthy (origin) {
-    // If origin is explicitly null, return false
-    if (origin == null || origin === 'null') return false
+  //    origin’s host ends with ".localhost" or ".localhost."
+  if (origin.hostname.endsWith('.localhost') || origin.hostname.endsWith('.localhost.')) {
+    return true
+  }
 
-    const originAsURL = new URL(origin)
+  // 6. If origin’s scheme is "file", return "Potentially Trustworthy".
+  if (origin.protocol === 'file:') {
+    return true
+  }
 
-    // If secure, return true
-    if (originAsURL.protocol === 'https:' || originAsURL.protocol === 'wss:') {
-      return true
-    }
+  // 7. If origin’s scheme component is one which the user agent considers to
+  // be authenticated, return "Potentially Trustworthy".
 
-    // If localhost or variants, return true
-    if (/^127(?:\.[0-9]+){0,2}\.[0-9]+$|^\[(?:0*:)*?:?0*1\]$/.test(originAsURL.hostname) ||
-     (originAsURL.hostname === 'localhost' || originAsURL.hostname.includes('localhost.')) ||
-     (originAsURL.hostname.endsWith('.localhost'))) {
-      return true
-    }
+  // 8. If origin has been configured as a trustworthy origin, return
+  //    "Potentially Trustworthy".
 
-    // If any other, return false
+  // 9. Return "Not Trustworthy".
+  return false
+}
+
+/**
+ * A potentially trustworthy URL is one which either inherits context from its
+ * creator (about:blank, about:srcdoc, data) or one whose origin is a
+ * potentially trustworthy origin.
+ *
+ * Return value `true` means `Potentially Trustworthy`.
+ * Return value `false` means `Not Trustworthy`.
+ *
+ * @see https://www.w3.org/TR/secure-contexts/#is-url-trustworthy
+ * @param {URL} url
+ * @returns {boolean}
+ */
+function isURLPotentiallyTrustworthy (url) {
+  // Given a URL record (url), the following algorithm returns "Potentially
+  // Trustworthy" or "Not Trustworthy" as appropriate:
+  if (!webidl.is.URL(url)) {
     return false
   }
+
+  // 1. If url is "about:blank" or "about:srcdoc",
+  //    return "Potentially Trustworthy".
+  if (url.href === 'about:blank' || url.href === 'about:srcdoc') {
+    return true
+  }
+
+  // 2. If url’s scheme is "data", return "Potentially Trustworthy".
+  if (url.protocol === 'data:') return true
+
+  // Note: The origin of blob: URLs is the origin of the context in which they
+  // were created. Therefore, blobs created in a trustworthy origin will
+  // themselves be potentially trustworthy.
+  if (url.protocol === 'blob:') return true
+
+  // 3. Return the result of executing § 3.1 Is origin potentially trustworthy?
+  // on url’s origin.
+  return isOriginPotentiallyTrustworthy(url.origin)
 }
 
 /**
@@ -1161,12 +1281,15 @@ async function readAllBytes (reader, successSteps, failureSteps) {
 /**
  * @see https://fetch.spec.whatwg.org/#is-local
  * @param {URL} url
+ * @returns {boolean}
  */
 function urlIsLocal (url) {
   assert('protocol' in url) // ensure it's a url object
 
   const protocol = url.protocol
 
+  // A URL is local if its scheme is a local scheme.
+  // A local scheme is "about", "blob", or "data".
   return protocol === 'about:' || protocol === 'blob:' || protocol === 'data:'
 }
 
@@ -1654,5 +1777,6 @@ module.exports = {
   extractMimeType,
   getDecodeSplit,
   utf8DecodeBytes,
-  environmentSettingsObject
+  environmentSettingsObject,
+  isOriginIPPotentiallyTrustworthy
 }