fix(benchmark): make it fair

Author: tsctx

benchmarks/_util/index.js

@@ -0,0 +1,53 @@
+'use strict'
+
+const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100
+
+function makeParallelRequests (cb) {
+  const promises = new Array(parallelRequests)
+  for (let i = 0; i < parallelRequests; ++i) {
+    promises[i] = new Promise(cb)
+  }
+  return Promise.all(promises)
+}
+
+function printResults (results) {
+  // Sort results by least performant first, then compare relative performances and also printing padding
+  let last
+
+  const rows = Object.entries(results)
+    // If any failed, put on the top of the list, otherwise order by mean, ascending
+    .sort((a, b) => (!a[1].success ? -1 : b[1].mean - a[1].mean))
+    .map(([name, result]) => {
+      if (!result.success) {
+        return {
+          Tests: name,
+          Samples: result.size,
+          Result: 'Errored',
+          Tolerance: 'N/A',
+          'Difference with Slowest': 'N/A'
+        }
+      }
+
+      // Calculate throughput and relative performance
+      const { size, mean, standardError } = result
+      const relative = last !== 0 ? (last / mean - 1) * 100 : 0
+
+      // Save the slowest for relative comparison
+      if (typeof last === 'undefined') {
+        last = mean
+      }
+
+      return {
+        Tests: name,
+        Samples: size,
+        Result: `${((parallelRequests * 1e9) / mean).toFixed(2)} req/sec`,
+        Tolerance: `± ${((standardError / mean) * 100).toFixed(2)} %`,
+        'Difference with slowest':
+          relative > 0 ? `+ ${relative.toFixed(2)} %` : '-'
+      }
+    })
+
+  return console.table(rows)
+}
+
+module.exports = { makeParallelRequests, printResults }

benchmarks/benchmark.js

@@ -6,22 +6,23 @@ const path = require('node:path')
 const { Writable } = require('node:stream')
 const { isMainThread } = require('node:worker_threads')
 
-const { Pool, Client, fetch, Agent, setGlobalDispatcher } = require('..')
+const { Pool, Client, fetch, Agent, setGlobalDispatcher, util } = require('..')
+
+const { makeParallelRequests, printResults } = require('./_util')
 
 let nodeFetch
 const axios = require('axios')
 let superagent
 let got
 
-const util = require('node:util')
-const _request = require('request')
-const request = util.promisify(_request)
+const { promisify } = require('node:util')
+const assert = require('node:assert')
+const request = promisify(require('request'))
 
 const iterations = (parseInt(process.env.SAMPLES, 10) || 10) + 1
 const errorThreshold = parseInt(process.env.ERROR_THRESHOLD, 10) || 3
 const connections = parseInt(process.env.CONNECTIONS, 10) || 50
 const pipelining = parseInt(process.env.PIPELINING, 10) || 10
-const parallelRequests = parseInt(process.env.PARALLEL, 10) || 100
 const headersTimeout = parseInt(process.env.HEADERS_TIMEOUT, 10) || 0
 const bodyTimeout = parseInt(process.env.BODY_TIMEOUT, 10) || 0
 const dest = {}
@@ -119,7 +120,8 @@ class SimpleRequest {
 
   onConnect (abort) { }
 
-  onHeaders (statusCode, headers, resume) {
+  onHeaders (statusCode, rawHeaders, resume) {
+    assert(util.parseHeaders(rawHeaders)['content-type'])
     this.dst.on('drain', resume)
   }
 
@@ -136,57 +138,11 @@ class SimpleRequest {
   }
 }
 
-function makeParallelRequests (cb) {
-  const promises = new Array(parallelRequests)
-  for (let i = 0; i < parallelRequests; ++i) {
-    promises[i] = new Promise(cb)
-  }
-  return Promise.all(promises)
-}
-
-function printResults (results) {
-  // Sort results by least performant first, then compare relative performances and also printing padding
-  let last
-
-  const rows = Object.entries(results)
-    // If any failed, put on the top of the list, otherwise order by mean, ascending
-    .sort((a, b) => (!a[1].success ? -1 : b[1].mean - a[1].mean))
-    .map(([name, result]) => {
-      if (!result.success) {
-        return {
-          Tests: name,
-          Samples: result.size,
-          Result: 'Errored',
-          Tolerance: 'N/A',
-          'Difference with Slowest': 'N/A'
-        }
-      }
-
-      // Calculate throughput and relative performance
-      const { size, mean, standardError } = result
-      const relative = last !== 0 ? (last / mean - 1) * 100 : 0
-
-      // Save the slowest for relative comparison
-      if (typeof last === 'undefined') {
-        last = mean
-      }
-
-      return {
-        Tests: name,
-        Samples: size,
-        Result: `${((parallelRequests * 1e9) / mean).toFixed(2)} req/sec`,
-        Tolerance: `± ${((standardError / mean) * 100).toFixed(2)} %`,
-        'Difference with slowest': relative > 0 ? `+ ${relative.toFixed(2)} %` : '-'
-      }
-    })
-
-  return console.table(rows)
-}
-
 const experiments = {
   'http - no keepalive' () {
     return makeParallelRequests(resolve => {
       http.get(httpNoKeepAliveOptions, res => {
+        assert(res.headers['content-type'])
         res
           .pipe(
             new Writable({
@@ -202,6 +158,7 @@ const experiments = {
   'http - keepalive' () {
     return makeParallelRequests(resolve => {
       http.get(httpKeepAliveOptions, res => {
+        assert(res.headers['content-type'])
         res
           .pipe(
             new Writable({
@@ -217,8 +174,8 @@ const experiments = {
   'undici - pipeline' () {
     return makeParallelRequests(resolve => {
       dispatcher
-        .pipeline(undiciOptions, data => {
-          return data.body
+        .pipeline(undiciOptions, ({ body }) => {
+          return body
         })
         .end()
         .pipe(
@@ -288,9 +245,15 @@ if (process.env.PORT) {
     })
   }
 
+  const axiosOptions = {
+    url: dest.url,
+    method: 'GET',
+    responseType: 'stream',
+    httpAgent: axiosAgent
+  }
   experiments.axios = () => {
     return makeParallelRequests(resolve => {
-      axios.get(dest.url, { responseType: 'stream', httpAgent: axiosAgent }).then(res => {
+      axios.request(axiosOptions).then(res => {
         res.data.pipe(new Writable({
           write (chunk, encoding, callback) {
             callback()
@@ -300,26 +263,37 @@ if (process.env.PORT) {
     })
   }
 
+  const gotOptions = {
+    url: dest.url,
+    method: 'GET',
+    agent: {
+      http: gotAgent
+    },
+    // avoid body processing
+    isStream: true
+  }
   experiments.got = () => {
     return makeParallelRequests(resolve => {
-      got.get(dest.url, { agent: { http: gotAgent } }).then(res => {
-        res.pipe(new Writable({
-          write (chunk, encoding, callback) {
-            callback()
-          }
-        })).on('finish', resolve)
-      }).catch(console.log)
+      got(gotOptions).pipe(new Writable({
+        write (chunk, encoding, callback) {
+          callback()
+        }
+      })).on('finish', resolve)
     })
   }
 
+  const requestOptions = {
+    url: dest.url,
+    method: 'GET',
+    agent: requestAgent,
+    // avoid body toString
+    encoding: null
+  }
   experiments.request = () => {
     return makeParallelRequests(resolve => {
-      request(dest.url, { agent: requestAgent }).then(res => {
-        res.pipe(new Writable({
-          write (chunk, encoding, callback) {
-            callback()
-          }
-        })).on('finish', resolve)
+      request(requestOptions).then(res => {
+        // already body consumed
+        resolve()
       }).catch(console.log)
     })
   }