feat: add initial zstd support

EDIT(@isaacs): Remove node 20 from ci workflow, since Zstd is not
available until node 22. The failure mode is handled in minizlib anyway.

PR-URL: #439
Credit: @JamieMagee
Close: #439
Reviewed-by: @isaacs

Author: JamieMagee

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ jobs:
   build:
     strategy:
       matrix:
-        node-version: [20.x, 22.x, 24.x]
+        node-version: [22.x, 24.x]
         platform:
         - os: ubuntu-latest
           shell: bash

package-lock.json

@@ -28,6 +28,7 @@
     "yallist": "^5.0.0"
   },
   "devDependencies": {
+    "@types/node": "^22.15.29",
     "chmodr": "^1.2.0",
     "end-of-stream": "^1.4.3",
     "events-to-array": "^2.0.3",

package.json

@@ -115,14 +115,31 @@ export interface TarOptions {
    *
    * If set `false`, then brotli options will not be used.
    *
-   * If both this and the `gzip` option are left `undefined`, then tar will
-   * attempt to infer the brotli compression status, but can only do so based
-   * on the filename. If the filename ends in `.tbr` or `.tar.br`, and the
-   * first 512 bytes are not a valid tar header, then brotli decompression
+   * If this, the `gzip`, and `zstd` options are left `undefined`, then tar
+   * will attempt to infer the brotli compression status, but can only do so
+   * based on the filename. If the filename ends in `.tbr` or `.tar.br`, and
+   * the first 512 bytes are not a valid tar header, then brotli decompression
    * will be attempted.
    */
   brotli?: boolean | ZlibOptions
 
+  /**
+   * Set to `true` or an object with settings for `zstd.compress()` to
+   * create a zstd-compressed archive
+   * 
+   * When extracting, this will cause the archive to be treated as a
+   * zstd-compressed file if set to `true` or a ZlibOptions object.
+   * 
+   * If set `false`, then zstd options will not be used.
+   * 
+   * If this, the `gzip`, and `brotli` options are left `undefined`, then tar
+   * will attempt to infer the zstd compression status, but can only do so
+   * based on the filename. If the filename ends in `.tzst` or `.tar.zst`, and
+   * the first 512 bytes are not a valid tar header, then zstd decompression
+   * will be attempted.
+   */
+  zstd?: boolean | ZlibOptions
+
   /**
    * A function that is called with `(path, stat)` when creating an archive, or
    * `(path, entry)` when extracting. Return true to process the file/entry, or

src/options.ts

@@ -81,7 +81,7 @@ export class Pack
   statCache: Exclude<TarOptions['statCache'], undefined>
   file: string
   portable: boolean
-  zip?: zlib.BrotliCompress | zlib.Gzip
+  zip?: zlib.BrotliCompress | zlib.Gzip | zlib.ZstdCompress
   readdirCache: Exclude<TarOptions['readdirCache'], undefined>
   noDirRecurse: boolean
   follow: boolean
@@ -128,9 +128,9 @@ export class Pack
 
     this.portable = !!opt.portable
 
-    if (opt.gzip || opt.brotli) {
-      if (opt.gzip && opt.brotli) {
-        throw new TypeError('gzip and brotli are mutually exclusive')
+    if (opt.gzip || opt.brotli || opt.zstd) {
+      if ((opt.gzip ? 1 : 0) + (opt.brotli ? 1 : 0) + (opt.zstd ? 1 : 0) > 1) {
+        throw new TypeError('gzip, brotli, zstd are mutually exclusive')
       }
       if (opt.gzip) {
         if (typeof opt.gzip !== 'object') {
@@ -147,6 +147,12 @@ export class Pack
         }
         this.zip = new zlib.BrotliCompress(opt.brotli)
       }
+      if (opt.zstd) {
+        if (typeof opt.zstd !== 'object') {
+          opt.zstd = {}
+        }
+        this.zip = new zlib.ZstdCompress(opt.zstd)
+      }
       /* c8 ignore next */
       if (!this.zip) throw new Error('impossible')
       const zip = this.zip

src/pack.ts

@@ -19,7 +19,7 @@
 // ignored entries get .resume() called on them straight away
 
 import { EventEmitter as EE } from 'events'
-import { BrotliDecompress, Unzip } from 'minizlib'
+import { BrotliDecompress, Unzip, ZstdDecompress } from 'minizlib'
 import { Header } from './header.js'
 import { TarOptions } from './options.js'
 import { Pax } from './pax.js'
@@ -32,6 +32,8 @@ import {
 
 const maxMetaEntrySize = 1024 * 1024
 const gzipHeader = Buffer.from([0x1f, 0x8b])
+const zstdHeader = Buffer.from([0x28, 0xb5, 0x2f, 0xfd])
+const ZIP_HEADER_LEN = Math.max(gzipHeader.length, zstdHeader.length)
 
 const STATE = Symbol('state')
 const WRITEENTRY = Symbol('writeEntry')
@@ -74,6 +76,7 @@ export class Parser extends EE implements Warner {
   maxMetaEntrySize: number
   filter: Exclude<TarOptions['filter'], undefined>
   brotli?: TarOptions['brotli']
+  zstd?: TarOptions['zstd']
 
   writable: true = true
   readable: false = false;
@@ -87,7 +90,7 @@ export class Parser extends EE implements Warner {
   [EX]?: Pax;
   [GEX]?: Pax;
   [ENDED]: boolean = false;
-  [UNZIP]?: false | Unzip | BrotliDecompress;
+  [UNZIP]?: false | Unzip | BrotliDecompress | ZstdDecompress;
   [ABORTED]: boolean = false;
   [SAW_VALID_ENTRY]?: boolean;
   [SAW_NULL_BLOCK]: boolean = false;
@@ -135,9 +138,19 @@ export class Parser extends EE implements Warner {
     // if it's a tbr file it MIGHT be brotli, but we don't know until
     // we look at it and verify it's not a valid tar file.
     this.brotli =
-      !opt.gzip && opt.brotli !== undefined ? opt.brotli
+      !(opt.gzip || opt.zstd) && opt.brotli !== undefined ? opt.brotli
       : isTBR ? undefined
-      : false
+        : false
+
+    // zstd has magic bytes to identify it, but we also support explicit options
+    // and file extension detection
+    const isTZST =
+      opt.file &&
+      (opt.file.endsWith('.tar.zst') || opt.file.endsWith('.tzst'))
+    this.zstd =
+      !(opt.gzip || opt.brotli) && opt.zstd !== undefined ? opt.zstd
+      : isTZST ? true
+        : undefined
 
     // have to set this so that streams are ok piping into it
     this.on('end', () => this[CLOSESTREAM]())
@@ -431,7 +444,7 @@ export class Parser extends EE implements Warner {
       return false
     }
 
-    // first write, might be gzipped
+    // first write, might be gzipped, zstd, or brotli compressed
     const needSniff =
       this[UNZIP] === undefined ||
       (this.brotli === undefined && this[UNZIP] === false)
@@ -440,7 +453,7 @@ export class Parser extends EE implements Warner {
         chunk = Buffer.concat([this[BUFFER], chunk])
         this[BUFFER] = undefined
       }
-      if (chunk.length < gzipHeader.length) {
+      if (chunk.length < ZIP_HEADER_LEN) {
         this[BUFFER] = chunk
         /* c8 ignore next */
         cb?.()
@@ -458,7 +471,19 @@ export class Parser extends EE implements Warner {
         }
       }
 
-      const maybeBrotli = this.brotli === undefined
+      // look for zstd header if gzip header not found
+      let isZstd = false
+      if (this[UNZIP] === false && this.zstd !== false) {
+        isZstd = true
+        for (let i = 0; i < zstdHeader.length; i++) {
+          if (chunk[i] !== zstdHeader[i]) {
+            isZstd = false
+            break
+          }
+        }
+      }
+
+      const maybeBrotli = this.brotli === undefined && !isZstd
       if (this[UNZIP] === false && maybeBrotli) {
         // read the first header to see if it's a valid tar file. If so,
         // we can safely assume that it's not actually brotli, despite the
@@ -487,13 +512,15 @@ export class Parser extends EE implements Warner {
 
       if (
         this[UNZIP] === undefined ||
-        (this[UNZIP] === false && this.brotli)
+        (this[UNZIP] === false && (this.brotli || isZstd))
       ) {
         const ended = this[ENDED]
         this[ENDED] = false
         this[UNZIP] =
           this[UNZIP] === undefined ?
             new Unzip({})
+          : isZstd ?
+            new ZstdDecompress({})
           : new BrotliDecompress({})
         this[UNZIP].on('data', chunk => this[CONSUMECHUNK](chunk))
         this[UNZIP].on('error', er => this.abort(er as Error))
@@ -674,7 +701,7 @@ export class Parser extends EE implements Warner {
         this[UNZIP].end()
       } else {
         this[ENDED] = true
-        if (this.brotli === undefined)
+        if (this.brotli === undefined || this.zstd === undefined)
           chunk = chunk || Buffer.alloc(0)
         if (chunk) this.write(chunk)
         this[MAYBEEND]()

src/parse.ts

@@ -286,6 +286,7 @@ export const replace = makeCommand(
     if (
       opt.gzip ||
       opt.brotli ||
+      opt.zstd ||
       opt.file.endsWith('.br') ||
       opt.file.endsWith('.tbr')
     ) {

src/replace.ts

@@ -476,6 +476,77 @@ t.test('brotli', async t => {
   })
 })
 
+t.test('zstd', async t => {
+  const file = path.resolve(__dirname, 'fixtures/example.tzst')
+  const dir = path.resolve(__dirname, 'zstd')
+
+  t.beforeEach(async () => {
+    await mkdirp(dir)
+  })
+
+  t.afterEach(async () => {
+    await rimraf(dir)
+  })
+
+  t.test('succeeds based on magic bytes', async t => {
+    // copy the file to a new location with a different extension
+    const unknownExtension = path.resolve(__dirname, 'zstd/example.unknown')
+    fs.copyFileSync(file, unknownExtension)
+
+    x({ sync: true, file: unknownExtension, C: dir })
+
+    t.same(fs.readdirSync(dir + '/x').sort(), [
+      '1',
+      '10',
+      '2',
+      '3',
+      '4',
+      '5',
+      '6',
+      '7',
+      '8',
+      '9',
+    ])
+    t.end()
+  })
+
+  t.test('succeeds based on file extension', t => {
+    x({ sync: true, file: file, C: dir })
+
+    t.same(fs.readdirSync(dir + '/x').sort(), [
+      '1',
+      '10',
+      '2',
+      '3',
+      '4',
+      '5',
+      '6',
+      '7',
+      '8',
+      '9',
+    ])
+    t.end()
+  })
+
+  t.test('succeeds when passed explicit option', t => {
+    x({ sync: true, file: file, C: dir, brotli: true })
+
+    t.same(fs.readdirSync(dir + '/x').sort(), [
+      '1',
+      '10',
+      '2',
+      '3',
+      '4',
+      '5',
+      '6',
+      '7',
+      '8',
+      '9',
+    ])
+    t.end()
+  })
+})
+
 t.test('verify long linkname is not a problem', async t => {
   // See: https://github.com/isaacs/node-tar/issues/312
   const file = path.resolve(__dirname, 'fixtures/long-linkname.tar')