Fix silent corruption for large backreference offsets

README.md

@@ -94,4 +94,4 @@ Unlike my Zlib implementation [`fflate`](https://github.com/101arrowz/fflate), W
 - Some WASM ports cannot operate without being provided the decompressed size of the data in advance. `fzstd` decides how much memory to allocate from the frame headers.
 - `fzstd` is absolutely tiny: at **8kB minified and 3.8kB after gzipping**, it's much smaller than most WASM implementations.
 
-Please note that unlike the reference implementation, `fzstd` only supports a maximum backreference distance of 2<sup>25</sup> bytes. If you need to decompress files with an "ultra" compression level (20 or greater) AND your files can be above 32MB decompressed, `fzstd` may fail to decompress properly. Consider using a WebAssembly port for files this large.
+Backreference distances are decoded up to the maximum frame window accepted by `fzstd` (about 2 GB). Frames that use `--long` or ultra compression levels can therefore be decoded when their offsets stay within that limit. Very large frames remain subject to JavaScript typed-array, memory, and frame-size limits, so a WebAssembly implementation may still be a better fit for multi-hundred-megabyte inputs.

src/index.ts

@@ -541,9 +541,12 @@ const rzb = (dat: Uint8Array, st: DZstdState, out?: Uint8Array) => {
         const ofc = oct.s[ost];
         const obtr = oct.n[ost];
 
+        if (ofc > 30) err(4);
         cbt = (spos -= ofc) >> 3;
-        const ofp = 1 << ofc;
-        let off = ofp + (((dat[cbt] | (dat[cbt + 1] << 8) | (dat[cbt + 2] << 16) | (dat[cbt + 3] << 24)) >>> (spos & 7)) & (ofp - 1));
+        const sh = spos & 7, ofp = 1 << ofc, ofm = ofp - 1;
+        let off = ofp + (((dat[cbt] | (dat[cbt + 1] << 8) | (dat[cbt + 2] << 16) | (dat[cbt + 3] << 24)) >>> sh) & ofm);
+        // Four bytes only guarantee 25 bits; large offsets can need a fifth.
+        if (sh + ofc > 32) off += (dat[cbt + 4] << (32 - sh)) & ofm;
         cbt = (spos -= mlb[mlc]) >> 3;
         let ml = mlbl[mlc] + (((dat[cbt] | (dat[cbt + 1] << 8) | (dat[cbt + 2] << 16)) >> (spos & 7)) & ((1 << mlb[mlc]) - 1));
         cbt = (spos -= llb[llc]) >> 3;

tests/README.md

@@ -1,9 +1,20 @@
-# fzstd Deno tests
+# fzstd tests
 
-This folder contains test cases for fzstd that can be run with [Deno](https://deno.land/)
+This folder contains test cases for fzstd.
 
 ## How to execute tests
 
+Original cases, run with [Deno](https://deno.land/):
+
+```bash
+deno test --no-check tests/simple_cases_test.ts
+```
+
+Streaming and large-offset regression cases, run with [Bun](https://bun.sh/):
+
 ```bash
-deno test --no-check
+bun test tests/streaming_regression_test.ts
+
+# Optional large-offset test (up to ~800 MB peak RAM, needs zstd on PATH).
+FZSTD_BIG_TESTS=1 bun test tests/streaming_regression_test.ts
 ```

tests/streaming_regression_test.ts

@@ -0,0 +1,148 @@
+// Regression tests for streaming decompression (issue #19 scenario) and
+// for large backreference offsets that cross a 4-byte bit-reading window.
+// Run with: bun test tests/streaming_regression_test.ts
+// The embedded frames are real zstd CLI output; expected payloads are
+// regenerated in JS, so no zstd binary is needed for the core tests.
+import { describe, it, expect } from 'bun:test';
+import * as fzstd from '../src/index.ts';
+
+const hex = (s: string) => new Uint8Array(s.match(/../g).map(b => parseInt(b, 16)));
+const enc = new TextEncoder();
+
+// "hello world " * 11000 (= 132000 B: blocks of 131072 + 928), zstd -3,
+// single-segment frame WITH xxh64 checksum. Same bytes as Go stdlib fixture
+// testdata/f2a8e35c.helloworld-11000x.zst (golang.org/cl/531255).
+const HW11000_CHECK = hex(
+  '28b52ffda4a0030200ac00006868656c6c6f20776f726c6420680100f0ffcfcb173d00000001009d5f1520d2e9d158'
+);
+// Same payload, zstd -3 --no-check via stdin: window-descriptor frame,
+// no content size, NO checksum. Frames like this ended exactly at the last
+// block and starved the pre-0.1.1 push() gate (fixed in 807a854).
+const HW11000_NOCHECK = hex(
+  '28b52ffd0058a400006068656c6c6f20776f726c64200100f1ffcf4b12450000087201009c2b2004'
+);
+// "hello world " * 21845 (= 262140 B: 131072 + 131068), zstd -3 --no-check.
+const HW21845_NOCHECK = hex(
+  '28b52ffd0058a400006068656c6c6f20776f726c64200100f1ffcf4b124d000008720100f8ff391002'
+);
+
+const CASES: [string, Uint8Array, Uint8Array][] = [
+  ['hw11000+checksum', HW11000_CHECK, enc.encode('hello world '.repeat(11000))],
+  ['hw11000 no-check', HW11000_NOCHECK, enc.encode('hello world '.repeat(11000))],
+  ['hw21845 no-check', HW21845_NOCHECK, enc.encode('hello world '.repeat(21845))],
+];
+
+// Streams data into a Decompress instance and returns the joined output.
+// mode: 'final-on-last' | 'empty-final' | 'never-final'
+const stream = (chunks: Uint8Array[], mode: string) => {
+  const parts: Uint8Array[] = [];
+  let sawFinal = false;
+  const d = new fzstd.Decompress((c, f) => { parts.push(c); if (f) sawFinal = true; });
+  for (let i = 0; i < chunks.length; ++i) {
+    d.push(chunks[i], mode == 'final-on-last' && i == chunks.length - 1);
+  }
+  if (mode == 'empty-final') d.push(new Uint8Array(0), true);
+  const out = new Uint8Array(parts.reduce((s, p) => s + p.length, 0));
+  let o = 0;
+  for (const p of parts) { out.set(p, o); o += p.length; }
+  return { out, sawFinal };
+};
+
+describe('issue #19 scenario: >128KiB output via streaming', () => {
+  for (const [name, zst, want] of CASES) {
+    it(`${name}: one-shot`, () => {
+      expect(fzstd.decompress(zst)).toEqual(want);
+    });
+
+    it(`${name}: single push with final=true`, () => {
+      const { out, sawFinal } = stream([zst], 'final-on-last');
+      expect(out).toEqual(want);
+      expect(sawFinal).toBe(true);
+    });
+
+    it(`${name}: push all, then empty final push`, () => {
+      const { out, sawFinal } = stream([zst], 'empty-final');
+      expect(out).toEqual(want);
+      expect(sawFinal).toBe(true);
+    });
+
+    it(`${name}: push without ever sending final`, () => {
+      // The #19 reporter's adapter never pushed final; all data must still
+      // be delivered through ondata.
+      expect(stream([zst], 'never-final').out).toEqual(want);
+    });
+
+    it(`${name}: every two-chunk split, all final modes`, () => {
+      // Splits at/after the last block's start starved fzstd <= 0.1.0 on
+      // checksum-less frames: silent truncation to 131072 bytes without
+      // final, "unexpected EOF" with final.
+      for (let s = 1; s < zst.length; ++s) {
+        const chunks = [zst.subarray(0, s), zst.subarray(s)];
+        for (const mode of ['final-on-last', 'empty-final', 'never-final']) {
+          expect(stream(chunks, mode).out).toEqual(want);
+        }
+      }
+    });
+
+    it(`${name}: one byte per push`, () => {
+      const chunks: Uint8Array[] = [];
+      for (let i = 0; i < zst.length; ++i) chunks.push(zst.subarray(i, i + 1));
+      expect(stream(chunks, 'empty-final').out).toEqual(want);
+    });
+  }
+});
+
+describe('multi-frame streams', () => {
+  const a = CASES[0], b = CASES[1];
+  const joined = new Uint8Array(a[1].length + b[1].length);
+  joined.set(a[1], 0), joined.set(b[1], a[1].length);
+  const want = new Uint8Array(a[2].length + b[2].length);
+  want.set(a[2], 0), want.set(b[2], a[2].length);
+
+  it('decodes both frames one-shot', () => {
+    expect(fzstd.decompress(joined)).toEqual(want);
+  });
+
+  it('decodes both frames streaming at every two-chunk split', () => {
+    for (let s = 1; s < joined.length; ++s) {
+      const chunks = [joined.subarray(0, s), joined.subarray(s)];
+      expect(stream(chunks, 'empty-final').out).toEqual(want);
+    }
+  });
+});
+
+// The first affected distance is 2^26 + 2^25 - 3 (about 96 MiB): the 4-byte
+// offset read supplies only 32 - (spos & 7) bits. The test can use up to
+// ~800 MB peak RAM and needs zstd on PATH, so it is opt-in.
+const bigTest = process.env.FZSTD_BIG_TESTS ? it : it.skip;
+describe('large backreference offsets (opt-in large test)', () => {
+  bigTest('decodes a --long=27 frame with ~100MB match distances', async () => {
+    const { spawnSync } = await import('child_process');
+    const probe = spawnSync('zstd', ['--version']);
+    if (probe.error) throw new Error('zstd CLI not found on PATH; cannot run big offset test');
+    // 100 MiB of seeded SplitMix32 noise + 500 short copies scattered from near the
+    // start: each copy needs a fresh ~100 MiB offset (no repeat-offset reuse),
+    // so offset code 26 gets read at many bit alignments, some straddling the
+    // 4-byte window that pre-fix code was limited to.
+    const M = 1 << 20;
+    const size = 100 * M;
+    const buf = new Uint8Array(size + 500 * 4000);
+    // splitmix32: unlike an LCG, has no short-period low/mid bits that zstd
+    // would match at small offsets
+    for (let i = 0, z = 0; i < size; ++i) {
+      z = (z + 0x9E3779B9) | 0;
+      let x = z;
+      x = Math.imul(x ^ (x >>> 16), 0x21F0AAAD);
+      x = Math.imul(x ^ (x >>> 15), 0x735A2D97);
+      buf[i] = x ^ (x >>> 15);
+    }
+    for (let i = 0; i < 500; ++i) {
+      buf.set(buf.subarray(i * 4096, i * 4096 + 4000), size + i * 4000);
+    }
+    const z = spawnSync('zstd', ['-1', '--long=27', '-c'], { input: buf, maxBuffer: 1 << 30 });
+    if (z.status !== 0) throw new Error('zstd failed: ' + z.stderr);
+    const out = fzstd.decompress(new Uint8Array(z.stdout));
+    expect(out.length).toBe(buf.length);
+    expect(Buffer.compare(out, buf)).toBe(0);
+  }, 300000);
+});