test: hermetic W3C N3 reasoning spec test suite

.github/workflows/nodejs.yml

@@ -56,6 +56,34 @@ jobs:
         shell: bash
         command: npm test
 
+  # W3C N3 reasoning spec tests. The upstream test suite is NOT committed to
+  # this repo: it is fetched at a pinned commit by scripts/fetch-n3-tests.ts
+  # and cached below with actions/cache so runs stay deterministic and fast.
+  # This job is intentionally NOT in the release job's `needs`: known EYE
+  # deviations are tracked in __tests__/spec/skip-list.json, and any newly
+  # surfacing deviation should be triaged there rather than blocking releases.
+  spec-tests:
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v6
+    - uses: actions/setup-node@v6
+      with:
+        node-version: 24.x
+    - run: npm ci
+    - uses: actions/download-artifact@v8
+      with:
+        name: dist
+        path: dist
+    # Cache the fetched W3C N3 test suite, keyed on the fetch script (which
+    # pins the upstream commit) so bumping the pin busts the cache.
+    - uses: actions/cache@v4
+      with:
+        path: __tests__/spec/.w3c-n3-tests
+        key: w3c-n3-tests-${{ hashFiles('scripts/fetch-n3-tests.ts') }}
+    - run: npm run test:spec
+
   benchmark:
     needs: build
     strategy:

.gitignore

@@ -6,3 +6,4 @@ lib/eye.ts
 lib/lingua.ts
 perf/output.txt
 perf/dynamic.mjs
+__tests__/spec/.w3c-n3-tests

__tests__/spec/README.md

@@ -0,0 +1,68 @@
+# W3C N3 reasoning spec tests
+
+This directory contains a jest runner for the reasoning part of the
+[W3C N3 test suite](https://github.com/w3c/N3/tree/master/tests) (issue #336).
+
+```
+npm run test:spec
+```
+
+It is a **separate jest project** (`jest.spec.config.json`) so that it does not
+interfere with the coverage-thresholded unit test suite, and it needs `dist`
+to be built first (`npm run build`).
+
+## How it works
+
+- The upstream test suite is **not committed to this repo**. `npm run test:spec`
+  first runs `npm run spec:fetch` (`scripts/fetch-n3-tests.ts`), which downloads
+  `tests/N3Tests` from [`w3c/N3`](https://github.com/w3c/N3) at a **pinned
+  commit** into the git-ignored `__tests__/spec/.w3c-n3-tests/`. Fetching is a
+  no-op once the directory is present, and in CI that directory is persisted
+  with `actions/cache` (keyed on `scripts/fetch-n3-tests.ts`, which holds the
+  pin) so runs are deterministic and cache-hit fast.
+- `spec-test.ts` parses the manifest with the `n3` parser, runs each
+  `test:TestN3Reason` entry through `n3reasoner`, and compares the derived
+  graph against the reference result with RDF isomorphism (`jest-rdf` /
+  `rdf-isomorphic`).
+- The cwm-style `test:options` are mapped onto EYE output modes:
+
+  | options | `n3reasoner` output |
+  |---|---|
+  | `test:conclusions` | `derivations` (`--pass-only-new`) |
+  | `test:data` | `deductive_closure` (`--pass`) |
+  | neither | `deductive_closure_plus_rules` (`--pass-all`) |
+
+  `test:think` (fixpoint) vs `test:rules` (single pass) cannot be
+  distinguished — EYE always reasons to fixpoint.
+
+## Skip list
+
+`skip-list.json` records every known deviation between EYE and the reference
+results, keyed by manifest entry id, with a reason. Skipped entries show up as
+skipped (not failed) so CI stays green while deviations are triaged. To triage
+one: remove it from the skip list, run `npm run test:spec`, and inspect the
+diff.
+
+## Updating the pinned version
+
+Bump `N3_TESTS_COMMIT` in `scripts/fetch-n3-tests.ts`. That value both drives
+the download and is hashed into the CI cache key, so a new pin refreshes the
+cache automatically. Delete `__tests__/spec/.w3c-n3-tests/` locally to force a
+re-fetch.
+
+## Known upstream manifest typo
+
+The pinned manifest concatenates two entries without whitespace in its
+`mf:entries` list — `:cwm_includes_t4:cwm_includes_t6` — which Turtle parses as
+a single, undefined prefixed name (`:` is legal in local names), so both tests
+would silently vanish. `spec-test.ts` applies a small, self-healing in-code
+patch (insert the missing space) before parsing the fetched manifest; it
+becomes a no-op once upstream fixes it. Filed upstream as
+[w3c-cg/N3#232](https://github.com/w3c-cg/N3/issues/232).
+
+## License
+
+The fetched tests are used under the
+[W3C Test Suite License](https://www.w3.org/Consortium/Legal/2008/04-testsuite-license)
+/ [W3C Software and Document License](https://www.w3.org/Consortium/Legal/copyright-software).
+They are downloaded at test time, not redistributed in this repository.

__tests__/spec/skip-list.json

@@ -0,0 +1,44 @@
+{
+  "$comment": "Known deviations between EYE (via n3reasoner) and the reference results of the W3C N3 reasoning test suite. Every entry maps a test id from w3c-n3-tests/manifest-reasoner.ttl to the reason it is skipped. Remove an entry once the deviation is resolved; the test then runs again in CI. Counts at the time of writing: 50/89 manifest entries pass, 39 skipped.",
+  "skip": {
+    "cwm_includes_conclusion_simple": "log:conclusion spawns a sub-reasoner, which needs shell/2 / exec — not available in the WASM build (see eye-js#873 and eyereasoner/eye#177)",
+    "cwm_includes_conclusion": "test loads a second file (cwm_other/daml-ex.n3) via log:semantics; n3reasoner only exposes the single input document to the reasoner FS",
+    "cwm_includes_t10": "test dereferences a sibling file (t10a.n3) via log:semantics; n3reasoner only exposes the single input document to the reasoner FS",
+    "cwm_includes_t11": "test dereferences a sibling file (foo.n3) via log:semantics; n3reasoner only exposes the single input document to the reasoner FS",
+    "cwm_includes_t6": "reference result file does not declare the prefixes it uses, so it cannot be parsed standalone",
+    "cwm_includes_t8": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got literal)",
+    "cwm_includes_builtins": "EYE derives a different graph than the cwm reference for this combination of log builtins",
+    "cwm_includes_t9br": "EYE derives a different graph than the cwm reference (log:includes over nested formulae)",
+    "cwm_includes_quantifiers_limited": "EYE handles the explicit quantifiers in this test differently than the cwm reference",
+    "cwm_includes_xsd": "uses test:strings (log:outputString concatenated plain-text output), which is not exposed through the n3reasoner API",
+    "log_content": "log:content on a datatyped literal raises type_error(atom, ...) in sub_atom/5 in the WASM build",
+    "log_parsedAsN3": "EYE derives a different graph than the reference for log:parsedAsN3",
+    "math_absoluteValue": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_big": "EYE misses some numeric edge cases of the reference (big number arithmetic)",
+    "math_ceiling": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_combo": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got .)",
+    "math_difference": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_exponentiation": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_floor": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_inf": "EYE raises zero_division for math:quotient on [0.0 0.0] instead of deriving INF like the reference",
+    "math_product": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_quotient": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_remainder": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_rounded": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_strings": "EYE misses some numeric edge cases of the reference (numeric/string conversion)",
+    "math_sum": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_trig": "EYE misses some numeric edge cases of the reference (trigonometry edge cases)",
+    "cwm_list_bug2": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got .)",
+    "cwm_list_r1": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got .)",
+    "cwm_list_unify5": "EYE derives a different graph than the cwm reference (list unification)",
+    "cwm_reason_t6": "EYE derives a different graph than the cwm reference",
+    "string_concatenation": "EYE derives a different graph than the reference for string:concatenation",
+    "string_format": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got .)",
+    "cwm_string_roughly": "EYE derives a different graph than the cwm reference (string:containsRoughly)",
+    "cwm_string_uriEncode": "reference result is not parseable by the n3.js parser (Expected entity but got literal)",
+    "cwm_supports_simple": "EYE derives a different graph than the cwm reference (log:supports)",
+    "cwm_time_t1": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got literal)",
+    "cwm_unify_unify1": "EYE derives a different graph than the cwm reference (formula unification)",
+    "cwm_unify_unify2": "EYE derives a different graph than the cwm reference (formula unification)"
+  }
+}

__tests__/spec/spec-test.ts

@@ -0,0 +1,182 @@
+/**
+ * W3C N3 reasoning spec tests.
+ *
+ * Runs every entry of `manifest-reasoner.ttl` from the W3C N3 test suite
+ * (https://github.com/w3c/N3) through `n3reasoner` and compares the derived
+ * graph against the reference results using RDF isomorphism.
+ *
+ * The test suite is NOT committed to this repo — it is fetched at a pinned
+ * commit into a git-ignored cache directory by `scripts/fetch-n3-tests.ts`
+ * (run via `npm run spec:fetch`, which `npm run test:spec` invokes first, and
+ * cached in CI with actions/cache). If the suite is missing, this file throws
+ * with a pointer to that script.
+ *
+ * Known deviations between EYE and the reference results are recorded in
+ * ./skip-list.json (with a reason each) so that this suite stays green while
+ * deviations are triaged individually.
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { DataFactory, Parser, Store } from 'n3';
+import type { Quad, Term } from '@rdfjs/types';
+import 'jest-rdf';
+import { n3reasoner } from '../../dist';
+import type { ICoreQueryOptions } from '../../dist';
+import { cacheDir } from '../../scripts/fetch-n3-tests';
+
+const { namedNode, quad } = DataFactory;
+
+const TESTS_DIR = cacheDir;
+const MANIFEST = 'manifest-reasoner.ttl';
+// The base against which the manifest is published; used to resolve the
+// mf:action / mf:result IRIs back to files in the fetched suite.
+const MANIFEST_BASE = 'https://w3c.github.io/N3/tests/N3Tests/';
+// `n3reasoner` loads string input into the reasoner as `data_0.n3s`, so this
+// is the base that EYE resolves relative IRIs in the input against. Parsing
+// the expected results against the same base keeps the two graphs aligned.
+const REASONER_BASE = 'file:///data_0.n3s';
+
+const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
+const MF = 'http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#';
+const TEST = 'https://w3c.github.io/N3/tests/test.n3#';
+
+/**
+ * Rewrites every IRI starting with `from` so that it starts with `to`
+ * instead (recursing into quoted triples). The reasoner sees its input as
+ * `data_0.n3s` while the reference results resolve relative IRIs against the
+ * original file name of the test input, so the two graphs only align after
+ * rebasing one onto the other.
+ */
+function rebaseTerm(term: Term, from: string, to: string): Term {
+  if (term.termType === 'NamedNode' && term.value.startsWith(from)) {
+    return namedNode(`${to}${term.value.slice(from.length)}`);
+  }
+  if (term.termType === 'Quad') {
+    const q = term as Quad;
+    return quad(
+      rebaseTerm(q.subject, from, to) as Quad['subject'],
+      rebaseTerm(q.predicate, from, to) as Quad['predicate'],
+      rebaseTerm(q.object, from, to) as Quad['object'],
+      rebaseTerm(q.graph, from, to) as Quad['graph'],
+    );
+  }
+  return term;
+}
+
+const skipList: Record<string, string> = JSON.parse(
+  fs.readFileSync(path.join(__dirname, 'skip-list.json'), 'utf-8'),
+).skip;
+
+interface SpecTest {
+  id: string;
+  name: string;
+  action: string;
+  result: string;
+  output: ICoreQueryOptions['output'];
+}
+
+function loadManifest(): SpecTest[] {
+  const manifestPath = path.join(TESTS_DIR, MANIFEST);
+  if (!fs.existsSync(manifestPath)) {
+    throw new Error(
+      `W3C N3 test suite not found at ${TESTS_DIR}. `
+      + 'Run `npm run spec:fetch` first (`npm run test:spec` does this automatically).',
+    );
+  }
+
+  let manifestText = fs.readFileSync(manifestPath, 'utf-8');
+  // Upstream typo (filed as w3c-cg/N3#232): two entries are concatenated
+  // without whitespace in the mf:entries list — `:cwm_includes_t4:cwm_includes_t6`
+  // — which Turtle parses as a single, undefined prefixed name rather than two
+  // list items, so both tests silently vanish. Patch it here on the fetched
+  // copy until upstream lands the fix; self-healing (a no-op once fixed).
+  manifestText = manifestText.replace(
+    ':cwm_includes_t4:cwm_includes_t6',
+    ':cwm_includes_t4 :cwm_includes_t6',
+  );
+
+  const store = new Store(new Parser({
+    format: 'text/n3',
+    baseIRI: `${MANIFEST_BASE}${MANIFEST}`,
+  }).parse(manifestText));
+
+  function one(subject: Term | undefined, predicate: string): Term | undefined {
+    return subject && store.getObjects(subject as Parameters<Store['getObjects']>[0], namedNode(predicate), null)[0];
+  }
+
+  // Walk the rdf:List of mf:entries
+  const entries: Term[] = [];
+  let head = one(namedNode(`${MANIFEST_BASE}${MANIFEST}`), `${MF}entries`);
+  while (head && head.value !== `${RDF}nil`) {
+    const first = one(head, `${RDF}first`);
+    if (first) {
+      entries.push(first);
+    }
+    head = one(head, `${RDF}rest`);
+  }
+
+  return entries.map((entry) => {
+    const options = one(entry, `${TEST}options`);
+    const flag = (name: string): boolean => one(options, `${TEST}${name}`)?.value === 'true';
+
+    // Map the cwm-style test options onto the EYE output modes:
+    //  - test:conclusions ("replace store with conclusions") -> derivations
+    //  - test:data ("remove all except plain RDF triples")   -> deductive_closure
+    //  - neither (full store, rules included)                -> deductive_closure_plus_rules
+    // test:think (fixpoint) vs test:rules (single pass) cannot be
+    // distinguished: EYE always reasons to fixpoint.
+    let output: ICoreQueryOptions['output'];
+    if (flag('conclusions')) {
+      output = 'derivations';
+    } else if (flag('data')) {
+      output = 'deductive_closure';
+    } else {
+      output = 'deductive_closure_plus_rules';
+    }
+
+    return {
+      id: entry.value.replace(`${MANIFEST_BASE}${MANIFEST}#`, ''),
+      name: one(entry, `${MF}name`)?.value ?? '(unnamed)',
+      action: one(entry, `${MF}action`)?.value.replace(MANIFEST_BASE, '') ?? '',
+      result: one(entry, `${MF}result`)?.value.replace(MANIFEST_BASE, '') ?? '',
+      output,
+    };
+  });
+}
+
+describe('W3C N3 reasoning spec tests (manifest-reasoner.ttl)', () => {
+  const tests = loadManifest();
+
+  it('loads all test entries from the manifest', () => {
+    expect(tests.length).toBeGreaterThan(0);
+  });
+
+  for (const test of tests) {
+    const runner = test.id in skipList ? it.skip : it;
+    // eslint-disable-next-line no-loop-func
+    runner(`${test.id} — ${test.name}`, async () => {
+      const action = fs.readFileSync(path.join(TESTS_DIR, test.action), 'utf-8');
+      const expectedText = fs.readFileSync(path.join(TESTS_DIR, test.result), 'utf-8');
+
+      // The reference results resolve relative IRIs against the original
+      // name of the input file (they were generated by running the reasoner
+      // on e.g. `reflexive.n3` in its own directory), while `n3reasoner`
+      // loads its input as `data_0.n3s`. Parse the reference against the
+      // input's file name and rebase the actual output onto the same base.
+      const expectedBase = `file:///${path.basename(test.action)}`;
+
+      const actual = await n3reasoner(action, undefined, {
+        output: test.output,
+        outputType: 'quads',
+      });
+      const expected = new Parser({
+        format: 'text/n3',
+        baseIRI: expectedBase,
+      }).parse(expectedText);
+
+      const rebased = actual.map((q) => rebaseTerm(q, REASONER_BASE, expectedBase) as Quad);
+
+      expect(rebased).toBeRdfIsomorphic(expected);
+    });
+  }
+});

jest.config.json

@@ -1,5 +1,6 @@
 {
   "preset": "ts-jest",
+  "testPathIgnorePatterns": ["/node_modules/", "<rootDir>/__tests__/spec/"],
   "collectCoverage": true,
   "coverageReporters": [
     "json-summary", 

jest.spec.config.json

@@ -0,0 +1,6 @@
+{
+  "preset": "ts-jest",
+  "testMatch": ["<rootDir>/__tests__/spec/**/*-test.ts"],
+  "testTimeout": 90000,
+  "maxWorkers": 1
+}

package.json

@@ -15,6 +15,8 @@
     "test:badges": "npm run test:coverage  && jest-coverage-badges",
     "test:unit": "npm run bundle:webpack && node --experimental-vm-modules node_modules/jest/bin/jest.js",
     "test:cli": "npm run bundle:webpack && node --experimental-vm-modules node_modules/jest/bin/jest.js cli-test",
+    "spec:fetch": "ts-node scripts/fetch-n3-tests",
+    "test:spec": "npm run spec:fetch && node --experimental-vm-modules node_modules/jest/bin/jest.js --config jest.spec.config.json",
     "test:memory": "npm run test:memory:node && npm run test:memory:node:error && npm run test:memory:browser",
     "test:memory:node": "node __tests_memory__/leakTest",
     "test:memory:node:error": "node __tests_memory__/leakTestOnError",