test: hermetic W3C N3 reasoning spec test suite

.github/workflows/nodejs.yml

@@ -56,6 +56,26 @@ jobs:
         shell: bash
         command: npm test
 
+  # W3C N3 reasoning spec tests (vendored snapshot under __tests__/spec/w3c-n3-tests).
+  # This job is intentionally NOT in the release job's `needs`: known EYE deviations
+  # are tracked in __tests__/spec/skip-list.json, and any newly surfacing deviation
+  # should be triaged there rather than blocking unrelated releases.
+  spec-tests:
+    needs: build
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v6
+    - uses: actions/setup-node@v6
+      with:
+        node-version: 24.x
+    - run: npm ci
+    - uses: actions/download-artifact@v8
+      with:
+        name: dist
+        path: dist
+    - run: npm run test:spec
+
   benchmark:
     needs: build
     strategy:

__tests__/spec/README.md

@@ -0,0 +1,63 @@
+# W3C N3 reasoning spec tests
+
+This directory contains a hermetic jest runner for the reasoning part of the
+[W3C N3 test suite](https://github.com/w3c/N3/tree/master/tests) (issue #336).
+
+```
+npm run test:spec
+```
+
+It is a **separate jest project** (`jest.spec.config.json`) so that it does not
+interfere with the coverage-thresholded unit test suite, and it needs `dist`
+to be built first (`npm run build`).
+
+## How it works
+
+- `w3c-n3-tests/` is a vendored snapshot of `tests/N3Tests/manifest-reasoner.ttl`
+  plus every file it references (`mf:action` / `mf:result`) from
+  [`w3c/N3`](https://github.com/w3c/N3), so the suite runs fully offline.
+  See "Updating the snapshot" below for provenance.
+- `spec-test.ts` parses the manifest with the `n3` parser, runs each
+  `test:TestN3Reason` entry through `n3reasoner`, and compares the derived
+  graph against the reference result with RDF isomorphism (`jest-rdf` /
+  `rdf-isomorphic`).
+- The cwm-style `test:options` are mapped onto EYE output modes:
+
+  | options | `n3reasoner` output |
+  |---|---|
+  | `test:conclusions` | `derivations` (`--pass-only-new`) |
+  | `test:data` | `deductive_closure` (`--pass`) |
+  | neither | `deductive_closure_plus_rules` (`--pass-all`) |
+
+  `test:think` (fixpoint) vs `test:rules` (single pass) cannot be
+  distinguished — EYE always reasons to fixpoint.
+
+## Skip list
+
+`skip-list.json` records every known deviation between EYE and the reference
+results, keyed by manifest entry id, with a reason. Skipped entries show up as
+skipped (not failed) so CI stays green while deviations are triaged. To triage
+one: remove it from the skip list, run `npm run test:spec`, and inspect the
+diff.
+
+## Updating the snapshot
+
+The snapshot was taken from [`w3c/N3`](https://github.com/w3c/N3) at commit
+`97653d42da0cd272289ce86f79208d4f0febdde9`. To refresh it, clone `w3c/N3` and
+copy `tests/N3Tests/manifest-reasoner.ttl`, every file referenced by an
+`mf:action`/`mf:result` in it, and `tests/LICENSE.md` into `w3c-n3-tests/`
+(preserving relative paths), then update this commit reference.
+
+### Local patches
+
+The snapshot carries one deliberate deviation from upstream:
+
+- `manifest-reasoner.ttl` line 24: inserted the missing whitespace in
+  `:cwm_includes_t4:cwm_includes_t6` (upstream typo — Turtle allows `:` in
+  local names, so the two entries otherwise parse as a single unknown entry
+  and neither test runs).
+
+The tests are used under the
+[W3C Test Suite License](https://www.w3.org/Consortium/Legal/2008/04-testsuite-license)
+/ [W3C Software and Document License](https://www.w3.org/Consortium/Legal/copyright-software)
+— see `w3c-n3-tests/LICENSE.md`.

__tests__/spec/skip-list.json

@@ -0,0 +1,44 @@
+{
+  "$comment": "Known deviations between EYE (via n3reasoner) and the reference results of the W3C N3 reasoning test suite. Every entry maps a test id from w3c-n3-tests/manifest-reasoner.ttl to the reason it is skipped. Remove an entry once the deviation is resolved; the test then runs again in CI. Counts at the time of writing: 50/89 manifest entries pass, 39 skipped.",
+  "skip": {
+    "cwm_includes_conclusion_simple": "log:conclusion spawns a sub-reasoner, which needs shell/2 / exec — not available in the WASM build (see eye-js#873 and eyereasoner/eye#177)",
+    "cwm_includes_conclusion": "test loads a second file (cwm_other/daml-ex.n3) via log:semantics; n3reasoner only exposes the single input document to the reasoner FS",
+    "cwm_includes_t10": "test dereferences a sibling file (t10a.n3) via log:semantics; n3reasoner only exposes the single input document to the reasoner FS",
+    "cwm_includes_t11": "test dereferences a sibling file (foo.n3) via log:semantics; n3reasoner only exposes the single input document to the reasoner FS",
+    "cwm_includes_t6": "reference result file does not declare the prefixes it uses, so it cannot be parsed standalone",
+    "cwm_includes_t8": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got literal)",
+    "cwm_includes_builtins": "EYE derives a different graph than the cwm reference for this combination of log builtins",
+    "cwm_includes_t9br": "EYE derives a different graph than the cwm reference (log:includes over nested formulae)",
+    "cwm_includes_quantifiers_limited": "EYE handles the explicit quantifiers in this test differently than the cwm reference",
+    "cwm_includes_xsd": "uses test:strings (log:outputString concatenated plain-text output), which is not exposed through the n3reasoner API",
+    "log_content": "log:content on a datatyped literal raises type_error(atom, ...) in sub_atom/5 in the WASM build",
+    "log_parsedAsN3": "EYE derives a different graph than the reference for log:parsedAsN3",
+    "math_absoluteValue": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_big": "EYE misses some numeric edge cases of the reference (big number arithmetic)",
+    "math_ceiling": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_combo": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got .)",
+    "math_difference": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_exponentiation": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_floor": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_inf": "EYE raises zero_division for math:quotient on [0.0 0.0] instead of deriving INF like the reference",
+    "math_product": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_quotient": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_remainder": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_rounded": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_strings": "EYE misses some numeric edge cases of the reference (numeric/string conversion)",
+    "math_sum": "EYE misses some numeric edge cases of the reference (literal datatype handling)",
+    "math_trig": "EYE misses some numeric edge cases of the reference (trigonometry edge cases)",
+    "cwm_list_bug2": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got .)",
+    "cwm_list_r1": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got .)",
+    "cwm_list_unify5": "EYE derives a different graph than the cwm reference (list unification)",
+    "cwm_reason_t6": "EYE derives a different graph than the cwm reference",
+    "string_concatenation": "EYE derives a different graph than the reference for string:concatenation",
+    "string_format": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got .)",
+    "cwm_string_roughly": "EYE derives a different graph than the cwm reference (string:containsRoughly)",
+    "cwm_string_uriEncode": "reference result is not parseable by the n3.js parser (Expected entity but got literal)",
+    "cwm_supports_simple": "EYE derives a different graph than the cwm reference (log:supports)",
+    "cwm_time_t1": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got literal)",
+    "cwm_unify_unify1": "EYE derives a different graph than the cwm reference (formula unification)",
+    "cwm_unify_unify2": "EYE derives a different graph than the cwm reference (formula unification)"
+  }
+}

__tests__/spec/spec-test.ts

@@ -0,0 +1,157 @@
+/**
+ * W3C N3 reasoning spec tests.
+ *
+ * Runs every entry of the vendored `manifest-reasoner.ttl` from the W3C N3
+ * test suite (https://github.com/w3c/N3, snapshot under ./w3c-n3-tests)
+ * through `n3reasoner` and compares the derived graph against the reference
+ * results using RDF isomorphism.
+ *
+ * Known deviations between EYE and the reference results are recorded in
+ * ./skip-list.json (with a reason each) so that this suite stays green while
+ * deviations are triaged individually.
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { DataFactory, Parser, Store } from 'n3';
+import type { Quad, Term } from '@rdfjs/types';
+import 'jest-rdf';
+import { n3reasoner } from '../../dist';
+import type { ICoreQueryOptions } from '../../dist';
+
+const { namedNode, quad } = DataFactory;
+
+const TESTS_DIR = path.join(__dirname, 'w3c-n3-tests');
+const MANIFEST = 'manifest-reasoner.ttl';
+// The base against which the manifest is published; used to resolve the
+// mf:action / mf:result IRIs back to files in the local snapshot.
+const MANIFEST_BASE = 'https://w3c.github.io/N3/tests/N3Tests/';
+// `n3reasoner` loads string input into the reasoner as `data_0.n3s`, so this
+// is the base that EYE resolves relative IRIs in the input against. Parsing
+// the expected results against the same base keeps the two graphs aligned.
+const REASONER_BASE = 'file:///data_0.n3s';
+
+const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#';
+const MF = 'http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#';
+const TEST = 'https://w3c.github.io/N3/tests/test.n3#';
+
+/**
+ * Rewrites every IRI starting with `from` so that it starts with `to`
+ * instead (recursing into quoted triples). The reasoner sees its input as
+ * `data_0.n3s` while the reference results resolve relative IRIs against the
+ * original file name of the test input, so the two graphs only align after
+ * rebasing one onto the other.
+ */
+function rebaseTerm(term: Term, from: string, to: string): Term {
+  if (term.termType === 'NamedNode' && term.value.startsWith(from)) {
+    return namedNode(`${to}${term.value.slice(from.length)}`);
+  }
+  if (term.termType === 'Quad') {
+    const q = term as Quad;
+    return quad(
+      rebaseTerm(q.subject, from, to) as Quad['subject'],
+      rebaseTerm(q.predicate, from, to) as Quad['predicate'],
+      rebaseTerm(q.object, from, to) as Quad['object'],
+      rebaseTerm(q.graph, from, to) as Quad['graph'],
+    );
+  }
+  return term;
+}
+
+const skipList: Record<string, string> = JSON.parse(
+  fs.readFileSync(path.join(__dirname, 'skip-list.json'), 'utf-8'),
+).skip;
+
+interface SpecTest {
+  id: string;
+  name: string;
+  action: string;
+  result: string;
+  output: ICoreQueryOptions['output'];
+}
+
+function loadManifest(): SpecTest[] {
+  const store = new Store(new Parser({
+    format: 'text/n3',
+    baseIRI: `${MANIFEST_BASE}${MANIFEST}`,
+  }).parse(fs.readFileSync(path.join(TESTS_DIR, MANIFEST), 'utf-8')));
+
+  function one(subject: Term | undefined, predicate: string): Term | undefined {
+    return subject && store.getObjects(subject as Parameters<Store['getObjects']>[0], namedNode(predicate), null)[0];
+  }
+
+  // Walk the rdf:List of mf:entries
+  const entries: Term[] = [];
+  let head = one(namedNode(`${MANIFEST_BASE}${MANIFEST}`), `${MF}entries`);
+  while (head && head.value !== `${RDF}nil`) {
+    const first = one(head, `${RDF}first`);
+    if (first) {
+      entries.push(first);
+    }
+    head = one(head, `${RDF}rest`);
+  }
+
+  return entries.map((entry) => {
+    const options = one(entry, `${TEST}options`);
+    const flag = (name: string): boolean => one(options, `${TEST}${name}`)?.value === 'true';
+
+    // Map the cwm-style test options onto the EYE output modes:
+    //  - test:conclusions ("replace store with conclusions") -> derivations
+    //  - test:data ("remove all except plain RDF triples")   -> deductive_closure
+    //  - neither (full store, rules included)                -> deductive_closure_plus_rules
+    // test:think (fixpoint) vs test:rules (single pass) cannot be
+    // distinguished: EYE always reasons to fixpoint.
+    let output: ICoreQueryOptions['output'];
+    if (flag('conclusions')) {
+      output = 'derivations';
+    } else if (flag('data')) {
+      output = 'deductive_closure';
+    } else {
+      output = 'deductive_closure_plus_rules';
+    }
+
+    return {
+      id: entry.value.replace(`${MANIFEST_BASE}${MANIFEST}#`, ''),
+      name: one(entry, `${MF}name`)?.value ?? '(unnamed)',
+      action: one(entry, `${MF}action`)?.value.replace(MANIFEST_BASE, '') ?? '',
+      result: one(entry, `${MF}result`)?.value.replace(MANIFEST_BASE, '') ?? '',
+      output,
+    };
+  });
+}
+
+describe('W3C N3 reasoning spec tests (manifest-reasoner.ttl)', () => {
+  const tests = loadManifest();
+
+  it('loads all test entries from the manifest', () => {
+    expect(tests.length).toBeGreaterThan(0);
+  });
+
+  for (const test of tests) {
+    const runner = test.id in skipList ? it.skip : it;
+    // eslint-disable-next-line no-loop-func
+    runner(`${test.id} — ${test.name}`, async () => {
+      const action = fs.readFileSync(path.join(TESTS_DIR, test.action), 'utf-8');
+      const expectedText = fs.readFileSync(path.join(TESTS_DIR, test.result), 'utf-8');
+
+      // The reference results resolve relative IRIs against the original
+      // name of the input file (they were generated by running the reasoner
+      // on e.g. `reflexive.n3` in its own directory), while `n3reasoner`
+      // loads its input as `data_0.n3s`. Parse the reference against the
+      // input's file name and rebase the actual output onto the same base.
+      const expectedBase = `file:///${path.basename(test.action)}`;
+
+      const actual = await n3reasoner(action, undefined, {
+        output: test.output,
+        outputType: 'quads',
+      });
+      const expected = new Parser({
+        format: 'text/n3',
+        baseIRI: expectedBase,
+      }).parse(expectedText);
+
+      const rebased = actual.map((q) => rebaseTerm(q, REASONER_BASE, expectedBase) as Quad);
+
+      expect(rebased).toBeRdfIsomorphic(expected);
+    });
+  }
+});

__tests__/spec/w3c-n3-tests/LICENSE.md

@@ -0,0 +1,4 @@
+All documents in this Repository are licensed by contributors
+under both the the [W3C Test Suite License](http://www.w3.org/Consortium/Legal/2008/04-testsuite-license) and 
+[W3C Software and Document License](https://www.w3.org/Consortium/Legal/copyright-software).
+

__tests__/spec/w3c-n3-tests/cwm_includes/bnode-conclude-ref.n3

@@ -0,0 +1,4 @@
+@prefix : <bnodeConclude.n3#> .
+
+[ a :Result ].
+

__tests__/spec/w3c-n3-tests/cwm_includes/bnodeConclude.n3

@@ -0,0 +1,3 @@
+@prefix log: <http://www.w3.org/2000/10/swap/log#> .
+
+{ {:a :b []} log:includes {:a :b ?X} } => {?X a :Result} . 

__tests__/spec/w3c-n3-tests/cwm_includes/builtins-ref.n3

@@ -0,0 +1,11 @@
+# From swap/test/ref/includes-builtins.n3
+@prefix : <builtins.n3#> .
+
+:test1     a :Success .
+
+:test2     a :Success .
+
+:test3     a :Success .
+
+:test4     a :Success .
+

__tests__/spec/w3c-n3-tests/cwm_includes/builtins.n3

@@ -0,0 +1,27 @@
+@prefix log: <http://www.w3.org/2000/10/swap/log#> .
+@prefix rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
+
+{ {} log:includes {:a log:equalTo :a} } => { :test1n a :Failure} .
+
+{ {} log:includes {{:a :b :c} log:includes {}} } => { :test2n a :Failure} .
+
+{ {} log:notIncludes {:a log:equalTo :a} } => { :test1 a :Success} .
+
+{ { :foo :bar (1 2) } log:includes { :foo :bar [rdf:first []]} }
+		 => {:test2 a :Success} .
+
+{ { :foo :bar (1 2) } log:notIncludes { :foo :bar [rdf:first []]} }
+		 => {:test2 a :FAILURE} .
+
+
+{ {} log:notIncludes {:a log:equalTo :a} } => { :test3 a :Success} .
+
+# All lists exist and non-empty lists have firsts.
+# For cwm this shows up as the list (1 2) being a List datatype,
+# and the rdf:first builtin which extracts its first element.
+# rdf:first works even with a plain log:includes because the
+# rdf:first property is part of the RDF model of a list.
+
+{   {} log:includes  {   (1 2) rdf:first [] } }  => { :test4 a :Success }.
+
+# ends

__tests__/spec/w3c-n3-tests/cwm_includes/concat-ref.n3

@@ -0,0 +1,20 @@
+@prefix log: <http://www.w3.org/2000/10/swap/log#>.
+@prefix : <concat.n3#> .
+
+log:implies a log:Chaff .
+:TEST13 a :success .
+:test13a a :success .
+:test13b a :success .
+:test13c a :success .
+:test13d a :success .
+:test13e a :success .
+:test13f a :success .
+:test13g a :success.
+:test13h a :success .
+
+:emptyString :is "" .
+:fooString :is "foo" .
+:www :is "WorldWideWeb" .
+:www2 :is "WorldWideWeb" .
+:www3 :is "WorldWIDEWeb" .
+:www5 :is "WorldWideWeb" .