diff --git a/.github/workflows/nodejs.yml b/.github/workflows/nodejs.yml index 815c8dd23..fa951581d 100644 --- a/.github/workflows/nodejs.yml +++ b/.github/workflows/nodejs.yml @@ -56,6 +56,34 @@ jobs: shell: bash command: npm test + # W3C N3 reasoning spec tests. The upstream test suite is NOT committed to + # this repo: it is fetched at a pinned commit by scripts/fetch-n3-tests.ts + # and cached below with actions/cache so runs stay deterministic and fast. + # This job is intentionally NOT in the release job's `needs`: known EYE + # deviations are tracked in __tests__/spec/skip-list.json, and any newly + # surfacing deviation should be triaged there rather than blocking releases. + spec-tests: + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-node@v6 + with: + node-version: 24.x + - run: npm ci + - uses: actions/download-artifact@v8 + with: + name: dist + path: dist + # Cache the fetched W3C N3 test suite, keyed on the fetch script (which + # pins the upstream commit) so bumping the pin busts the cache. + - uses: actions/cache@v4 + with: + path: __tests__/spec/.w3c-n3-tests + key: w3c-n3-tests-${{ hashFiles('scripts/fetch-n3-tests.ts') }} + - run: npm run test:spec + benchmark: needs: build strategy: diff --git a/.gitignore b/.gitignore index 0b1d698a7..8937810ea 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ lib/eye.ts lib/lingua.ts perf/output.txt perf/dynamic.mjs +__tests__/spec/.w3c-n3-tests diff --git a/__tests__/spec/README.md b/__tests__/spec/README.md new file mode 100644 index 000000000..421b85c03 --- /dev/null +++ b/__tests__/spec/README.md @@ -0,0 +1,68 @@ +# W3C N3 reasoning spec tests + +This directory contains a jest runner for the reasoning part of the +[W3C N3 test suite](https://github.com/w3c/N3/tree/master/tests) (issue #336). + +``` +npm run test:spec +``` + +It is a **separate jest project** (`jest.spec.config.json`) so that it does not +interfere with the coverage-thresholded unit test suite, and it needs `dist` +to be built first (`npm run build`). + +## How it works + +- The upstream test suite is **not committed to this repo**. `npm run test:spec` + first runs `npm run spec:fetch` (`scripts/fetch-n3-tests.ts`), which downloads + `tests/N3Tests` from [`w3c/N3`](https://github.com/w3c/N3) at a **pinned + commit** into the git-ignored `__tests__/spec/.w3c-n3-tests/`. Fetching is a + no-op once the directory is present, and in CI that directory is persisted + with `actions/cache` (keyed on `scripts/fetch-n3-tests.ts`, which holds the + pin) so runs are deterministic and cache-hit fast. +- `spec-test.ts` parses the manifest with the `n3` parser, runs each + `test:TestN3Reason` entry through `n3reasoner`, and compares the derived + graph against the reference result with RDF isomorphism (`jest-rdf` / + `rdf-isomorphic`). +- The cwm-style `test:options` are mapped onto EYE output modes: + + | options | `n3reasoner` output | + |---|---| + | `test:conclusions` | `derivations` (`--pass-only-new`) | + | `test:data` | `deductive_closure` (`--pass`) | + | neither | `deductive_closure_plus_rules` (`--pass-all`) | + + `test:think` (fixpoint) vs `test:rules` (single pass) cannot be + distinguished — EYE always reasons to fixpoint. + +## Skip list + +`skip-list.json` records every known deviation between EYE and the reference +results, keyed by manifest entry id, with a reason. Skipped entries show up as +skipped (not failed) so CI stays green while deviations are triaged. To triage +one: remove it from the skip list, run `npm run test:spec`, and inspect the +diff. + +## Updating the pinned version + +Bump `N3_TESTS_COMMIT` in `scripts/fetch-n3-tests.ts`. That value both drives +the download and is hashed into the CI cache key, so a new pin refreshes the +cache automatically. Delete `__tests__/spec/.w3c-n3-tests/` locally to force a +re-fetch. + +## Known upstream manifest typo + +The pinned manifest concatenates two entries without whitespace in its +`mf:entries` list — `:cwm_includes_t4:cwm_includes_t6` — which Turtle parses as +a single, undefined prefixed name (`:` is legal in local names), so both tests +would silently vanish. `spec-test.ts` applies a small, self-healing in-code +patch (insert the missing space) before parsing the fetched manifest; it +becomes a no-op once upstream fixes it. Filed upstream as +[w3c-cg/N3#232](https://github.com/w3c-cg/N3/issues/232). + +## License + +The fetched tests are used under the +[W3C Test Suite License](https://www.w3.org/Consortium/Legal/2008/04-testsuite-license) +/ [W3C Software and Document License](https://www.w3.org/Consortium/Legal/copyright-software). +They are downloaded at test time, not redistributed in this repository. diff --git a/__tests__/spec/skip-list.json b/__tests__/spec/skip-list.json new file mode 100644 index 000000000..49bd43cb9 --- /dev/null +++ b/__tests__/spec/skip-list.json @@ -0,0 +1,44 @@ +{ + "$comment": "Known deviations between EYE (via n3reasoner) and the reference results of the W3C N3 reasoning test suite. Every entry maps a test id from w3c-n3-tests/manifest-reasoner.ttl to the reason it is skipped. Remove an entry once the deviation is resolved; the test then runs again in CI. Counts at the time of writing: 50/89 manifest entries pass, 39 skipped.", + "skip": { + "cwm_includes_conclusion_simple": "log:conclusion spawns a sub-reasoner, which needs shell/2 / exec — not available in the WASM build (see eye-js#873 and eyereasoner/eye#177)", + "cwm_includes_conclusion": "test loads a second file (cwm_other/daml-ex.n3) via log:semantics; n3reasoner only exposes the single input document to the reasoner FS", + "cwm_includes_t10": "test dereferences a sibling file (t10a.n3) via log:semantics; n3reasoner only exposes the single input document to the reasoner FS", + "cwm_includes_t11": "test dereferences a sibling file (foo.n3) via log:semantics; n3reasoner only exposes the single input document to the reasoner FS", + "cwm_includes_t6": "reference result file does not declare the prefixes it uses, so it cannot be parsed standalone", + "cwm_includes_t8": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got literal)", + "cwm_includes_builtins": "EYE derives a different graph than the cwm reference for this combination of log builtins", + "cwm_includes_t9br": "EYE derives a different graph than the cwm reference (log:includes over nested formulae)", + "cwm_includes_quantifiers_limited": "EYE handles the explicit quantifiers in this test differently than the cwm reference", + "cwm_includes_xsd": "uses test:strings (log:outputString concatenated plain-text output), which is not exposed through the n3reasoner API", + "log_content": "log:content on a datatyped literal raises type_error(atom, ...) in sub_atom/5 in the WASM build", + "log_parsedAsN3": "EYE derives a different graph than the reference for log:parsedAsN3", + "math_absoluteValue": "EYE misses some numeric edge cases of the reference (literal datatype handling)", + "math_big": "EYE misses some numeric edge cases of the reference (big number arithmetic)", + "math_ceiling": "EYE misses some numeric edge cases of the reference (literal datatype handling)", + "math_combo": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got .)", + "math_difference": "EYE misses some numeric edge cases of the reference (literal datatype handling)", + "math_exponentiation": "EYE misses some numeric edge cases of the reference (literal datatype handling)", + "math_floor": "EYE misses some numeric edge cases of the reference (literal datatype handling)", + "math_inf": "EYE raises zero_division for math:quotient on [0.0 0.0] instead of deriving INF like the reference", + "math_product": "EYE misses some numeric edge cases of the reference (literal datatype handling)", + "math_quotient": "EYE misses some numeric edge cases of the reference (literal datatype handling)", + "math_remainder": "EYE misses some numeric edge cases of the reference (literal datatype handling)", + "math_rounded": "EYE misses some numeric edge cases of the reference (literal datatype handling)", + "math_strings": "EYE misses some numeric edge cases of the reference (numeric/string conversion)", + "math_sum": "EYE misses some numeric edge cases of the reference (literal datatype handling)", + "math_trig": "EYE misses some numeric edge cases of the reference (trigonometry edge cases)", + "cwm_list_bug2": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got .)", + "cwm_list_r1": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got .)", + "cwm_list_unify5": "EYE derives a different graph than the cwm reference (list unification)", + "cwm_reason_t6": "EYE derives a different graph than the cwm reference", + "string_concatenation": "EYE derives a different graph than the reference for string:concatenation", + "string_format": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got .)", + "cwm_string_roughly": "EYE derives a different graph than the cwm reference (string:containsRoughly)", + "cwm_string_uriEncode": "reference result is not parseable by the n3.js parser (Expected entity but got literal)", + "cwm_supports_simple": "EYE derives a different graph than the cwm reference (log:supports)", + "cwm_time_t1": "EYE output for this test is not parseable by the n3.js parser (Expected entity but got literal)", + "cwm_unify_unify1": "EYE derives a different graph than the cwm reference (formula unification)", + "cwm_unify_unify2": "EYE derives a different graph than the cwm reference (formula unification)" + } +} diff --git a/__tests__/spec/spec-test.ts b/__tests__/spec/spec-test.ts new file mode 100644 index 000000000..fa31d4372 --- /dev/null +++ b/__tests__/spec/spec-test.ts @@ -0,0 +1,182 @@ +/** + * W3C N3 reasoning spec tests. + * + * Runs every entry of `manifest-reasoner.ttl` from the W3C N3 test suite + * (https://github.com/w3c/N3) through `n3reasoner` and compares the derived + * graph against the reference results using RDF isomorphism. + * + * The test suite is NOT committed to this repo — it is fetched at a pinned + * commit into a git-ignored cache directory by `scripts/fetch-n3-tests.ts` + * (run via `npm run spec:fetch`, which `npm run test:spec` invokes first, and + * cached in CI with actions/cache). If the suite is missing, this file throws + * with a pointer to that script. + * + * Known deviations between EYE and the reference results are recorded in + * ./skip-list.json (with a reason each) so that this suite stays green while + * deviations are triaged individually. + */ +import * as fs from 'fs'; +import * as path from 'path'; +import { DataFactory, Parser, Store } from 'n3'; +import type { Quad, Term } from '@rdfjs/types'; +import 'jest-rdf'; +import { n3reasoner } from '../../dist'; +import type { ICoreQueryOptions } from '../../dist'; +import { cacheDir } from '../../scripts/fetch-n3-tests'; + +const { namedNode, quad } = DataFactory; + +const TESTS_DIR = cacheDir; +const MANIFEST = 'manifest-reasoner.ttl'; +// The base against which the manifest is published; used to resolve the +// mf:action / mf:result IRIs back to files in the fetched suite. +const MANIFEST_BASE = 'https://w3c.github.io/N3/tests/N3Tests/'; +// `n3reasoner` loads string input into the reasoner as `data_0.n3s`, so this +// is the base that EYE resolves relative IRIs in the input against. Parsing +// the expected results against the same base keeps the two graphs aligned. +const REASONER_BASE = 'file:///data_0.n3s'; + +const RDF = 'http://www.w3.org/1999/02/22-rdf-syntax-ns#'; +const MF = 'http://www.w3.org/2001/sw/DataAccess/tests/test-manifest#'; +const TEST = 'https://w3c.github.io/N3/tests/test.n3#'; + +/** + * Rewrites every IRI starting with `from` so that it starts with `to` + * instead (recursing into quoted triples). The reasoner sees its input as + * `data_0.n3s` while the reference results resolve relative IRIs against the + * original file name of the test input, so the two graphs only align after + * rebasing one onto the other. + */ +function rebaseTerm(term: Term, from: string, to: string): Term { + if (term.termType === 'NamedNode' && term.value.startsWith(from)) { + return namedNode(`${to}${term.value.slice(from.length)}`); + } + if (term.termType === 'Quad') { + const q = term as Quad; + return quad( + rebaseTerm(q.subject, from, to) as Quad['subject'], + rebaseTerm(q.predicate, from, to) as Quad['predicate'], + rebaseTerm(q.object, from, to) as Quad['object'], + rebaseTerm(q.graph, from, to) as Quad['graph'], + ); + } + return term; +} + +const skipList: Record = JSON.parse( + fs.readFileSync(path.join(__dirname, 'skip-list.json'), 'utf-8'), +).skip; + +interface SpecTest { + id: string; + name: string; + action: string; + result: string; + output: ICoreQueryOptions['output']; +} + +function loadManifest(): SpecTest[] { + const manifestPath = path.join(TESTS_DIR, MANIFEST); + if (!fs.existsSync(manifestPath)) { + throw new Error( + `W3C N3 test suite not found at ${TESTS_DIR}. ` + + 'Run `npm run spec:fetch` first (`npm run test:spec` does this automatically).', + ); + } + + let manifestText = fs.readFileSync(manifestPath, 'utf-8'); + // Upstream typo (filed as w3c-cg/N3#232): two entries are concatenated + // without whitespace in the mf:entries list — `:cwm_includes_t4:cwm_includes_t6` + // — which Turtle parses as a single, undefined prefixed name rather than two + // list items, so both tests silently vanish. Patch it here on the fetched + // copy until upstream lands the fix; self-healing (a no-op once fixed). + manifestText = manifestText.replace( + ':cwm_includes_t4:cwm_includes_t6', + ':cwm_includes_t4 :cwm_includes_t6', + ); + + const store = new Store(new Parser({ + format: 'text/n3', + baseIRI: `${MANIFEST_BASE}${MANIFEST}`, + }).parse(manifestText)); + + function one(subject: Term | undefined, predicate: string): Term | undefined { + return subject && store.getObjects(subject as Parameters[0], namedNode(predicate), null)[0]; + } + + // Walk the rdf:List of mf:entries + const entries: Term[] = []; + let head = one(namedNode(`${MANIFEST_BASE}${MANIFEST}`), `${MF}entries`); + while (head && head.value !== `${RDF}nil`) { + const first = one(head, `${RDF}first`); + if (first) { + entries.push(first); + } + head = one(head, `${RDF}rest`); + } + + return entries.map((entry) => { + const options = one(entry, `${TEST}options`); + const flag = (name: string): boolean => one(options, `${TEST}${name}`)?.value === 'true'; + + // Map the cwm-style test options onto the EYE output modes: + // - test:conclusions ("replace store with conclusions") -> derivations + // - test:data ("remove all except plain RDF triples") -> deductive_closure + // - neither (full store, rules included) -> deductive_closure_plus_rules + // test:think (fixpoint) vs test:rules (single pass) cannot be + // distinguished: EYE always reasons to fixpoint. + let output: ICoreQueryOptions['output']; + if (flag('conclusions')) { + output = 'derivations'; + } else if (flag('data')) { + output = 'deductive_closure'; + } else { + output = 'deductive_closure_plus_rules'; + } + + return { + id: entry.value.replace(`${MANIFEST_BASE}${MANIFEST}#`, ''), + name: one(entry, `${MF}name`)?.value ?? '(unnamed)', + action: one(entry, `${MF}action`)?.value.replace(MANIFEST_BASE, '') ?? '', + result: one(entry, `${MF}result`)?.value.replace(MANIFEST_BASE, '') ?? '', + output, + }; + }); +} + +describe('W3C N3 reasoning spec tests (manifest-reasoner.ttl)', () => { + const tests = loadManifest(); + + it('loads all test entries from the manifest', () => { + expect(tests.length).toBeGreaterThan(0); + }); + + for (const test of tests) { + const runner = test.id in skipList ? it.skip : it; + // eslint-disable-next-line no-loop-func + runner(`${test.id} — ${test.name}`, async () => { + const action = fs.readFileSync(path.join(TESTS_DIR, test.action), 'utf-8'); + const expectedText = fs.readFileSync(path.join(TESTS_DIR, test.result), 'utf-8'); + + // The reference results resolve relative IRIs against the original + // name of the input file (they were generated by running the reasoner + // on e.g. `reflexive.n3` in its own directory), while `n3reasoner` + // loads its input as `data_0.n3s`. Parse the reference against the + // input's file name and rebase the actual output onto the same base. + const expectedBase = `file:///${path.basename(test.action)}`; + + const actual = await n3reasoner(action, undefined, { + output: test.output, + outputType: 'quads', + }); + const expected = new Parser({ + format: 'text/n3', + baseIRI: expectedBase, + }).parse(expectedText); + + const rebased = actual.map((q) => rebaseTerm(q, REASONER_BASE, expectedBase) as Quad); + + expect(rebased).toBeRdfIsomorphic(expected); + }); + } +}); diff --git a/jest.config.json b/jest.config.json index 87b96f452..01bf19a06 100644 --- a/jest.config.json +++ b/jest.config.json @@ -1,5 +1,6 @@ { "preset": "ts-jest", + "testPathIgnorePatterns": ["/node_modules/", "/__tests__/spec/"], "collectCoverage": true, "coverageReporters": [ "json-summary", diff --git a/jest.spec.config.json b/jest.spec.config.json new file mode 100644 index 000000000..6f5f2fe21 --- /dev/null +++ b/jest.spec.config.json @@ -0,0 +1,6 @@ +{ + "preset": "ts-jest", + "testMatch": ["/__tests__/spec/**/*-test.ts"], + "testTimeout": 90000, + "maxWorkers": 1 +} diff --git a/package.json b/package.json index 241a0b3dd..949d6ecce 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,8 @@ "test:badges": "npm run test:coverage && jest-coverage-badges", "test:unit": "npm run bundle:webpack && node --experimental-vm-modules node_modules/jest/bin/jest.js", "test:cli": "npm run bundle:webpack && node --experimental-vm-modules node_modules/jest/bin/jest.js cli-test", + "spec:fetch": "ts-node scripts/fetch-n3-tests", + "test:spec": "npm run spec:fetch && node --experimental-vm-modules node_modules/jest/bin/jest.js --config jest.spec.config.json", "test:memory": "npm run test:memory:node && npm run test:memory:node:error && npm run test:memory:browser", "test:memory:node": "node __tests_memory__/leakTest", "test:memory:node:error": "node __tests_memory__/leakTestOnError", diff --git a/scripts/fetch-n3-tests.ts b/scripts/fetch-n3-tests.ts new file mode 100644 index 000000000..d649abdd8 --- /dev/null +++ b/scripts/fetch-n3-tests.ts @@ -0,0 +1,74 @@ +/** + * Fetches the W3C N3 test suite (https://github.com/w3c/N3) at a pinned commit + * into a local, git-ignored cache directory so the reasoning spec tests + * (__tests__/spec/spec-test.ts) can run against it without the suite being + * committed to this repo. + * + * In CI the cache directory is persisted with actions/cache keyed on this + * pinned commit (see .github/workflows/nodejs.yml), so the download only + * happens on a cache miss. Locally, re-running is a no-op once the suite is + * present. Set N3_TESTS_DIR to override the cache location (must match the + * runner and the CI cache path). + */ +import { execFileSync } from 'child_process'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; + +// Pinned upstream commit of w3c/N3. Bump this (and let the CI cache key, which +// hashes this file, refresh) to pull in new/updated tests. +export const N3_TESTS_COMMIT = '97653d42da0cd272289ce86f79208d4f0febdde9'; + +export const cacheDir = process.env.N3_TESTS_DIR + ?? path.join(__dirname, '..', '__tests__', 'spec', '.w3c-n3-tests'); + +// The subtree of the upstream repo the reasoning suite reads from. +const SUBTREE = 'tests/N3Tests'; + +async function main() { + const manifest = path.join(cacheDir, 'manifest-reasoner.ttl'); + if (fs.existsSync(manifest)) { + // Already fetched (local re-run or CI cache hit). + process.stdout.write(`N3 test suite already present at ${cacheDir}\n`); + return; + } + + process.stdout.write(`Fetching w3c/N3@${N3_TESTS_COMMIT} into ${cacheDir}\n`); + const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'n3-tests-')); + const tarball = path.join(tmp, 'n3.tar.gz'); + + try { + const url = `https://codeload.github.com/w3c/N3/tar.gz/${N3_TESTS_COMMIT}`; + const res = await fetch(url); + if (!res.ok) { + throw new Error(`Failed to download ${url}: ${res.status} ${res.statusText}`); + } + fs.writeFileSync(tarball, new Uint8Array(await res.arrayBuffer())); + + // Extract only the reasoning-tests subtree, stripping the + // `N3-/tests/N3Tests/` prefix so files land directly in cacheDir. + fs.mkdirSync(cacheDir, { recursive: true }); + execFileSync('tar', [ + '-xzf', tarball, + '-C', cacheDir, + '--strip-components', '3', + `N3-${N3_TESTS_COMMIT}/${SUBTREE}`, + ], { stdio: 'inherit' }); + + if (!fs.existsSync(manifest)) { + throw new Error(`Extraction did not produce ${manifest}`); + } + process.stdout.write('N3 test suite ready\n'); + } finally { + fs.rmSync(tmp, { recursive: true, force: true }); + } +} + +// Only fetch when run directly (`ts-node scripts/fetch-n3-tests`), not when +// the spec-test runner imports this module for `cacheDir` / `N3_TESTS_COMMIT`. +if (require.main === module) { + main().catch((err) => { + process.stderr.write(`${err instanceof Error ? err.stack : String(err)}\n`); + process.exitCode = 1; + }); +}