From 2fe15b5e974ee4a67ac2800e965a97a03a807aa3 Mon Sep 17 00:00:00 2001 From: Jesse Wright <63333554+jeswr@users.noreply.github.com> Date: Sat, 4 Jul 2026 23:21:57 +0000 Subject: [PATCH 1/3] fix: propagate reasoner failures to the exit code + EDT benchmark Two related changes for the extended-deep-taxonomy issues: - perf/bench.ts: add an 'extended deep taxonomy benchmark [1000]' case in the canonical EDT shape (individual at the bottom of a 1000-level rdfs:subClassOf chain with 3-way branching, backward rule {?X a ?D} <= {?C rdfs:subClassOf ?D. ?X a ?C}, target membership via --query). EYE proves it in linear time (<0.5s warm locally), so the CI benchmark stays bounded. The case deliberately boots a fresh module per iteration: re-running main() on a pre-loaded module re-asserts the backward rule and the duplicated clauses make the backward search explode (documented in the case comment). - lib/bin/main.ts: the CLI used to exit 0 with empty output when the reasoner failed (e.g. resource errors on large N3 files, parse errors). EYE reports failures on stderr with an '** ERROR **' marker while the underlying Prolog goal still succeeds, so mainFunc now captures stderr, checks for the marker (and for uncaught Prolog exceptions, which lib/query.ts qaQuery now surfaces by returning the final call result), and sets process.exitCode = 1 on failure. - __tests__/cli-test.ts: regression tests asserting a non-zero exit code for an erroring input and an untouched exit code on success. Closes #337 Closes #338 Co-Authored-By: Claude Fable 5 --- __tests__/cli-test.ts | 21 ++++++++++++++++++--- lib/bin/main.ts | 28 ++++++++++++++++++++++++++-- lib/query.ts | 14 +++++++++++--- perf/bench.ts | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 8 deletions(-) diff --git a/__tests__/cli-test.ts b/__tests__/cli-test.ts index 7d6eb6ade..8fbfac676 100644 --- a/__tests__/cli-test.ts +++ b/__tests__/cli-test.ts @@ -22,6 +22,7 @@ const files = { { :Let :output ?out } => { 1 log:outputString ?out } . `, [path.join(__dirname, 'ask.n3')]: askQuery, + [path.join(__dirname, 'invalid.n3')]: 'this is not valid N3 {{{', }; jest.mock('fs', () => ({ @@ -43,7 +44,7 @@ async function getConsoleOutput(args: string[]) { const restoreConsole = mockConsole(); const stdin = new ReadStream(); - await mainFunc({ + const proc = { argv: ['/bin/node', 'eyereasoner', ...args], cwd: () => __dirname, stdin, @@ -54,7 +55,9 @@ async function getConsoleOutput(args: string[]) { } }, }, - } as NodeJS.Process); + } as NodeJS.Process; + + await mainFunc(proc); // @ts-ignore // eslint-disable-next-line no-console @@ -68,7 +71,7 @@ async function getConsoleOutput(args: string[]) { const stdout = calls.map((call) => call.join(' ')).join('\n'); const stderr = stderrCalls.map((call) => call.join(' ')).join('\n'); - return { stdout, stderr }; + return { stdout, stderr, exitCode: proc.exitCode }; } describe('Testing convertToPosixPath', () => { @@ -126,4 +129,16 @@ describe('Testing CLI', () => { const { stdout } = await getConsoleOutput(['--nope', '--quiet', './ask.n3']); expect(new Parser().parse(stdout)).toBeRdfIsomorphic(new Parser().parse(askResult)); }); + + it('Should not set an exit code on success', async () => { + const { exitCode } = await getConsoleOutput(['--nope', '--quiet', './socrates.n3', '--query', './socrates-query.n3']); + expect(exitCode).toBeUndefined(); + }); + + it('Should set a non-zero exit code when the reasoner errors', async () => { + const { stdout, stderr, exitCode } = await getConsoleOutput(['--nope', '--quiet', './invalid.n3']); + expect(stdout).toEqual(''); + expect(stderr).toContain('** ERROR **'); + expect(exitCode).toEqual(1); + }); }); diff --git a/lib/bin/main.ts b/lib/bin/main.ts index 526ffafb8..3ed346871 100644 --- a/lib/bin/main.ts +++ b/lib/bin/main.ts @@ -22,7 +22,16 @@ export async function mainFunc(proc: NodeJS.Process) { output: proc.stdout, }); - const Module = await SwiplEye(); + // Capture the reasoner's stderr (in addition to echoing it) so that + // failures can be propagated to the process exit code below. + const errorLines: string[] = []; + const Module = await SwiplEye({ + printErr: (str: string) => { + errorLines.push(str); + // eslint-disable-next-line no-console + console.error(str); + }, + }); const posixArgv: string[] = []; // Make any local files available to the reasoner @@ -46,6 +55,21 @@ export async function mainFunc(proc: NodeJS.Process) { } } - await qaQuery(Module, 'main', posixArgv, (q) => rl.question(`${q}\n|: `)); + let failed = false; + try { + const res = await qaQuery(Module, 'main', posixArgv, (q) => rl.question(`${q}\n|: `)); + failed = res.error === true; + } /* istanbul ignore next: defensive — EYE reports its errors on stderr instead */ catch (e) { + // eslint-disable-next-line no-console + console.error(e instanceof Error ? e.message : String(e)); + failed = true; + } rl.close(); + + // EYE reports reasoning failures (parse errors, resource errors, ...) on + // stderr with an `** ERROR **` marker while the underlying Prolog goal + // still succeeds, so a marker on stderr must also fail the process. + if (failed || errorLines.some((line) => line.includes('** ERROR **'))) { + proc.exitCode = 1; + } } diff --git a/lib/query.ts b/lib/query.ts index ec43510e9..572e7d3c5 100644 --- a/lib/query.ts +++ b/lib/query.ts @@ -1,7 +1,14 @@ import type { SWIPLModule } from 'swipl-wasm'; -interface CallOutput { +export interface CallResult { done: boolean; + /** Set when the query terminated with an uncaught Prolog exception */ + error?: boolean; + /** The exception message, when `error` is set */ + message?: string; +} + +interface CallOutput extends CallResult { resume: (res: string) => void; yield: string; } @@ -19,19 +26,20 @@ export function buildQuery(name: string, args: string | string[]) { * @param name The name of the query function * @param args The arguments of the query function * @param cb The callback for question/answering - * @returns The result of the query + * @returns The final result of the query */ export async function qaQuery( module: SWIPLModule, queryString: string, args: string | string[], cb: (res: string) => Promise, -) { +): Promise { let res = module.prolog.call(buildQuery(queryString, args), { async: true }) as CallOutput; while (!res.done) { // eslint-disable-next-line no-await-in-loop res = res.resume(await cb(res.yield)) as unknown as CallOutput; } + return res; } /** diff --git a/perf/bench.ts b/perf/bench.ts index be2abd223..8dc5698be 100644 --- a/perf/bench.ts +++ b/perf/bench.ts @@ -22,6 +22,36 @@ const deepTaxonomyBenchmark100 = [ ...(new Parser({ format: 'n3' })).parse('{ ?s a ?o . ?o ?o2 . } => { ?s a ?o2 . } .'), ] +// The extended deep taxonomy benchmark in its canonical shape (from the +// deep taxonomy benchmark at https://eulersharp.sourceforge.net/2009/12dtb/, +// see also #337/#338): an individual at the bottom of an N-level +// rdfs:subClassOf chain with 3-way branching, a single *backward* rule, and +// the target class membership asked via --query. EYE proves this in linear +// time, so even N=1000 reasons well under a second and the CI benchmark +// remains bounded. +// +// NOTE: unlike the forward-rule deep taxonomy cases above, this case must +// not reuse a pre-loaded module across runs: re-running main() re-asserts +// the backward rule, and the duplicated rule clauses make the backward +// search explode (a second run on the same module does not terminate in +// minutes, where the first takes <1s). n3reasoner boots a fresh module per +// call, which keeps every iteration independent. +function generateExtendedDeepTaxonomy(size: number): { data: string, query: string } { + const prefixes = '@prefix rdfs: .\n' + + '@prefix : .\n'; + const lines = [prefixes, `:i${size} a :N0.`]; + for (let i = 0; i < size; i += 1) { + lines.push(`:N${i} rdfs:subClassOf :N${i + 1}, :I${i + 1}, :J${i + 1}.`); + } + lines.push('{?X a ?D} <= {?C rdfs:subClassOf ?D. ?X a ?C}.'); + return { + data: lines.join('\n'), + query: `${prefixes}{:i${size} a :N${size}} => {:i${size} a :N${size}}.\n`, + }; +} + +const extendedDeepTaxonomy1000 = generateExtendedDeepTaxonomy(1000); + function deferred(fn: () => Promise): Benchmark.Options { return { defer: true, @@ -90,6 +120,9 @@ async function main() { ).add( 'Run deep taxonomy benchmark [100] [reasoning only]', () => queryOnce(LoadedDeep100, 'main', ['--nope', '--quiet', './data.n3', '--pass-only-new']), + ).add( + 'Run extended deep taxonomy benchmark [1000]', + deferred(() => n3reasoner(extendedDeepTaxonomy1000.data, extendedDeepTaxonomy1000.query)), ).add( 'Run timbl + foaf + rdfs rules', deferred(() => n3reasoner(timblFoafRdfs, undefined, { outputType: 'string' })), From b387b517c09685f905c0ea8c0b6f3ce9849af19c Mon Sep 17 00:00:00 2001 From: Jesse Wright <63333554+jeswr@users.noreply.github.com> Date: Sun, 5 Jul 2026 00:21:06 +0000 Subject: [PATCH 2/3] style: scoped eslint disable for intentional proc.exitCode assignment `proc` is dependency-injected into mainFunc so tests can observe the exit code; assigning to it is the point of the change, so silence no-param-reassign for that one line. Lint-only, no behavior change. Co-Authored-By: Claude Fable 5 --- lib/bin/main.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/bin/main.ts b/lib/bin/main.ts index 3ed346871..ee51f91de 100644 --- a/lib/bin/main.ts +++ b/lib/bin/main.ts @@ -70,6 +70,8 @@ export async function mainFunc(proc: NodeJS.Process) { // stderr with an `** ERROR **` marker while the underlying Prolog goal // still succeeds, so a marker on stderr must also fail the process. if (failed || errorLines.some((line) => line.includes('** ERROR **'))) { + // `proc` is dependency-injected so tests can observe the exit code + // eslint-disable-next-line no-param-reassign proc.exitCode = 1; } } From ed86bdf552781ddd14348c6be82396b1878f5504 Mon Sep 17 00:00:00 2001 From: Jesse Wright <63333554+jeswr@users.noreply.github.com> Date: Sun, 5 Jul 2026 22:02:45 +0000 Subject: [PATCH 3/3] test: cover the CLI catch path and the query-result parse failure Coverage thresholds were failing on the new exit-code logic: - dist/bin/main.js 64-65: the qaQuery catch block was never executed (its istanbul-ignore hint was emitted by tsc as a trailing comment of the try block, so it was ignored). Rather than fixing the hint, drop it and exercise the path for real: two new CLI tests reject qaQuery with an Error and a non-Error, covering both sides of the `e instanceof Error` ternary and the failed exit-code branch. - dist/transformers.js 86: the "Error while parsing query result" throw in parse() was uncovered; a new test injects a stub SWIPL build whose output is invalid N3 and asserts the rejection. No library behavior changes and no threshold changes; local run is now 100% statements/lines/functions, 98.71% branches (the only remaining miss is the tsc __importDefault helper branch, pre-existing on main). Co-Authored-By: Claude Fable 5 --- __test_utils__/util.ts | 14 ++++++++++++++ __tests__/cli-test.ts | 17 +++++++++++++++++ lib/bin/main.ts | 2 +- 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/__test_utils__/util.ts b/__test_utils__/util.ts index 5bda7dcb2..e0554bc29 100644 --- a/__test_utils__/util.ts +++ b/__test_utils__/util.ts @@ -295,6 +295,20 @@ export function universalTests() { await expect(n3reasoner('invalid', 'invalid')).rejects.toThrow('Error while executing query'); }); + it('should throw error when the query result cannot be parsed', async () => { + // A stub SWIPL build whose reasoner emits output that is not valid N3 + const swipl = (async (opts: { print: (str: string) => void }) => { + opts.print('this is not valid N3 {{{'); + return { + FS: { writeFile: () => {} }, + prolog: { query: () => ({ once: () => {} }) }, + }; + }) as any; + + await expect(n3reasoner(dataQuads, undefined, { SWIPL: swipl })) + .rejects.toThrow('Error while parsing query result'); + }); + it('should execute the n3reasoner on surface query', async () => { await expect(n3reasoner(surfaceQuery)).rejects.toThrow(/inference_fuse/); await expect(n3reasoner(surfaceQueryQuads)).rejects.toThrow(/inference_fuse/); diff --git a/__tests__/cli-test.ts b/__tests__/cli-test.ts index 8fbfac676..3a020e9f3 100644 --- a/__tests__/cli-test.ts +++ b/__tests__/cli-test.ts @@ -5,6 +5,7 @@ import { EventEmitter } from 'events'; import 'jest-rdf'; import { query, data, result } from '../data/socrates'; import { mainFunc, convertToPosixPath } from '../dist/bin/main'; +import * as queryLib from '../dist/query'; import { askCallback, askQuery, askResult } from '../data/ask'; const files = { @@ -141,4 +142,20 @@ describe('Testing CLI', () => { expect(stderr).toContain('** ERROR **'); expect(exitCode).toEqual(1); }); + + it('Should set a non-zero exit code and report the message when the query rejects with an Error', async () => { + const qaQuerySpy = jest.spyOn(queryLib, 'qaQuery').mockRejectedValueOnce(new Error('unexpected reasoner failure')); + const { stderr, exitCode } = await getConsoleOutput(['--nope', '--quiet', './socrates.n3']); + qaQuerySpy.mockRestore(); + expect(stderr).toContain('unexpected reasoner failure'); + expect(exitCode).toEqual(1); + }); + + it('Should set a non-zero exit code and report the value when the query rejects with a non-Error', async () => { + const qaQuerySpy = jest.spyOn(queryLib, 'qaQuery').mockRejectedValueOnce('non-error reasoner failure'); + const { stderr, exitCode } = await getConsoleOutput(['--nope', '--quiet', './socrates.n3']); + qaQuerySpy.mockRestore(); + expect(stderr).toContain('non-error reasoner failure'); + expect(exitCode).toEqual(1); + }); }); diff --git a/lib/bin/main.ts b/lib/bin/main.ts index ee51f91de..5da585a03 100644 --- a/lib/bin/main.ts +++ b/lib/bin/main.ts @@ -59,7 +59,7 @@ export async function mainFunc(proc: NodeJS.Process) { try { const res = await qaQuery(Module, 'main', posixArgv, (q) => rl.question(`${q}\n|: `)); failed = res.error === true; - } /* istanbul ignore next: defensive — EYE reports its errors on stderr instead */ catch (e) { + } catch (e) { // eslint-disable-next-line no-console console.error(e instanceof Error ? e.message : String(e)); failed = true;