diff --git a/__test_utils__/util.ts b/__test_utils__/util.ts index 5bda7dcb2..e0554bc29 100644 --- a/__test_utils__/util.ts +++ b/__test_utils__/util.ts @@ -295,6 +295,20 @@ export function universalTests() { await expect(n3reasoner('invalid', 'invalid')).rejects.toThrow('Error while executing query'); }); + it('should throw error when the query result cannot be parsed', async () => { + // A stub SWIPL build whose reasoner emits output that is not valid N3 + const swipl = (async (opts: { print: (str: string) => void }) => { + opts.print('this is not valid N3 {{{'); + return { + FS: { writeFile: () => {} }, + prolog: { query: () => ({ once: () => {} }) }, + }; + }) as any; + + await expect(n3reasoner(dataQuads, undefined, { SWIPL: swipl })) + .rejects.toThrow('Error while parsing query result'); + }); + it('should execute the n3reasoner on surface query', async () => { await expect(n3reasoner(surfaceQuery)).rejects.toThrow(/inference_fuse/); await expect(n3reasoner(surfaceQueryQuads)).rejects.toThrow(/inference_fuse/); diff --git a/__tests__/cli-test.ts b/__tests__/cli-test.ts index 7d6eb6ade..3a020e9f3 100644 --- a/__tests__/cli-test.ts +++ b/__tests__/cli-test.ts @@ -5,6 +5,7 @@ import { EventEmitter } from 'events'; import 'jest-rdf'; import { query, data, result } from '../data/socrates'; import { mainFunc, convertToPosixPath } from '../dist/bin/main'; +import * as queryLib from '../dist/query'; import { askCallback, askQuery, askResult } from '../data/ask'; const files = { @@ -22,6 +23,7 @@ const files = { { :Let :output ?out } => { 1 log:outputString ?out } . `, [path.join(__dirname, 'ask.n3')]: askQuery, + [path.join(__dirname, 'invalid.n3')]: 'this is not valid N3 {{{', }; jest.mock('fs', () => ({ @@ -43,7 +45,7 @@ async function getConsoleOutput(args: string[]) { const restoreConsole = mockConsole(); const stdin = new ReadStream(); - await mainFunc({ + const proc = { argv: ['/bin/node', 'eyereasoner', ...args], cwd: () => __dirname, stdin, @@ -54,7 +56,9 @@ async function getConsoleOutput(args: string[]) { } }, }, - } as NodeJS.Process); + } as NodeJS.Process; + + await mainFunc(proc); // @ts-ignore // eslint-disable-next-line no-console @@ -68,7 +72,7 @@ async function getConsoleOutput(args: string[]) { const stdout = calls.map((call) => call.join(' ')).join('\n'); const stderr = stderrCalls.map((call) => call.join(' ')).join('\n'); - return { stdout, stderr }; + return { stdout, stderr, exitCode: proc.exitCode }; } describe('Testing convertToPosixPath', () => { @@ -126,4 +130,32 @@ describe('Testing CLI', () => { const { stdout } = await getConsoleOutput(['--nope', '--quiet', './ask.n3']); expect(new Parser().parse(stdout)).toBeRdfIsomorphic(new Parser().parse(askResult)); }); + + it('Should not set an exit code on success', async () => { + const { exitCode } = await getConsoleOutput(['--nope', '--quiet', './socrates.n3', '--query', './socrates-query.n3']); + expect(exitCode).toBeUndefined(); + }); + + it('Should set a non-zero exit code when the reasoner errors', async () => { + const { stdout, stderr, exitCode } = await getConsoleOutput(['--nope', '--quiet', './invalid.n3']); + expect(stdout).toEqual(''); + expect(stderr).toContain('** ERROR **'); + expect(exitCode).toEqual(1); + }); + + it('Should set a non-zero exit code and report the message when the query rejects with an Error', async () => { + const qaQuerySpy = jest.spyOn(queryLib, 'qaQuery').mockRejectedValueOnce(new Error('unexpected reasoner failure')); + const { stderr, exitCode } = await getConsoleOutput(['--nope', '--quiet', './socrates.n3']); + qaQuerySpy.mockRestore(); + expect(stderr).toContain('unexpected reasoner failure'); + expect(exitCode).toEqual(1); + }); + + it('Should set a non-zero exit code and report the value when the query rejects with a non-Error', async () => { + const qaQuerySpy = jest.spyOn(queryLib, 'qaQuery').mockRejectedValueOnce('non-error reasoner failure'); + const { stderr, exitCode } = await getConsoleOutput(['--nope', '--quiet', './socrates.n3']); + qaQuerySpy.mockRestore(); + expect(stderr).toContain('non-error reasoner failure'); + expect(exitCode).toEqual(1); + }); }); diff --git a/lib/bin/main.ts b/lib/bin/main.ts index 526ffafb8..5da585a03 100644 --- a/lib/bin/main.ts +++ b/lib/bin/main.ts @@ -22,7 +22,16 @@ export async function mainFunc(proc: NodeJS.Process) { output: proc.stdout, }); - const Module = await SwiplEye(); + // Capture the reasoner's stderr (in addition to echoing it) so that + // failures can be propagated to the process exit code below. + const errorLines: string[] = []; + const Module = await SwiplEye({ + printErr: (str: string) => { + errorLines.push(str); + // eslint-disable-next-line no-console + console.error(str); + }, + }); const posixArgv: string[] = []; // Make any local files available to the reasoner @@ -46,6 +55,23 @@ export async function mainFunc(proc: NodeJS.Process) { } } - await qaQuery(Module, 'main', posixArgv, (q) => rl.question(`${q}\n|: `)); + let failed = false; + try { + const res = await qaQuery(Module, 'main', posixArgv, (q) => rl.question(`${q}\n|: `)); + failed = res.error === true; + } catch (e) { + // eslint-disable-next-line no-console + console.error(e instanceof Error ? e.message : String(e)); + failed = true; + } rl.close(); + + // EYE reports reasoning failures (parse errors, resource errors, ...) on + // stderr with an `** ERROR **` marker while the underlying Prolog goal + // still succeeds, so a marker on stderr must also fail the process. + if (failed || errorLines.some((line) => line.includes('** ERROR **'))) { + // `proc` is dependency-injected so tests can observe the exit code + // eslint-disable-next-line no-param-reassign + proc.exitCode = 1; + } } diff --git a/lib/query.ts b/lib/query.ts index ec43510e9..572e7d3c5 100644 --- a/lib/query.ts +++ b/lib/query.ts @@ -1,7 +1,14 @@ import type { SWIPLModule } from 'swipl-wasm'; -interface CallOutput { +export interface CallResult { done: boolean; + /** Set when the query terminated with an uncaught Prolog exception */ + error?: boolean; + /** The exception message, when `error` is set */ + message?: string; +} + +interface CallOutput extends CallResult { resume: (res: string) => void; yield: string; } @@ -19,19 +26,20 @@ export function buildQuery(name: string, args: string | string[]) { * @param name The name of the query function * @param args The arguments of the query function * @param cb The callback for question/answering - * @returns The result of the query + * @returns The final result of the query */ export async function qaQuery( module: SWIPLModule, queryString: string, args: string | string[], cb: (res: string) => Promise, -) { +): Promise { let res = module.prolog.call(buildQuery(queryString, args), { async: true }) as CallOutput; while (!res.done) { // eslint-disable-next-line no-await-in-loop res = res.resume(await cb(res.yield)) as unknown as CallOutput; } + return res; } /** diff --git a/perf/bench.ts b/perf/bench.ts index be2abd223..8dc5698be 100644 --- a/perf/bench.ts +++ b/perf/bench.ts @@ -22,6 +22,36 @@ const deepTaxonomyBenchmark100 = [ ...(new Parser({ format: 'n3' })).parse('{ ?s a ?o . ?o ?o2 . } => { ?s a ?o2 . } .'), ] +// The extended deep taxonomy benchmark in its canonical shape (from the +// deep taxonomy benchmark at https://eulersharp.sourceforge.net/2009/12dtb/, +// see also #337/#338): an individual at the bottom of an N-level +// rdfs:subClassOf chain with 3-way branching, a single *backward* rule, and +// the target class membership asked via --query. EYE proves this in linear +// time, so even N=1000 reasons well under a second and the CI benchmark +// remains bounded. +// +// NOTE: unlike the forward-rule deep taxonomy cases above, this case must +// not reuse a pre-loaded module across runs: re-running main() re-asserts +// the backward rule, and the duplicated rule clauses make the backward +// search explode (a second run on the same module does not terminate in +// minutes, where the first takes <1s). n3reasoner boots a fresh module per +// call, which keeps every iteration independent. +function generateExtendedDeepTaxonomy(size: number): { data: string, query: string } { + const prefixes = '@prefix rdfs: .\n' + + '@prefix : .\n'; + const lines = [prefixes, `:i${size} a :N0.`]; + for (let i = 0; i < size; i += 1) { + lines.push(`:N${i} rdfs:subClassOf :N${i + 1}, :I${i + 1}, :J${i + 1}.`); + } + lines.push('{?X a ?D} <= {?C rdfs:subClassOf ?D. ?X a ?C}.'); + return { + data: lines.join('\n'), + query: `${prefixes}{:i${size} a :N${size}} => {:i${size} a :N${size}}.\n`, + }; +} + +const extendedDeepTaxonomy1000 = generateExtendedDeepTaxonomy(1000); + function deferred(fn: () => Promise): Benchmark.Options { return { defer: true, @@ -90,6 +120,9 @@ async function main() { ).add( 'Run deep taxonomy benchmark [100] [reasoning only]', () => queryOnce(LoadedDeep100, 'main', ['--nope', '--quiet', './data.n3', '--pass-only-new']), + ).add( + 'Run extended deep taxonomy benchmark [1000]', + deferred(() => n3reasoner(extendedDeepTaxonomy1000.data, extendedDeepTaxonomy1000.query)), ).add( 'Run timbl + foaf + rdfs rules', deferred(() => n3reasoner(timblFoafRdfs, undefined, { outputType: 'string' })),