fix: propagate reasoner failures to the exit code + extended deep taxonomy benchmark

__test_utils__/util.ts

@@ -295,6 +295,20 @@ export function universalTests() {
       await expect(n3reasoner('invalid', 'invalid')).rejects.toThrow('Error while executing query');
     });
 
+    it('should throw error when the query result cannot be parsed', async () => {
+      // A stub SWIPL build whose reasoner emits output that is not valid N3
+      const swipl = (async (opts: { print: (str: string) => void }) => {
+        opts.print('this is not valid N3 {{{');
+        return {
+          FS: { writeFile: () => {} },
+          prolog: { query: () => ({ once: () => {} }) },
+        };
+      }) as any;
+
+      await expect(n3reasoner(dataQuads, undefined, { SWIPL: swipl }))
+        .rejects.toThrow('Error while parsing query result');
+    });
+
     it('should execute the n3reasoner on surface query', async () => {
       await expect(n3reasoner(surfaceQuery)).rejects.toThrow(/inference_fuse/);
       await expect(n3reasoner(surfaceQueryQuads)).rejects.toThrow(/inference_fuse/);

__tests__/cli-test.ts

@@ -5,6 +5,7 @@ import { EventEmitter } from 'events';
 import 'jest-rdf';
 import { query, data, result } from '../data/socrates';
 import { mainFunc, convertToPosixPath } from '../dist/bin/main';
+import * as queryLib from '../dist/query';
 import { askCallback, askQuery, askResult } from '../data/ask';
 
 const files = {
@@ -22,6 +23,7 @@ const files = {
   { :Let :output ?out } => { 1 log:outputString ?out } .
   `,
   [path.join(__dirname, 'ask.n3')]: askQuery,
+  [path.join(__dirname, 'invalid.n3')]: 'this is not valid N3 {{{',
 };
 
 jest.mock('fs', () => ({
@@ -43,7 +45,7 @@ async function getConsoleOutput(args: string[]) {
   const restoreConsole = mockConsole();
   const stdin = new ReadStream();
 
-  await mainFunc({
+  const proc = {
     argv: ['/bin/node', 'eyereasoner', ...args],
     cwd: () => __dirname,
     stdin,
@@ -54,7 +56,9 @@ async function getConsoleOutput(args: string[]) {
         }
       },
     },
-  } as NodeJS.Process);
+  } as NodeJS.Process;
+
+  await mainFunc(proc);
 
   // @ts-ignore
   // eslint-disable-next-line no-console
@@ -68,7 +72,7 @@ async function getConsoleOutput(args: string[]) {
   const stdout = calls.map((call) => call.join(' ')).join('\n');
   const stderr = stderrCalls.map((call) => call.join(' ')).join('\n');
 
-  return { stdout, stderr };
+  return { stdout, stderr, exitCode: proc.exitCode };
 }
 
 describe('Testing convertToPosixPath', () => {
@@ -126,4 +130,32 @@ describe('Testing CLI', () => {
     const { stdout } = await getConsoleOutput(['--nope', '--quiet', './ask.n3']);
     expect(new Parser().parse(stdout)).toBeRdfIsomorphic(new Parser().parse(askResult));
   });
+
+  it('Should not set an exit code on success', async () => {
+    const { exitCode } = await getConsoleOutput(['--nope', '--quiet', './socrates.n3', '--query', './socrates-query.n3']);
+    expect(exitCode).toBeUndefined();
+  });
+
+  it('Should set a non-zero exit code when the reasoner errors', async () => {
+    const { stdout, stderr, exitCode } = await getConsoleOutput(['--nope', '--quiet', './invalid.n3']);
+    expect(stdout).toEqual('');
+    expect(stderr).toContain('** ERROR **');
+    expect(exitCode).toEqual(1);
+  });
+
+  it('Should set a non-zero exit code and report the message when the query rejects with an Error', async () => {
+    const qaQuerySpy = jest.spyOn(queryLib, 'qaQuery').mockRejectedValueOnce(new Error('unexpected reasoner failure'));
+    const { stderr, exitCode } = await getConsoleOutput(['--nope', '--quiet', './socrates.n3']);
+    qaQuerySpy.mockRestore();
+    expect(stderr).toContain('unexpected reasoner failure');
+    expect(exitCode).toEqual(1);
+  });
+
+  it('Should set a non-zero exit code and report the value when the query rejects with a non-Error', async () => {
+    const qaQuerySpy = jest.spyOn(queryLib, 'qaQuery').mockRejectedValueOnce('non-error reasoner failure');
+    const { stderr, exitCode } = await getConsoleOutput(['--nope', '--quiet', './socrates.n3']);
+    qaQuerySpy.mockRestore();
+    expect(stderr).toContain('non-error reasoner failure');
+    expect(exitCode).toEqual(1);
+  });
 });

lib/bin/main.ts

@@ -22,7 +22,16 @@ export async function mainFunc(proc: NodeJS.Process) {
     output: proc.stdout,
   });
 
-  const Module = await SwiplEye();
+  // Capture the reasoner's stderr (in addition to echoing it) so that
+  // failures can be propagated to the process exit code below.
+  const errorLines: string[] = [];
+  const Module = await SwiplEye({
+    printErr: (str: string) => {
+      errorLines.push(str);
+      // eslint-disable-next-line no-console
+      console.error(str);
+    },
+  });
 
   const posixArgv: string[] = [];
   // Make any local files available to the reasoner
@@ -46,6 +55,23 @@ export async function mainFunc(proc: NodeJS.Process) {
     }
   }
 
-  await qaQuery(Module, 'main', posixArgv, (q) => rl.question(`${q}\n|: `));
+  let failed = false;
+  try {
+    const res = await qaQuery(Module, 'main', posixArgv, (q) => rl.question(`${q}\n|: `));
+    failed = res.error === true;
+  } catch (e) {
+    // eslint-disable-next-line no-console
+    console.error(e instanceof Error ? e.message : String(e));
+    failed = true;
+  }
   rl.close();
+
+  // EYE reports reasoning failures (parse errors, resource errors, ...) on
+  // stderr with an `** ERROR **` marker while the underlying Prolog goal
+  // still succeeds, so a marker on stderr must also fail the process.
+  if (failed || errorLines.some((line) => line.includes('** ERROR **'))) {
+    // `proc` is dependency-injected so tests can observe the exit code
+    // eslint-disable-next-line no-param-reassign
+    proc.exitCode = 1;
+  }
 }

lib/query.ts

@@ -1,7 +1,14 @@
 import type { SWIPLModule } from 'swipl-wasm';
 
-interface CallOutput {
+export interface CallResult {
   done: boolean;
+  /** Set when the query terminated with an uncaught Prolog exception */
+  error?: boolean;
+  /** The exception message, when `error` is set */
+  message?: string;
+}
+
+interface CallOutput extends CallResult {
   resume: (res: string) => void; yield: string;
 }
 
@@ -19,19 +26,20 @@ export function buildQuery(name: string, args: string | string[]) {
  * @param name The name of the query function
  * @param args The arguments of the query function
  * @param cb The callback for question/answering
- * @returns The result of the query
+ * @returns The final result of the query
  */
 export async function qaQuery(
   module: SWIPLModule,
   queryString: string,
   args: string | string[],
   cb: (res: string) => Promise<string>,
-) {
+): Promise<CallResult> {
   let res = module.prolog.call(buildQuery(queryString, args), { async: true }) as CallOutput;
   while (!res.done) {
     // eslint-disable-next-line no-await-in-loop
     res = res.resume(await cb(res.yield)) as unknown as CallOutput;
   }
+  return res;
 }
 
 /**

perf/bench.ts

@@ -22,6 +22,36 @@ const deepTaxonomyBenchmark100 = [
   ...(new Parser({ format: 'n3' })).parse('{ ?s a ?o . ?o <http://www.w3.org/2000/01/rdf-schema#subClassOf> ?o2 . } => { ?s a ?o2 . } .'),
 ]
 
+// The extended deep taxonomy benchmark in its canonical shape (from the
+// deep taxonomy benchmark at https://eulersharp.sourceforge.net/2009/12dtb/,
+// see also #337/#338): an individual at the bottom of an N-level
+// rdfs:subClassOf chain with 3-way branching, a single *backward* rule, and
+// the target class membership asked via --query. EYE proves this in linear
+// time, so even N=1000 reasons well under a second and the CI benchmark
+// remains bounded.
+//
+// NOTE: unlike the forward-rule deep taxonomy cases above, this case must
+// not reuse a pre-loaded module across runs: re-running main() re-asserts
+// the backward rule, and the duplicated rule clauses make the backward
+// search explode (a second run on the same module does not terminate in
+// minutes, where the first takes <1s). n3reasoner boots a fresh module per
+// call, which keeps every iteration independent.
+function generateExtendedDeepTaxonomy(size: number): { data: string, query: string } {
+  const prefixes = '@prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>.\n'
+    + '@prefix : <http://eulersharp.sourceforge.net/2009/12dtb/test#>.\n';
+  const lines = [prefixes, `:i${size} a :N0.`];
+  for (let i = 0; i < size; i += 1) {
+    lines.push(`:N${i} rdfs:subClassOf :N${i + 1}, :I${i + 1}, :J${i + 1}.`);
+  }
+  lines.push('{?X a ?D} <= {?C rdfs:subClassOf ?D. ?X a ?C}.');
+  return {
+    data: lines.join('\n'),
+    query: `${prefixes}{:i${size} a :N${size}} => {:i${size} a :N${size}}.\n`,
+  };
+}
+
+const extendedDeepTaxonomy1000 = generateExtendedDeepTaxonomy(1000);
+
 function deferred(fn: () => Promise<any>): Benchmark.Options {
   return {
     defer: true,
@@ -90,6 +120,9 @@ async function main() {
     ).add(
       'Run deep taxonomy benchmark [100] [reasoning only]',
       () => queryOnce(LoadedDeep100, 'main', ['--nope', '--quiet', './data.n3', '--pass-only-new']),
+    ).add(
+      'Run extended deep taxonomy benchmark [1000]',
+      deferred(() => n3reasoner(extendedDeepTaxonomy1000.data, extendedDeepTaxonomy1000.query)),
     ).add(
       'Run timbl + foaf + rdfs rules',
       deferred(() => n3reasoner(timblFoafRdfs,  undefined, { outputType: 'string' })),