From 99817be591be3709ab5460145f4a493159ef1333 Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Thu, 9 Jul 2026 11:09:00 -0500 Subject: [PATCH 1/5] Add glob to expressionlanguage --- src/schema/expressionLanguage.test.ts | 184 +++++++++++++++++++++++++- src/schema/expressionLanguage.ts | 120 +++++++++++++++++ 2 files changed, 303 insertions(+), 1 deletion(-) diff --git a/src/schema/expressionLanguage.test.ts b/src/schema/expressionLanguage.test.ts index da27dec62..353081b0b 100644 --- a/src/schema/expressionLanguage.test.ts +++ b/src/schema/expressionLanguage.test.ts @@ -1,5 +1,14 @@ import { assert, assertEquals } from '@std/assert' -import { contextFunction, expressionFunctions, formatter, prepareContext } from './expressionLanguage.ts' +import { + contextFunction, + createMatcher, + expressionFunctions, + formatter, + glob, + matchRecursive, + parsePattern, + prepareContext, +} from './expressionLanguage.ts' import { dataFile, rootFileTree } from './fixtures.test.ts' import { BIDSContext } from './context.ts' import type { DatasetIssues } from '../issues/datasetIssues.ts' @@ -7,6 +16,12 @@ import type { DatasetIssues } from '../issues/datasetIssues.ts' Deno.test('test expression functions', async (t) => { const context = new BIDSContext(dataFile, undefined, rootFileTree) + await t.step('glob function', () => { + assert(glob.bind(context)('sub-*').length === 1) + assert(glob.bind(context)('wat-*').length === 0) + assert(glob.bind(context)('**/*task-*').length === 6) + assert(glob.bind(context)('*/**/*task-*').length === 4) + }) await t.step('index function', () => { const index = expressionFunctions.index assert(index([1, 2, 3], 2) === 1) @@ -314,3 +329,170 @@ Deno.test('formatter test', async (t) => { } }) }) + +Deno.test('test parsePattern helper', async (t) => { + await t.step('parsePattern normalizes input', () => { + // Leading/trailing slashes removed + assert(equal(parsePattern('/sub-*'), ['sub-*'])) + assert(equal(parsePattern('sub-*'), ['sub-*'])) + assert(equal(parsePattern('sub-*/'), ['sub-*'])) + assert(equal(parsePattern('/sub-*/'), ['sub-*'])) + }) + + await t.step('parsePattern splits by /', () => { + assert(equal(parsePattern('sub-*/ses-*'), ['sub-*', 'ses-*'])) + }) + + await t.step('parsePattern preserves ** as single component', () => { + assert(equal(parsePattern('**/*task-*'), ['**', '*task-*'])) + assert(equal(parsePattern('**/sub-*/ses-*'), ['**', 'sub-*', 'ses-*'])) + assert(equal(parsePattern('sub-*/**/*_T1w.nii.gz'), ['sub-*', '**', '*_T1w.nii.gz'])) + }) + + await t.step('parsePattern handles edge cases', () => { + assert(equal(parsePattern(''), [])) + assert(equal(parsePattern('/'), [])) + assert(equal(parsePattern('///'), [])) + }) +}) + +Deno.test('test createMatcher helper', async (t) => { + await t.step('createMatcher ** matches everything', () => { + const matcher = createMatcher('**') + assert(matcher('anything') === true) + assert(matcher('sub-01') === true) + assert(matcher('') === true) + }) + + await t.step('createMatcher * matches any name', () => { + const matcher = createMatcher('*') + assert(matcher('sub-01') === true) + assert(matcher('anything') === true) + assert(matcher('a') === true) + }) + + await t.step('createMatcher ? matches single character', () => { + const matcher = createMatcher('?') + assert(matcher('a') === true) + assert(matcher('1') === true) + assert(matcher('ab') === false) + assert(matcher('') === false) + }) + + await t.step('createMatcher literal glob patterns', () => { + const subMatcher = createMatcher('sub-*') + assert(subMatcher('sub-01') === true) + assert(subMatcher('sub-02') === true) + assert(subMatcher('ses-01') === false) + + const taskMatcher = createMatcher('*task-*') + assert(taskMatcher('task-rest') === true) + assert(taskMatcher('boldtask-rest') === true) + assert(taskMatcher('no-match') === false) + }) + + await t.step('createMatcher ? in patterns', () => { + const matcher = createMatcher('sub-?') + assert(matcher('sub-0') === true) + assert(matcher('sub-a') === true) + assert(matcher('sub-01') === false) + }) +}) + +Deno.test('test matchRecursive helper', async (t) => { + const context = await makeBIDSContext(dataFile, undefined, rootFileTree) + + await t.step('matchRecursive basic functionality', () => { + // Match at root level + const results = Array.from( + matchRecursive(context.dataset.tree, ['sub-*'], ''), + ) as string[] + assert(results.length == 1) + assert(results[0].startsWith('sub-')) + }) + + await t.step('matchRecursive with multiple components', () => { + // Match nested directories + const results = Array.from( + matchRecursive(context.dataset.tree, ['sub-*', 'ses-*'], ''), + ) as string[] + assert(results.every((r) => r.includes('sub-') && r.includes('ses-'))) + }) + + await t.step('matchRecursive with ** at start', () => { + // ** at start should match sub-* at root level + const results = Array.from( + matchRecursive(context.dataset.tree, ['**', 'sub-*'], ''), + ) as string[] + assert(results.length > 1) + assert(results.some((r) => r.startsWith('sub-'))) + assert(results.every((r) => r == 'sub-01' || r.includes('/sub-01_'))) + }) + + await t.step('matchRecursive with no matches', () => { + const results = Array.from( + matchRecursive(context.dataset.tree, ['nonexistent-*'], ''), + ) as string[] + assert(results.length === 0) + }) +}) + +Deno.test('test glob edge cases and root-level matching', async (t) => { + const context = await makeBIDSContext(dataFile, undefined, rootFileTree) + + await t.step('glob with ** at start matches at root level', () => { + // **/sub-* should match sub-* directories at root + const results = glob.bind(context)('**/sub-*') + assert(results.length > 0) + assert(results.some((r) => r.startsWith('sub-'))) + }) + + await t.step('glob with just ** returns all paths', () => { + const results = glob.bind(context)('**') + assert(results.length > 0) + }) + + await t.step('glob with just * returns root-level items', () => { + const results = glob.bind(context)('*') + assert(results.every((r) => !r.includes('/'))) + }) + + await t.step('glob with ? matches single characters', () => { + let results = glob.bind(context)('?') + assert(results.length === 0) + results = glob.bind(context)('?ataset_description.json') + assert(results.length === 1) + results = glob.bind(context)('sub-??') + assert(results.length === 1) + }) + + await t.step('glob deep patterns work correctly', () => { + const results = glob.bind(context)('**/sub-*/ses-*/anat/*T1*') + assert(Array.isArray(results)) + // Verify all results contain expected components + assert( + results.every( + (r) => r.includes('sub-') && r.includes('ses-') && r.includes('anat'), + ), + ) + }) + + await t.step('** does not result in duplicate matches', async () => { + const tree = pathsToTree(['/a/b/c/d/e']) + const datafile = tree.get('a/b/c/d/e') as BIDSFile + const context = await makeBIDSContext(datafile, undefined, tree) + + let results = glob.bind(context)('a/**/e') + assertEquals(results.length, 1) + results = glob.bind(context)('a/*/**/e') + assertEquals(results.length, 1) + results = glob.bind(context)('a/**/*/e') + assertEquals(results.length, 1) + }) + + await t.step('glob returns both files and directories', () => { + const results = glob.bind(context)('**/*') + assert(results.some((r) => r.endsWith('anat'))) // Directory + assert(results.some((r) => r.endsWith('T1w.nii.gz'))) // File + }) +}) diff --git a/src/schema/expressionLanguage.ts b/src/schema/expressionLanguage.ts index 964b5e555..70cbfb028 100644 --- a/src/schema/expressionLanguage.ts +++ b/src/schema/expressionLanguage.ts @@ -1,4 +1,5 @@ import type { BIDSContext } from './context.ts' +import type { FileTree } from '../types/filetree.ts' import { memoize } from '../utils/memoize.ts' function exists(this: BIDSContext, list: string[], rule: string = 'dataset'): number { @@ -36,6 +37,124 @@ function exists(this: BIDSContext, list: string[], rule: string = 'dataset'): nu } } +/* + * Glob utility function. Breaks initial glob pattern into per directory components. + */ +export function parsePattern(pattern: string): string[] { + // Remove leading/trailing slashes + pattern = pattern.replace(/^\/+|\/+$/g, '') + + if (!pattern) return [] + + // Split by '/', filter out empty strings + return pattern.split('/').filter((c) => c.length > 0) +} + +/* + * Glob utility function. Generates function to match individual components of glob. + */ +export function createMatcher(component: string): (name: string) => boolean { + if (component === '**') { + return () => true // Special case, handled in recursion + } + + if (component === '*') { + return (name) => true // Match any single name + } + + if (component === '?') { + return (name) => name.length === 1 + } + + // Glob pattern with * and ? wildcards + // Convert to regex: sub-* → /^sub-.*$/ + const pattern = '^' + component.replace(/\*/g, '.*').replace(/\?/g, '.') + '$' + const re = new RegExp(pattern) + return (name) => re.test(name) +} + +/* + * Glob utility function. Recurses filetree applying match functions to the + * appropriate directories and files. + */ +export function* matchRecursive( + tree: FileTree, + components: string[], + currentPath: string, +): Generator { + if (components.length === 0) { + return + } + + const component = components[0] + const remaining = components.slice(1) + + // Special handling for '**': match zero or more directories + if (component === '**') { + if (remaining.length === 0) { + // Just '**' - match all files and directories recursively + yield* matchAll(tree, currentPath) + return + } + + // '**' followed by more patterns + // Try to match the next pattern at this level (zero directories) + yield* matchRecursive(tree, remaining, currentPath) + + // Also try deeper (one or more directories) + for (const dir of tree.directories) { + yield* matchRecursive(dir, components, `${currentPath}/${dir.name}`.replace(/^\//, '')) + } + return + } + + const matcher = createMatcher(component) + + // Check files at this level + if (remaining.length === 0) { + for (const file of tree.files) { + if (matcher(file.name)) { + yield currentPath ? `${currentPath}/${file.name}` : file.name + } + } + } + + // Check directories at this level + for (const dir of tree.directories) { + if (matcher(dir.name)) { + if (remaining.length === 0) { + yield currentPath ? `${currentPath}/${dir.name}` : dir.name + } else { + // Recurse into matched directory + yield* matchRecursive(dir, remaining, `${currentPath}/${dir.name}`.replace(/^\//, '')) + } + } + } +} + +/* + * Glob utility function. Used to handle '**' in glob pattern. + */ +function* matchAll(tree: FileTree, currentPath: string): Generator { + for (const file of tree.files) { + yield currentPath ? `${currentPath}/${file.name}` : file.name + } + for (const dir of tree.directories) { + const newPath = `${currentPath}/${dir.name}`.replace(/^\//, '') + yield newPath + yield* matchAll(dir, newPath) + } +} + +export function glob(this: BIDSContext, toMatch: string): string[] { + const components = parsePattern(toMatch) + if (components.length === 0) { + return [] + } + + return Array.from(matchRecursive(this.dataset.tree, components, '')) +} + export const expressionFunctions = { index: (list: T[], item: T): number | null => { const index = list.indexOf(item) @@ -124,6 +243,7 @@ export const expressionFunctions = { allequal: (a: T[], b: T[]): boolean => { return (a != null && b != null) && a.length === b.length && a.every((v, i) => v === b[i]) }, + glob: glob, } /** From 3a115b6e34dc682c01f343337dac985832c3a8e7 Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Thu, 9 Jul 2026 11:09:53 -0500 Subject: [PATCH 2/5] Add ability to validate index columns for tabular files specified in sidecar. --- src/schema/expressionLanguage.test.ts | 8 +++-- src/schema/tables.test.ts | 49 +++++++++++++++++++++++++++ src/schema/tables.ts | 20 +++++++++-- 3 files changed, 71 insertions(+), 6 deletions(-) diff --git a/src/schema/expressionLanguage.test.ts b/src/schema/expressionLanguage.test.ts index 353081b0b..fb1bc8ea1 100644 --- a/src/schema/expressionLanguage.test.ts +++ b/src/schema/expressionLanguage.test.ts @@ -1,4 +1,4 @@ -import { assert, assertEquals } from '@std/assert' +import { assert, assertEquals, equal } from '@std/assert' import { contextFunction, createMatcher, @@ -10,8 +10,10 @@ import { prepareContext, } from './expressionLanguage.ts' import { dataFile, rootFileTree } from './fixtures.test.ts' -import { BIDSContext } from './context.ts' -import type { DatasetIssues } from '../issues/datasetIssues.ts' +import type { BIDSContext } from './context.ts' +import { makeBIDSContext } from './context.test.ts' +import type { BIDSFile } from '../types/filetree.ts' +import { pathsToTree } from '../files/filetree.test.ts' Deno.test('test expression functions', async (t) => { const context = new BIDSContext(dataFile, undefined, rootFileTree) diff --git a/src/schema/tables.test.ts b/src/schema/tables.test.ts index c81b39f83..84cc1302f 100644 --- a/src/schema/tables.test.ts +++ b/src/schema/tables.test.ts @@ -275,6 +275,55 @@ Deno.test('tables eval* tests', async (t) => { ) }) + await t.step('verify sidecar specified column index works', () => { + const context = { + path: '/sub-01/sub-01_scans.tsv', + extension: '.tsv', + sidecar: { 'IndexColumns': ['onset', 'filename'] }, + columns: new ColumnsMap(Object.entries({ + onset: ['1900-01-01:00:00', '1900-01-01:00:00'], + filename: ['func/sub-01_task-rest_bold.nii.gz', 'func/sub-01_task-unrest_bold.nii.gz'], + })), + dataset: { issues: new DatasetIssues() }, + } + const rule = schemaDefs.rules.tabular_data.modality_agnostic.Scans + evalIndexColumns(rule, context, schema, 'rules.tabular_data.modality_agnostic.Scans') + assertEquals( + context.dataset.issues.get({ code: 'TSV_INDEX_VALUE_NOT_UNIQUE' }).length, + 0, + ) + + // Now test index columns that violate the multi column uniqueness constraint. + context.columns = new ColumnsMap(Object.entries({ + onset: ['1900-01-01:00:00', '1900-01-01:00:00'], + filename: ['func/sub-01_task-rest_bold.nii.gz', 'func/sub-01_task-rest_bold.nii.gz'], + })) + evalIndexColumns(rule, context, schema, 'rules.tabular_data.modality_agnostic.Scans') + assertEquals( + context.dataset.issues.get({ code: 'TSV_INDEX_VALUE_NOT_UNIQUE' }).length, + 1, + ) + }) + + await t.step('verify sidecar specified column index errors on missing column', () => { + const context = { + path: '/sub-01/sub-01_scans.tsv', + extension: '.tsv', + sidecar: { 'IndexColumns': ['onset', 'badcol'] }, + columns: new ColumnsMap(Object.entries({ + onset: ['1900-01-01:00:00', '1900-01-01:00:00'], + filename: ['func/sub-01_task-rest_bold.nii.gz', 'func/sub-01_task-rest_bold.nii.gz'], + })), + dataset: { issues: new DatasetIssues() }, + } + const rule = schemaDefs.rules.tabular_data.modality_agnostic.Scans + evalIndexColumns(rule, context, schema, 'rules.tabular_data.modality_agnostic.Scans') + assertEquals( + context.dataset.issues.get({ code: 'TSV_COLUMN_MISSING' }).length, + 1, + ) + }) + await t.step('verify not allowed additional columns', () => { const context = { path: '/sub-01/sub-01_scans.tsv', diff --git a/src/schema/tables.ts b/src/schema/tables.ts index 5966fb5d7..8c87ea983 100644 --- a/src/schema/tables.ts +++ b/src/schema/tables.ts @@ -369,10 +369,24 @@ export function evalIndexColumns( ) { return } + const uniqueIndexValues = new Set() - const index_columns = rule.index_columns.map((col: string) => { - return schema.objects.columns[col].name - }).filter((col: string) => context.columns[col]) + let index_columns: string[] = [] + if (context.sidecar.IndexColumns) { + index_columns = context.sidecar.IndexColumns as string[] + const missingColumns = index_columns.filter((col) => !context.columns[col]) + if (missingColumns.length) { + context.dataset.issues.add({ + code: 'TSV_COLUMN_MISSING', + location: context.path, + issueMessage: `Sidecar specified IndexColumns are Missing: ${missingColumns}`, + }) + } + } else if (rule.index_columns) { + index_columns = rule.index_columns.map((col: string) => { + return schema.objects.columns[col].name + }).filter((col: string) => context.columns[col]) + } const rowCount = (context.columns[index_columns[0]] as string[])?.length || 0 for (let i = 0; i < rowCount; i++) { From cea9cb2ec307fa4a4eec269ead75bec8f662cef4 Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Thu, 9 Jul 2026 10:59:16 -0500 Subject: [PATCH 3/5] add changelog --- ...rosswilsonblair_bep036_extensions_redux.md | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 changelog.d/20260709_105449_rosswilsonblair_bep036_extensions_redux.md diff --git a/changelog.d/20260709_105449_rosswilsonblair_bep036_extensions_redux.md b/changelog.d/20260709_105449_rosswilsonblair_bep036_extensions_redux.md new file mode 100644 index 000000000..eccdc6b51 --- /dev/null +++ b/changelog.d/20260709_105449_rosswilsonblair_bep036_extensions_redux.md @@ -0,0 +1,48 @@ + + +### Added + +- Added glob to expression language +- evalIndexColumns now checks for IndexColumns field in sidecar nad uses it for validation. + + + + + + + From 6181535ffcf844df32e5c6958dd795b0f0af9b3b Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Thu, 9 Jul 2026 11:06:29 -0500 Subject: [PATCH 4/5] lint changes --- src/schema/expressionLanguage.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schema/expressionLanguage.ts b/src/schema/expressionLanguage.ts index 70cbfb028..5183120d0 100644 --- a/src/schema/expressionLanguage.ts +++ b/src/schema/expressionLanguage.ts @@ -59,7 +59,7 @@ export function createMatcher(component: string): (name: string) => boolean { } if (component === '*') { - return (name) => true // Match any single name + return (_name) => true // Match any single name } if (component === '?') { From eca86eff967ecf222df142e3d6eea23e45f26487 Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Wed, 15 Jul 2026 09:59:29 -0500 Subject: [PATCH 5/5] additional fmt and lint run --- src/schema/expressionLanguage.test.ts | 33 +++++++++++++++++---------- src/schema/tables.ts | 5 ++-- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/src/schema/expressionLanguage.test.ts b/src/schema/expressionLanguage.test.ts index fb1bc8ea1..e25f63fc9 100644 --- a/src/schema/expressionLanguage.test.ts +++ b/src/schema/expressionLanguage.test.ts @@ -314,19 +314,28 @@ Deno.test('formatter test', async (t) => { }) await t.step('format strings', () => { const context = prepareContext( - {a: 'stringa', b: 'stringb', c: 3, d: {e: 4}, f: [0, 1, 2], g: [1, 2, 3]} as unknown as BIDSContext + { + a: 'stringa', + b: 'stringb', + c: 3, + d: { e: 4 }, + f: [0, 1, 2], + g: [1, 2, 3], + } as unknown as BIDSContext, ) - for (const [str, expected] of [ - ['{a}', 'stringa'], - ['`{a}`', '`stringa`'], // Backticks are preserved - ['`````{a}`````', '`````stringa`````'], - ['{a} and {b} and {c}', 'stringa and stringb and 3'], - ['{a}\\n{d.e}', 'stringa\\n4'], // Backslashes are preserved - ['{intersects(f, g)}', '1,2'], // expressions are evaluated - ['{z}', 'undefined'], - // Unsupported Pythonisms - // ['{{a}}', '{a}'], - ]) { + for ( + const [str, expected] of [ + ['{a}', 'stringa'], + ['`{a}`', '`stringa`'], // Backticks are preserved + ['`````{a}`````', '`````stringa`````'], + ['{a} and {b} and {c}', 'stringa and stringb and 3'], + ['{a}\\n{d.e}', 'stringa\\n4'], // Backslashes are preserved + ['{intersects(f, g)}', '1,2'], // expressions are evaluated + ['{z}', 'undefined'], + // Unsupported Pythonisms + // ['{{a}}', '{a}'], + ] + ) { assertEquals(formatter(str)(context), expected) } }) diff --git a/src/schema/tables.ts b/src/schema/tables.ts index 8c87ea983..f73564f70 100644 --- a/src/schema/tables.ts +++ b/src/schema/tables.ts @@ -218,7 +218,6 @@ export function evalColumns( schemaPath: string, ): void { if (!rule.columns || !['.tsv', '.tsv.gz'].includes(context.extension)) return - const columns = rule.columns as Record const columnLookup = Object.fromEntries( Object.keys(rule.columns).map((col) => [schema.objects.columns[col].name, col]), @@ -256,7 +255,7 @@ export function evalColumns( try { signature = getValueSignature(columnObject, sidecarDef) - } catch (e: any) { + } catch (e: unknown) { if (e?.code) { context.dataset.issues.add({ ...e, @@ -334,7 +333,7 @@ export function evalInitialColumns( } }).filter(({ requirement, index }) => requirement === 'required' || index !== -1) // Validate ordering of present and/or required initial columns - columns.forEach(({ name, requirement, index }, targetIndex) => { + columns.forEach(({ name, _requirement, index }, targetIndex) => { if (index === -1) { context.dataset.issues.add({ code: 'TSV_COLUMN_MISSING',