From 91c31a83c2823ed908a8e6989ee056fdf276d3ed Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Thu, 9 Jul 2026 10:29:23 -0500 Subject: [PATCH 1/3] Add glob to expressionlanguage --- src/schema/expressionLanguage.test.ts | 177 ++++++++++++++++++++++++++ src/schema/expressionLanguage.ts | 120 +++++++++++++++++ 2 files changed, 297 insertions(+) diff --git a/src/schema/expressionLanguage.test.ts b/src/schema/expressionLanguage.test.ts index 0810051d..801d978e 100644 --- a/src/schema/expressionLanguage.test.ts +++ b/src/schema/expressionLanguage.test.ts @@ -1,8 +1,12 @@ import { assert, assertEquals } from '@std/assert' import { contextFunction, + createMatcher, expressionFunctions, formatter, + glob, + matchRecursive, + parsePattern, prepareContext, } from './expressionLanguage.ts' import { dataFile, rootFileTree } from './fixtures.test.ts' @@ -12,6 +16,12 @@ import { makeBIDSContext } from './context.test.ts' Deno.test('test expression functions', async (t) => { const context = await makeBIDSContext(dataFile, undefined, rootFileTree) + await t.step('glob function', () => { + assert(glob.bind(context)('sub-*').length === 1) + assert(glob.bind(context)('wat-*').length === 0) + assert(glob.bind(context)('**/*task-*').length === 6) + assert(glob.bind(context)('*/**/*task-*').length === 4) + }) await t.step('index function', () => { const index = expressionFunctions.index assert(index([1, 2, 3], 2) === 1) @@ -328,3 +338,170 @@ Deno.test('formatter test', async (t) => { } }) }) + +Deno.test('test parsePattern helper', async (t) => { + await t.step('parsePattern normalizes input', () => { + // Leading/trailing slashes removed + assert(equal(parsePattern('/sub-*'), ['sub-*'])) + assert(equal(parsePattern('sub-*'), ['sub-*'])) + assert(equal(parsePattern('sub-*/'), ['sub-*'])) + assert(equal(parsePattern('/sub-*/'), ['sub-*'])) + }) + + await t.step('parsePattern splits by /', () => { + assert(equal(parsePattern('sub-*/ses-*'), ['sub-*', 'ses-*'])) + }) + + await t.step('parsePattern preserves ** as single component', () => { + assert(equal(parsePattern('**/*task-*'), ['**', '*task-*'])) + assert(equal(parsePattern('**/sub-*/ses-*'), ['**', 'sub-*', 'ses-*'])) + assert(equal(parsePattern('sub-*/**/*_T1w.nii.gz'), ['sub-*', '**', '*_T1w.nii.gz'])) + }) + + await t.step('parsePattern handles edge cases', () => { + assert(equal(parsePattern(''), [])) + assert(equal(parsePattern('/'), [])) + assert(equal(parsePattern('///'), [])) + }) +}) + +Deno.test('test createMatcher helper', async (t) => { + await t.step('createMatcher ** matches everything', () => { + const matcher = createMatcher('**') + assert(matcher('anything') === true) + assert(matcher('sub-01') === true) + assert(matcher('') === true) + }) + + await t.step('createMatcher * matches any name', () => { + const matcher = createMatcher('*') + assert(matcher('sub-01') === true) + assert(matcher('anything') === true) + assert(matcher('a') === true) + }) + + await t.step('createMatcher ? matches single character', () => { + const matcher = createMatcher('?') + assert(matcher('a') === true) + assert(matcher('1') === true) + assert(matcher('ab') === false) + assert(matcher('') === false) + }) + + await t.step('createMatcher literal glob patterns', () => { + const subMatcher = createMatcher('sub-*') + assert(subMatcher('sub-01') === true) + assert(subMatcher('sub-02') === true) + assert(subMatcher('ses-01') === false) + + const taskMatcher = createMatcher('*task-*') + assert(taskMatcher('task-rest') === true) + assert(taskMatcher('boldtask-rest') === true) + assert(taskMatcher('no-match') === false) + }) + + await t.step('createMatcher ? in patterns', () => { + const matcher = createMatcher('sub-?') + assert(matcher('sub-0') === true) + assert(matcher('sub-a') === true) + assert(matcher('sub-01') === false) + }) +}) + +Deno.test('test matchRecursive helper', async (t) => { + const context = await makeBIDSContext(dataFile, undefined, rootFileTree) + + await t.step('matchRecursive basic functionality', () => { + // Match at root level + const results = Array.from( + matchRecursive(context.dataset.tree, ['sub-*'], ''), + ) as string[] + assert(results.length == 1) + assert(results[0].startsWith('sub-')) + }) + + await t.step('matchRecursive with multiple components', () => { + // Match nested directories + const results = Array.from( + matchRecursive(context.dataset.tree, ['sub-*', 'ses-*'], ''), + ) as string[] + assert(results.every((r) => r.includes('sub-') && r.includes('ses-'))) + }) + + await t.step('matchRecursive with ** at start', () => { + // ** at start should match sub-* at root level + const results = Array.from( + matchRecursive(context.dataset.tree, ['**', 'sub-*'], ''), + ) as string[] + assert(results.length > 1) + assert(results.some((r) => r.startsWith('sub-'))) + assert(results.every((r) => r == 'sub-01' || r.includes('/sub-01_'))) + }) + + await t.step('matchRecursive with no matches', () => { + const results = Array.from( + matchRecursive(context.dataset.tree, ['nonexistent-*'], ''), + ) as string[] + assert(results.length === 0) + }) +}) + +Deno.test('test glob edge cases and root-level matching', async (t) => { + const context = await makeBIDSContext(dataFile, undefined, rootFileTree) + + await t.step('glob with ** at start matches at root level', () => { + // **/sub-* should match sub-* directories at root + const results = glob.bind(context)('**/sub-*') + assert(results.length > 0) + assert(results.some((r) => r.startsWith('sub-'))) + }) + + await t.step('glob with just ** returns all paths', () => { + const results = glob.bind(context)('**') + assert(results.length > 0) + }) + + await t.step('glob with just * returns root-level items', () => { + const results = glob.bind(context)('*') + assert(results.every((r) => !r.includes('/'))) + }) + + await t.step('glob with ? matches single characters', () => { + let results = glob.bind(context)('?') + assert(results.length === 0) + results = glob.bind(context)('?ataset_description.json') + assert(results.length === 1) + results = glob.bind(context)('sub-??') + assert(results.length === 1) + }) + + await t.step('glob deep patterns work correctly', () => { + const results = glob.bind(context)('**/sub-*/ses-*/anat/*T1*') + assert(Array.isArray(results)) + // Verify all results contain expected components + assert( + results.every( + (r) => r.includes('sub-') && r.includes('ses-') && r.includes('anat'), + ), + ) + }) + + await t.step('** does not result in duplicate matches', async () => { + const tree = pathsToTree(['/a/b/c/d/e']) + const datafile = tree.get('a/b/c/d/e') as BIDSFile + const context = await makeBIDSContext(datafile, undefined, tree) + + let results = glob.bind(context)('a/**/e') + assertEquals(results.length, 1) + results = glob.bind(context)('a/*/**/e') + assertEquals(results.length, 1) + results = glob.bind(context)('a/**/*/e') + assertEquals(results.length, 1) + }) + + await t.step('glob returns both files and directories', () => { + const results = glob.bind(context)('**/*') + assert(results.some((r) => r.endsWith('anat'))) // Directory + assert(results.some((r) => r.endsWith('T1w.nii.gz'))) // File + }) +}) diff --git a/src/schema/expressionLanguage.ts b/src/schema/expressionLanguage.ts index 486a89d3..36a195b8 100644 --- a/src/schema/expressionLanguage.ts +++ b/src/schema/expressionLanguage.ts @@ -1,4 +1,5 @@ import type { BIDSContext } from './context.ts' +import type { FileTree } from '../types/filetree.ts' import { memoize } from '../utils/memoize.ts' function exists(this: BIDSContext, list: string[], rule: string = 'dataset'): number { @@ -36,6 +37,124 @@ function exists(this: BIDSContext, list: string[], rule: string = 'dataset'): nu } } +/* + * Glob utility function. Breaks initial glob pattern into per directory components. + */ +export function parsePattern(pattern: string): string[] { + // Remove leading/trailing slashes + pattern = pattern.replace(/^\/+|\/+$/g, '') + + if (!pattern) return [] + + // Split by '/', filter out empty strings + return pattern.split('/').filter((c) => c.length > 0) +} + +/* + * Glob utility function. Generates function to match individual components of glob. + */ +export function createMatcher(component: string): (name: string) => boolean { + if (component === '**') { + return () => true // Special case, handled in recursion + } + + if (component === '*') { + return (name) => true // Match any single name + } + + if (component === '?') { + return (name) => name.length === 1 + } + + // Glob pattern with * and ? wildcards + // Convert to regex: sub-* → /^sub-.*$/ + const pattern = '^' + component.replace(/\*/g, '.*').replace(/\?/g, '.') + '$' + const re = new RegExp(pattern) + return (name) => re.test(name) +} + +/* + * Glob utility function. Recurses filetree applying match functions to the + * appropriate directories and files. + */ +export function* matchRecursive( + tree: FileTree, + components: string[], + currentPath: string, +): Generator { + if (components.length === 0) { + return + } + + const component = components[0] + const remaining = components.slice(1) + + // Special handling for '**': match zero or more directories + if (component === '**') { + if (remaining.length === 0) { + // Just '**' - match all files and directories recursively + yield* matchAll(tree, currentPath) + return + } + + // '**' followed by more patterns + // Try to match the next pattern at this level (zero directories) + yield* matchRecursive(tree, remaining, currentPath) + + // Also try deeper (one or more directories) + for (const dir of tree.directories) { + yield* matchRecursive(dir, components, `${currentPath}/${dir.name}`.replace(/^\//, '')) + } + return + } + + const matcher = createMatcher(component) + + // Check files at this level + if (remaining.length === 0) { + for (const file of tree.files) { + if (matcher(file.name)) { + yield currentPath ? `${currentPath}/${file.name}` : file.name + } + } + } + + // Check directories at this level + for (const dir of tree.directories) { + if (matcher(dir.name)) { + if (remaining.length === 0) { + yield currentPath ? `${currentPath}/${dir.name}` : dir.name + } else { + // Recurse into matched directory + yield* matchRecursive(dir, remaining, `${currentPath}/${dir.name}`.replace(/^\//, '')) + } + } + } +} + +/* + * Glob utility function. Used to handle '**' in glob pattern. + */ +function* matchAll(tree: FileTree, currentPath: string): Generator { + for (const file of tree.files) { + yield currentPath ? `${currentPath}/${file.name}` : file.name + } + for (const dir of tree.directories) { + const newPath = `${currentPath}/${dir.name}`.replace(/^\//, '') + yield newPath + yield* matchAll(dir, newPath) + } +} + +export function glob(this: BIDSContext, toMatch: string): string[] { + const components = parsePattern(toMatch) + if (components.length === 0) { + return [] + } + + return Array.from(matchRecursive(this.dataset.tree, components, '')) +} + export const expressionFunctions = { index: (list: T[], item: T): number | null => { const index = list.indexOf(item) @@ -136,6 +255,7 @@ export const expressionFunctions = { allequal: (a: T[], b: T[]): boolean => { return (a != null && b != null) && a.length === b.length && a.every((v, i) => v === b[i]) }, + glob: glob, } /** From 73fa99bd7cf525dc9ea3093e3e3725c0520ad661 Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Thu, 9 Jul 2026 10:30:55 -0500 Subject: [PATCH 2/3] Add ability to validate index columns for tabular files specified in sidecar. --- src/schema/expressionLanguage.test.ts | 4 ++- src/schema/tables.test.ts | 49 +++++++++++++++++++++++++++ src/schema/tables.ts | 20 +++++++++-- 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/src/schema/expressionLanguage.test.ts b/src/schema/expressionLanguage.test.ts index 801d978e..b71c6cd5 100644 --- a/src/schema/expressionLanguage.test.ts +++ b/src/schema/expressionLanguage.test.ts @@ -1,4 +1,4 @@ -import { assert, assertEquals } from '@std/assert' +import { assert, assertEquals, equal } from '@std/assert' import { contextFunction, createMatcher, @@ -12,6 +12,8 @@ import { import { dataFile, rootFileTree } from './fixtures.test.ts' import type { BIDSContext } from './context.ts' import { makeBIDSContext } from './context.test.ts' +import type { BIDSFile } from '../types/filetree.ts' +import { pathsToTree } from '../files/filetree.test.ts' Deno.test('test expression functions', async (t) => { const context = await makeBIDSContext(dataFile, undefined, rootFileTree) diff --git a/src/schema/tables.test.ts b/src/schema/tables.test.ts index dd54a004..beca5daa 100644 --- a/src/schema/tables.test.ts +++ b/src/schema/tables.test.ts @@ -271,6 +271,55 @@ Deno.test('tables eval* tests', async (t) => { ) }) + await t.step('verify sidecar specified column index works', () => { + const context = { + path: '/sub-01/sub-01_scans.tsv', + extension: '.tsv', + sidecar: { 'IndexColumns': ['onset', 'filename'] }, + columns: new ColumnsMap(Object.entries({ + onset: ['1900-01-01:00:00', '1900-01-01:00:00'], + filename: ['func/sub-01_task-rest_bold.nii.gz', 'func/sub-01_task-unrest_bold.nii.gz'], + })), + dataset: { issues: new DatasetIssues() }, + } + const rule = schemaDefs.rules.tabular_data.modality_agnostic.Scans + evalIndexColumns(rule, context, schema, 'rules.tabular_data.modality_agnostic.Scans') + assertEquals( + context.dataset.issues.get({ code: 'TSV_INDEX_VALUE_NOT_UNIQUE' }).length, + 0, + ) + + // Now test index columns that violate the multi column uniqueness constraint. + context.columns = new ColumnsMap(Object.entries({ + onset: ['1900-01-01:00:00', '1900-01-01:00:00'], + filename: ['func/sub-01_task-rest_bold.nii.gz', 'func/sub-01_task-rest_bold.nii.gz'], + })) + evalIndexColumns(rule, context, schema, 'rules.tabular_data.modality_agnostic.Scans') + assertEquals( + context.dataset.issues.get({ code: 'TSV_INDEX_VALUE_NOT_UNIQUE' }).length, + 1, + ) + }) + + await t.step('verify sidecar specified column index errors on missing column', () => { + const context = { + path: '/sub-01/sub-01_scans.tsv', + extension: '.tsv', + sidecar: { 'IndexColumns': ['onset', 'badcol'] }, + columns: new ColumnsMap(Object.entries({ + onset: ['1900-01-01:00:00', '1900-01-01:00:00'], + filename: ['func/sub-01_task-rest_bold.nii.gz', 'func/sub-01_task-rest_bold.nii.gz'], + })), + dataset: { issues: new DatasetIssues() }, + } + const rule = schemaDefs.rules.tabular_data.modality_agnostic.Scans + evalIndexColumns(rule, context, schema, 'rules.tabular_data.modality_agnostic.Scans') + assertEquals( + context.dataset.issues.get({ code: 'TSV_COLUMN_MISSING' }).length, + 1, + ) + }) + await t.step('verify not allowed additional columns', () => { const context = { path: '/sub-01/sub-01_scans.tsv', diff --git a/src/schema/tables.ts b/src/schema/tables.ts index a934fe43..9648ec9c 100644 --- a/src/schema/tables.ts +++ b/src/schema/tables.ts @@ -370,10 +370,24 @@ export function evalIndexColumns( ) { return } + const uniqueIndexValues = new Set() - const index_columns = rule.index_columns.map((col: string) => { - return schema.objects.columns[col].name - }).filter((col: string) => context.columns[col]) + let index_columns: string[] = [] + if (context.sidecar.IndexColumns) { + index_columns = context.sidecar.IndexColumns as string[] + const missingColumns = index_columns.filter((col) => !context.columns[col]) + if (missingColumns.length) { + context.dataset.issues.add({ + code: 'TSV_COLUMN_MISSING', + location: context.path, + issueMessage: `Sidecar specified IndexColumns are Missing: ${missingColumns}`, + }) + } + } else if (rule.index_columns) { + index_columns = rule.index_columns.map((col: string) => { + return schema.objects.columns[col].name + }).filter((col: string) => context.columns[col]) + } const rowCount = (context.columns[index_columns[0]] as string[])?.length || 0 for (let i = 0; i < rowCount; i++) { From 2c66f3a361704db72cd4b9404fb8dc52aec0a9a1 Mon Sep 17 00:00:00 2001 From: Ross Blair Date: Thu, 9 Jul 2026 10:59:16 -0500 Subject: [PATCH 3/3] add changelog --- ...rosswilsonblair_bep036_extensions_redux.md | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 changelog.d/20260709_105449_rosswilsonblair_bep036_extensions_redux.md diff --git a/changelog.d/20260709_105449_rosswilsonblair_bep036_extensions_redux.md b/changelog.d/20260709_105449_rosswilsonblair_bep036_extensions_redux.md new file mode 100644 index 00000000..eccdc6b5 --- /dev/null +++ b/changelog.d/20260709_105449_rosswilsonblair_bep036_extensions_redux.md @@ -0,0 +1,48 @@ + + +### Added + +- Added glob to expression language +- evalIndexColumns now checks for IndexColumns field in sidecar nad uses it for validation. + + + + + + +