From 9dc66378c9ca55bd344b7038ff41f6dc2d04d3be Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 5 Mar 2026 19:57:22 -0500 Subject: [PATCH 1/6] feat(schema): Add glob and zip functions to expression language --- src/schema/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/schema/README.md b/src/schema/README.md index c0da0fa55e..6edc43e19f 100644 --- a/src/schema/README.md +++ b/src/schema/README.md @@ -286,6 +286,7 @@ The following functions should be defined by an interpreter: | `allequal(a: array, b: array) -> bool` | `true` if arrays have the same length and paired elements are equal | `allequal(sorted(columns.onset, "numeric"), columns.onset)` | True if the array columns.onset is sorted numerically. | | `count(arg: array, val: any) -> int` | Number of elements in an array equal to `val` | `count(columns.type, "EEG")` | The number of times "EEG" appears in the column "type" of the current TSV file | | `exists(arg: str \| array, rule: str) -> int` | Count of files in an array that exist in the dataset. String is array with length 1. See following section for the meanings of rules. | `exists(sidecar.IntendedFor, "subject")` | True if all files in `IntendedFor` exist, relative to the subject directory. | +| `glob(arg: str) -> array` | An array of files in the dataset that match the `arg` string. Strings may include single and double stars (`*`, `**`). | `glob("sub-*")` | Non-empty array if any subject directories are present in the dataset. | | `index(arg: array, val: any) -> int` | Index of first element in an array equal to `val`, `null` if not found | `index(["i", "j", "k"], axis)` | The number, from 0-2 corresponding to the string `axis` | | `intersects(a: array, b: array) -> array \| bool` | The intersection of arrays `a` and `b`, or `false` if there are no shared values. | `intersects(dataset.modalities, ["pet", "mri"])` | Non-empty array if either PET or MRI data is found in dataset, otherwise false | | `length(arg: array) -> int` | Number of elements in an array | `length(columns.onset) > 0` | True if there is at least one value in the onset column | @@ -296,6 +297,7 @@ The following functions should be defined by an interpreter: | `substr(arg: str, start: int, end: int) -> str` | The portion of the input string spanning from start position to end position | `substr(path, 0, length(path) - 3)` | `path` with the last three characters dropped | | `type(arg: Any) -> str` | The name of the type, including `"array"`, `"object"`, `"null"` | `type(datatypes)` | Returns `"array"` | | `unique(arg: array) -> array)` | The unique values of the input array, retaining their input order. Equal float and int values are not considered distinct. | `length(unique(columns.X)) == length(columns.X)` | True if column `X` contains no duplicate values. | +| `zip(a: array, b: array, ...) -> array` | An array of arrays, where the i-th element of the result array contains the i-th element of each input array, in order. | `zip(columns.participant_id, columns.session_id)` | Pairs of participant_id and session_id entries in a TSV file. | #### The `exists()` function From 37df2e4f28d6375965dee98849e44d852950dd2d Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 5 Mar 2026 19:58:02 -0500 Subject: [PATCH 2/6] feat(schema): Add dataset.{participants,sessions}_tsv to context --- src/schema/meta/context.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/schema/meta/context.yaml b/src/schema/meta/context.yaml index 7b630d61f5..700b64982c 100644 --- a/src/schema/meta/context.yaml +++ b/src/schema/meta/context.yaml @@ -63,6 +63,20 @@ properties: type: array items: type: string + participants_tsv: + description: 'Contents of /participants.tsv, accessed by column header' + type: object + additionalProperties: + type: array + items: + type: string + sessions_tsv: + description: 'Contents of /sessions.tsv, accessed by column header' + type: object + additionalProperties: + type: array + items: + type: string subjects: description: 'Collections of subjects in dataset' type: object From 878bc4afd45000dadb0bb19729879d542b3b0575 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 5 Mar 2026 19:59:32 -0500 Subject: [PATCH 3/6] rf(schema): Rewrite rules to use new context, glob() --- src/schema/rules/checks/dataset.yaml | 10 ++++------ src/schema/rules/checks/phenotype.yaml | 8 +++----- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/schema/rules/checks/dataset.yaml b/src/schema/rules/checks/dataset.yaml index 5fbf91c4a6..ca87e6b071 100644 --- a/src/schema/rules/checks/dataset.yaml +++ b/src/schema/rules/checks/dataset.yaml @@ -12,7 +12,7 @@ SubjectFolders: - path == '/dataset_description.json' - dataset.dataset_description.DatasetType != "study" checks: - - length(dataset.subjects.sub_dirs) > 0 + - length(glob("sub-*")) > 0 NoSubjectFolders: issue: @@ -24,7 +24,7 @@ NoSubjectFolders: - path == '/dataset_description.json' - dataset.dataset_description.DatasetType == "study" checks: - - length(dataset.subjects.sub_dirs) == 0 + - length(glob("sub-*")) == 0 # 49 ParticipantIDMismatch: @@ -38,10 +38,8 @@ ParticipantIDMismatch: - path == '/participants.tsv' checks: - | - allequal( - sorted(intersects(columns.participant_id, dataset.subjects.sub_dirs)), - sorted(dataset.subjects.sub_dirs) - ) + length(intersects(columns.participant_id, glob("sub-*"))) + == length(glob("sub-*")) # 214 SamplesTSVMissing: diff --git a/src/schema/rules/checks/phenotype.yaml b/src/schema/rules/checks/phenotype.yaml index 1e836508a5..c359671447 100644 --- a/src/schema/rules/checks/phenotype.yaml +++ b/src/schema/rules/checks/phenotype.yaml @@ -12,10 +12,8 @@ PhenotypeSubjectsMissing: selectors: - datatype == 'phenotype' - suffix == '.tsv' - - type(dataset.subjects.participant_id) != 'null' + - type(dataset.participant_tsv.participant_id) != 'null' checks: - | - allequal( - sorted(intersects(columns.participant_id, dataset.subjects.participant_id)), - sorted(dataset.subjects.participant_id) - ) + length(intersects(columns.participant_id, dataset.participant_tsv.participant_id)) + == length(dataset.participant_tsv.participant_id) From 0708e30d0fcc7309e56b3d7012c0305cf82dd10c Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 5 Mar 2026 20:00:30 -0500 Subject: [PATCH 4/6] rf(schema): Remove unused dataset.subjects from context --- src/schema/meta/context.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/schema/meta/context.yaml b/src/schema/meta/context.yaml index 700b64982c..80949e8ff4 100644 --- a/src/schema/meta/context.yaml +++ b/src/schema/meta/context.yaml @@ -39,7 +39,6 @@ properties: - ignored - datatypes - modalities - - subjects additionalProperties: false properties: dataset_description: @@ -77,23 +76,6 @@ properties: type: array items: type: string - subjects: - description: 'Collections of subjects in dataset' - type: object - required: - - sub_dirs - additionalProperties: false - properties: - sub_dirs: - description: 'Subjects as determined by sub-* directories' - type: array - items: - type: string - participant_id: - description: 'The participant_id column of participants.tsv' - type: array - items: - type: string subject: description: 'Properties and contents of the current subject' type: object From 4b3fdef96c11b2828b7862f3615f061c817cdeda Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 5 Mar 2026 21:47:55 -0500 Subject: [PATCH 5/6] test(schema): Add expression tests --- src/schema/meta/expression_tests.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/schema/meta/expression_tests.yaml b/src/schema/meta/expression_tests.yaml index ff6e1ab73e..9b65a38839 100644 --- a/src/schema/meta/expression_tests.yaml +++ b/src/schema/meta/expression_tests.yaml @@ -26,6 +26,8 @@ result: false - expression: intersects(null, []) result: false +- expression: glob(null) + result: null - expression: allequal([], null) result: false - expression: allequal(null, []) @@ -48,6 +50,14 @@ result: null - expression: unique(null) result: null +- expression: zip([], null) + result: null +- expression: zip(null, []) + result: null +- expression: zip(null, [1]) + result: null +- expression: zip([1], null) + result: null - expression: type(null) result: 'null' - expression: null == false @@ -157,3 +167,7 @@ result: [1] - expression: unique([1.0, 1]) result: [1.0] +- expression: zip([1, 2, 3], [4, 5, 6]) + result: [[1, 4], [2, 5], [3, 6]] +- expression: zip([1, 2, 3], [4, 5, 6], [7, 8, 9]) + result: [[1, 4, 7], [2, 5, 8], [3, 6, 9]] From 46f2777e9c202fc919992c7082ae13f6fbf151ca Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 5 Mar 2026 21:51:18 -0500 Subject: [PATCH 6/6] test(bst): Drop Subjects class from context types --- tools/schemacode/tests/test_context_types.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/schemacode/tests/test_context_types.py b/tools/schemacode/tests/test_context_types.py index d0692ee6d7..99f7050813 100644 --- a/tools/schemacode/tests/test_context_types.py +++ b/tools/schemacode/tests/test_context_types.py @@ -20,16 +20,12 @@ def test_assignability() -> None: uvx --with=. mypy tests """ - subjects: p.Subjects = ctx.Subjects([]) - subjects = ctx.Subjects([], []) - dataset: p.Dataset = ctx.Dataset( dataset_description={}, tree={}, ignored=[], datatypes=[], modalities=[], - subjects=subjects, ) magnitude: p.Magnitude = ctx.Magnitude("path")