From 9dc66378c9ca55bd344b7038ff41f6dc2d04d3be Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 5 Mar 2026 19:57:22 -0500 Subject: [PATCH 01/14] feat(schema): Add glob and zip functions to expression language --- src/schema/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/schema/README.md b/src/schema/README.md index c0da0fa55e..6edc43e19f 100644 --- a/src/schema/README.md +++ b/src/schema/README.md @@ -286,6 +286,7 @@ The following functions should be defined by an interpreter: | `allequal(a: array, b: array) -> bool` | `true` if arrays have the same length and paired elements are equal | `allequal(sorted(columns.onset, "numeric"), columns.onset)` | True if the array columns.onset is sorted numerically. | | `count(arg: array, val: any) -> int` | Number of elements in an array equal to `val` | `count(columns.type, "EEG")` | The number of times "EEG" appears in the column "type" of the current TSV file | | `exists(arg: str \| array, rule: str) -> int` | Count of files in an array that exist in the dataset. String is array with length 1. See following section for the meanings of rules. | `exists(sidecar.IntendedFor, "subject")` | True if all files in `IntendedFor` exist, relative to the subject directory. | +| `glob(arg: str) -> array` | An array of files in the dataset that match the `arg` string. Strings may include single and double stars (`*`, `**`). | `glob("sub-*")` | Non-empty array if any subject directories are present in the dataset. | | `index(arg: array, val: any) -> int` | Index of first element in an array equal to `val`, `null` if not found | `index(["i", "j", "k"], axis)` | The number, from 0-2 corresponding to the string `axis` | | `intersects(a: array, b: array) -> array \| bool` | The intersection of arrays `a` and `b`, or `false` if there are no shared values. | `intersects(dataset.modalities, ["pet", "mri"])` | Non-empty array if either PET or MRI data is found in dataset, otherwise false | | `length(arg: array) -> int` | Number of elements in an array | `length(columns.onset) > 0` | True if there is at least one value in the onset column | @@ -296,6 +297,7 @@ The following functions should be defined by an interpreter: | `substr(arg: str, start: int, end: int) -> str` | The portion of the input string spanning from start position to end position | `substr(path, 0, length(path) - 3)` | `path` with the last three characters dropped | | `type(arg: Any) -> str` | The name of the type, including `"array"`, `"object"`, `"null"` | `type(datatypes)` | Returns `"array"` | | `unique(arg: array) -> array)` | The unique values of the input array, retaining their input order. Equal float and int values are not considered distinct. | `length(unique(columns.X)) == length(columns.X)` | True if column `X` contains no duplicate values. | +| `zip(a: array, b: array, ...) -> array` | An array of arrays, where the i-th element of the result array contains the i-th element of each input array, in order. | `zip(columns.participant_id, columns.session_id)` | Pairs of participant_id and session_id entries in a TSV file. | #### The `exists()` function From 37df2e4f28d6375965dee98849e44d852950dd2d Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 5 Mar 2026 19:58:02 -0500 Subject: [PATCH 02/14] feat(schema): Add dataset.{participants,sessions}_tsv to context --- src/schema/meta/context.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/schema/meta/context.yaml b/src/schema/meta/context.yaml index 7b630d61f5..700b64982c 100644 --- a/src/schema/meta/context.yaml +++ b/src/schema/meta/context.yaml @@ -63,6 +63,20 @@ properties: type: array items: type: string + participants_tsv: + description: 'Contents of /participants.tsv, accessed by column header' + type: object + additionalProperties: + type: array + items: + type: string + sessions_tsv: + description: 'Contents of /sessions.tsv, accessed by column header' + type: object + additionalProperties: + type: array + items: + type: string subjects: description: 'Collections of subjects in dataset' type: object From 878bc4afd45000dadb0bb19729879d542b3b0575 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 5 Mar 2026 19:59:32 -0500 Subject: [PATCH 03/14] rf(schema): Rewrite rules to use new context, glob() --- src/schema/rules/checks/dataset.yaml | 10 ++++------ src/schema/rules/checks/phenotype.yaml | 8 +++----- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/schema/rules/checks/dataset.yaml b/src/schema/rules/checks/dataset.yaml index 5fbf91c4a6..ca87e6b071 100644 --- a/src/schema/rules/checks/dataset.yaml +++ b/src/schema/rules/checks/dataset.yaml @@ -12,7 +12,7 @@ SubjectFolders: - path == '/dataset_description.json' - dataset.dataset_description.DatasetType != "study" checks: - - length(dataset.subjects.sub_dirs) > 0 + - length(glob("sub-*")) > 0 NoSubjectFolders: issue: @@ -24,7 +24,7 @@ NoSubjectFolders: - path == '/dataset_description.json' - dataset.dataset_description.DatasetType == "study" checks: - - length(dataset.subjects.sub_dirs) == 0 + - length(glob("sub-*")) == 0 # 49 ParticipantIDMismatch: @@ -38,10 +38,8 @@ ParticipantIDMismatch: - path == '/participants.tsv' checks: - | - allequal( - sorted(intersects(columns.participant_id, dataset.subjects.sub_dirs)), - sorted(dataset.subjects.sub_dirs) - ) + length(intersects(columns.participant_id, glob("sub-*"))) + == length(glob("sub-*")) # 214 SamplesTSVMissing: diff --git a/src/schema/rules/checks/phenotype.yaml b/src/schema/rules/checks/phenotype.yaml index 1e836508a5..c359671447 100644 --- a/src/schema/rules/checks/phenotype.yaml +++ b/src/schema/rules/checks/phenotype.yaml @@ -12,10 +12,8 @@ PhenotypeSubjectsMissing: selectors: - datatype == 'phenotype' - suffix == '.tsv' - - type(dataset.subjects.participant_id) != 'null' + - type(dataset.participant_tsv.participant_id) != 'null' checks: - | - allequal( - sorted(intersects(columns.participant_id, dataset.subjects.participant_id)), - sorted(dataset.subjects.participant_id) - ) + length(intersects(columns.participant_id, dataset.participant_tsv.participant_id)) + == length(dataset.participant_tsv.participant_id) From 0708e30d0fcc7309e56b3d7012c0305cf82dd10c Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 5 Mar 2026 20:00:30 -0500 Subject: [PATCH 04/14] rf(schema): Remove unused dataset.subjects from context --- src/schema/meta/context.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/src/schema/meta/context.yaml b/src/schema/meta/context.yaml index 700b64982c..80949e8ff4 100644 --- a/src/schema/meta/context.yaml +++ b/src/schema/meta/context.yaml @@ -39,7 +39,6 @@ properties: - ignored - datatypes - modalities - - subjects additionalProperties: false properties: dataset_description: @@ -77,23 +76,6 @@ properties: type: array items: type: string - subjects: - description: 'Collections of subjects in dataset' - type: object - required: - - sub_dirs - additionalProperties: false - properties: - sub_dirs: - description: 'Subjects as determined by sub-* directories' - type: array - items: - type: string - participant_id: - description: 'The participant_id column of participants.tsv' - type: array - items: - type: string subject: description: 'Properties and contents of the current subject' type: object From 4b3fdef96c11b2828b7862f3615f061c817cdeda Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 5 Mar 2026 21:47:55 -0500 Subject: [PATCH 05/14] test(schema): Add expression tests --- src/schema/meta/expression_tests.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/schema/meta/expression_tests.yaml b/src/schema/meta/expression_tests.yaml index ff6e1ab73e..9b65a38839 100644 --- a/src/schema/meta/expression_tests.yaml +++ b/src/schema/meta/expression_tests.yaml @@ -26,6 +26,8 @@ result: false - expression: intersects(null, []) result: false +- expression: glob(null) + result: null - expression: allequal([], null) result: false - expression: allequal(null, []) @@ -48,6 +50,14 @@ result: null - expression: unique(null) result: null +- expression: zip([], null) + result: null +- expression: zip(null, []) + result: null +- expression: zip(null, [1]) + result: null +- expression: zip([1], null) + result: null - expression: type(null) result: 'null' - expression: null == false @@ -157,3 +167,7 @@ result: [1] - expression: unique([1.0, 1]) result: [1.0] +- expression: zip([1, 2, 3], [4, 5, 6]) + result: [[1, 4], [2, 5], [3, 6]] +- expression: zip([1, 2, 3], [4, 5, 6], [7, 8, 9]) + result: [[1, 4, 7], [2, 5, 8], [3, 6, 9]] From 46f2777e9c202fc919992c7082ae13f6fbf151ca Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Thu, 5 Mar 2026 21:51:18 -0500 Subject: [PATCH 06/14] test(bst): Drop Subjects class from context types --- tools/schemacode/tests/test_context_types.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tools/schemacode/tests/test_context_types.py b/tools/schemacode/tests/test_context_types.py index d0692ee6d7..99f7050813 100644 --- a/tools/schemacode/tests/test_context_types.py +++ b/tools/schemacode/tests/test_context_types.py @@ -20,16 +20,12 @@ def test_assignability() -> None: uvx --with=. mypy tests """ - subjects: p.Subjects = ctx.Subjects([]) - subjects = ctx.Subjects([], []) - dataset: p.Dataset = ctx.Dataset( dataset_description={}, tree={}, ignored=[], datatypes=[], modalities=[], - subjects=subjects, ) magnitude: p.Magnitude = ctx.Magnitude("path") From 882734ada03732a00d4458bb622def2616e3260f Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 16 Mar 2026 22:12:24 -0400 Subject: [PATCH 07/14] feat(schema): Add check for session_id columns --- src/schema/rules/checks/phenotype.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/schema/rules/checks/phenotype.yaml b/src/schema/rules/checks/phenotype.yaml index c359671447..5692252e82 100644 --- a/src/schema/rules/checks/phenotype.yaml +++ b/src/schema/rules/checks/phenotype.yaml @@ -17,3 +17,17 @@ PhenotypeSubjectsMissing: - | length(intersects(columns.participant_id, dataset.participant_tsv.participant_id)) == length(dataset.participant_tsv.participant_id) + +PhenotypeSessionsRequired: + issue: + code: PHENOTYPE_SESSIONS_REQUIRED + message: | + Phenotype TSV files must have a session_id column if a dataset contains multiple sessions. + level: error + selectors: + - datatype == 'phenotype' + - suffix == '.tsv' + - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"]) + - type(dataset.session_tsv) != 'null' + checks: + - type(columns.session_id) != 'null' From abf16ca8b2b81844db02a721f54f23481e865b51 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 16 Mar 2026 22:24:00 -0400 Subject: [PATCH 08/14] feat(schema): Require aggregated session files --- src/schema/rules/checks/general.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/schema/rules/checks/general.yaml b/src/schema/rules/checks/general.yaml index f23e81b824..bfdd7111cc 100644 --- a/src/schema/rules/checks/general.yaml +++ b/src/schema/rules/checks/general.yaml @@ -35,3 +35,18 @@ DuplicateReadmes: - match(path, '^/README.*') checks: - exists(["README", "README.md", "README.rst", "README.txt"], "dataset") == 1 + +SegregatedSessions: + issue: + code: SEGREGATED_SESSIONS + message: | + Session files were detected outside the dataset root and + "Phenotype" additional validation has been enabled. + Aggregate session data in a root-level `sessions.tsv` file. + level: error + selectors: + - suffix == 'sessions' + - extension == '.tsv' + - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"]) + checks: + - type(entities.subject) == 'null' From 2ec8859dc327bb3347e6d194832916ccbe92975d Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Mon, 16 Mar 2026 22:28:16 -0400 Subject: [PATCH 09/14] feat(schema): Require session directories if sessions.tsv is defined --- src/schema/rules/checks/general.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/src/schema/rules/checks/general.yaml b/src/schema/rules/checks/general.yaml index bfdd7111cc..bde5510aee 100644 --- a/src/schema/rules/checks/general.yaml +++ b/src/schema/rules/checks/general.yaml @@ -50,3 +50,18 @@ SegregatedSessions: - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"]) checks: - type(entities.subject) == 'null' + +SessionRequired: + issue: + code: SESSION_REQUIRED + message: | + Multi-session data is detected in the dataset, and + "Phenotype" additional validation has been enabled. + Every data file MUST be found in a session directory. + level: error + selectors: + - "!intersects([extension], ['.tsv', '.json'])" + - exists("sessions.tsv", "dataset") + - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"]) + checks: + - type(entities.session) != 'null' From b0501990fc7a8966de89db1114aabf71302118b9 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 18 Mar 2026 14:27:48 -0400 Subject: [PATCH 10/14] fix: Require subject entity to trigger session entity check --- src/schema/rules/checks/general.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/schema/rules/checks/general.yaml b/src/schema/rules/checks/general.yaml index bde5510aee..a7e80d9810 100644 --- a/src/schema/rules/checks/general.yaml +++ b/src/schema/rules/checks/general.yaml @@ -60,8 +60,9 @@ SessionRequired: Every data file MUST be found in a session directory. level: error selectors: + - type(entities.subject) != 'null' - "!intersects([extension], ['.tsv', '.json'])" - - exists("sessions.tsv", "dataset") + - type(dataset.sessions_tsv) != 'null' - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"]) checks: - type(entities.session) != 'null' From 512c8d4d16259678a3a042e0b88a7fb39e9ef7c5 Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 18 Mar 2026 14:28:33 -0400 Subject: [PATCH 11/14] feat: Check for sessions.tsv file on any file with session entity --- src/schema/rules/checks/general.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/schema/rules/checks/general.yaml b/src/schema/rules/checks/general.yaml index a7e80d9810..adb10d2886 100644 --- a/src/schema/rules/checks/general.yaml +++ b/src/schema/rules/checks/general.yaml @@ -66,3 +66,19 @@ SessionRequired: - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"]) checks: - type(entities.session) != 'null' + +SessionsTSVRequired: + issue: + code: SESSIONS_TSV_REQUIRED + message: | + Multi-session data is detected in the dataset, and + "Phenotype" additional validation has been enabled. + A sessions.tsv file MUST exist in the dataset root. + level: error + location: /sessions.tsv + selectors: + - "!intersects([extension], ['.tsv', '.json'])" + - type(entities.session) != 'null' + - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"]) + checks: + - type(dataset.sessions_tsv) != 'null' From 1c9fef62a9b50697cbd73cbb40da8a3ce83cabda Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Wed, 18 Mar 2026 15:04:46 -0400 Subject: [PATCH 12/14] feat(metaschema): Allow for more complete issues --- src/metaschema.json | 41 +++++++++++++++++++++++++---------------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/src/metaschema.json b/src/metaschema.json index 7d5b83ebf5..4a843788d4 100644 --- a/src/metaschema.json +++ b/src/metaschema.json @@ -326,8 +326,11 @@ "type": "object", "properties": { "issue": { - "allOf": [{ "$ref": "#/definitions/ruleTypes/issue" }], - "required": ["level"] + "allOf": [ + { "$ref": "#/definitions/ruleTypes/issue" }, + { "required": ["level"] } + ], + "unevaluatedProperties": false }, "selectors": { "$ref": "#/definitions/ruleTypes/expressionList" @@ -473,17 +476,18 @@ "type": "object", "patternProperties": { "^[a-zA-Z0-9_]+$": { - "type": "object", - "properties": { - "code": { "type": "string" }, - "message": { "type": "string" }, - "level": { "enum": ["error", "warning"] }, - "selectors": { - "$ref": "#/definitions/ruleTypes/expressionList" + "allOf": [ + { "$ref": "#/definitions/ruleTypes/issue" }, + { + "properties": { + "selectors": { + "$ref": "#/definitions/ruleTypes/expressionList" + } + }, + "required": ["level"] } - }, - "required": ["message", "level"], - "additionalProperties": false + ], + "unevaluatedProperties": false } }, "additionalProperties": false @@ -595,7 +599,10 @@ "level": { "$ref": "#/definitions/enums/requirement_level" }, "level_addendum": { "type": "string" }, "description_addendum": { "type": "string" }, - "issue": { "$ref": "#/definitions/ruleTypes/issue" } + "issue": { + "$ref": "#/definitions/ruleTypes/issue", + "unevaluatedProperties": false + } }, "required": ["level"], "additionalProperties": false @@ -606,11 +613,13 @@ "type": "object", "properties": { "code": { "type": "string" }, + "subCode": { "type": "string" }, + "level": { "enum": ["error", "warning"] }, + "location": { "type": "string" }, "message": { "type": "string" }, - "level": { "enum": ["error", "warning"] } + "suggestion": { "type": "string" } }, - "required": ["code", "message"], - "additionalProperties": false + "required": ["code", "message"] }, "expressionList": { "type": "array", From ecda3ad8a278f9911e0efda8735664495806106a Mon Sep 17 00:00:00 2001 From: "Christopher J. Markiewicz" Date: Fri, 20 Mar 2026 14:56:45 -0400 Subject: [PATCH 13/14] feat(schema): add BEP036 phenotype check rules Fix PhenotypeSubjectsMissing selector (suffix -> extension) and check direction (phenotype participants subset of participants.tsv). Add PhenotypeSessionFileRequired, PhenotypeSessionAggregation, PhenotypeSessionsMissing, and PhenotypeSessionLevels check rules for phenotype validation under AdditionalValidation. --- src/schema/rules/checks/phenotype.yaml | 67 ++++++++++++++++++++++++-- 1 file changed, 63 insertions(+), 4 deletions(-) diff --git a/src/schema/rules/checks/phenotype.yaml b/src/schema/rules/checks/phenotype.yaml index 5692252e82..ed8f0452cf 100644 --- a/src/schema/rules/checks/phenotype.yaml +++ b/src/schema/rules/checks/phenotype.yaml @@ -1,6 +1,5 @@ # Rules for phenotype files --- - # 51 PhenotypeSubjectsMissing: issue: @@ -11,12 +10,12 @@ PhenotypeSubjectsMissing: level: error selectors: - datatype == 'phenotype' - - suffix == '.tsv' + - extension == '.tsv' - type(dataset.participant_tsv.participant_id) != 'null' checks: - | length(intersects(columns.participant_id, dataset.participant_tsv.participant_id)) - == length(dataset.participant_tsv.participant_id) + == length(unique(columns.participant_id)) PhenotypeSessionsRequired: issue: @@ -26,8 +25,68 @@ PhenotypeSessionsRequired: level: error selectors: - datatype == 'phenotype' - - suffix == '.tsv' + - extension == '.tsv' - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"]) - type(dataset.session_tsv) != 'null' checks: - type(columns.session_id) != 'null' + +PhenotypeSessionFileRequired: + issue: + code: PHENOTYPE_SESSION_FILE_REQUIRED + message: | + A phenotype TSV file contains a session_id column but no sessions.tsv file + was found at the dataset root. A sessions.tsv file is required to define + valid session labels. + level: error + selectors: + - datatype == 'phenotype' + - extension == '.tsv' + - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"]) + - type(columns.session_id) != 'null' + checks: + - exists('sessions.tsv', 'dataset') + +PhenotypeSessionAggregation: + issue: + code: PHENOTYPE_SESSION_AGGREGATION + message: | + Participant-level sessions files (sub-