diff --git a/src/metaschema.json b/src/metaschema.json index 7d5b83ebf5..4a843788d4 100644 --- a/src/metaschema.json +++ b/src/metaschema.json @@ -326,8 +326,11 @@ "type": "object", "properties": { "issue": { - "allOf": [{ "$ref": "#/definitions/ruleTypes/issue" }], - "required": ["level"] + "allOf": [ + { "$ref": "#/definitions/ruleTypes/issue" }, + { "required": ["level"] } + ], + "unevaluatedProperties": false }, "selectors": { "$ref": "#/definitions/ruleTypes/expressionList" @@ -473,17 +476,18 @@ "type": "object", "patternProperties": { "^[a-zA-Z0-9_]+$": { - "type": "object", - "properties": { - "code": { "type": "string" }, - "message": { "type": "string" }, - "level": { "enum": ["error", "warning"] }, - "selectors": { - "$ref": "#/definitions/ruleTypes/expressionList" + "allOf": [ + { "$ref": "#/definitions/ruleTypes/issue" }, + { + "properties": { + "selectors": { + "$ref": "#/definitions/ruleTypes/expressionList" + } + }, + "required": ["level"] } - }, - "required": ["message", "level"], - "additionalProperties": false + ], + "unevaluatedProperties": false } }, "additionalProperties": false @@ -595,7 +599,10 @@ "level": { "$ref": "#/definitions/enums/requirement_level" }, "level_addendum": { "type": "string" }, "description_addendum": { "type": "string" }, - "issue": { "$ref": "#/definitions/ruleTypes/issue" } + "issue": { + "$ref": "#/definitions/ruleTypes/issue", + "unevaluatedProperties": false + } }, "required": ["level"], "additionalProperties": false @@ -606,11 +613,13 @@ "type": "object", "properties": { "code": { "type": "string" }, + "subCode": { "type": "string" }, + "level": { "enum": ["error", "warning"] }, + "location": { "type": "string" }, "message": { "type": "string" }, - "level": { "enum": ["error", "warning"] } + "suggestion": { "type": "string" } }, - "required": ["code", "message"], - "additionalProperties": false + "required": ["code", "message"] }, "expressionList": { "type": "array", diff --git a/src/schema/README.md b/src/schema/README.md index c0da0fa55e..6edc43e19f 100644 --- a/src/schema/README.md +++ b/src/schema/README.md @@ -286,6 +286,7 @@ The following functions should be defined by an interpreter: | `allequal(a: array, b: array) -> bool` | `true` if arrays have the same length and paired elements are equal | `allequal(sorted(columns.onset, "numeric"), columns.onset)` | True if the array columns.onset is sorted numerically. | | `count(arg: array, val: any) -> int` | Number of elements in an array equal to `val` | `count(columns.type, "EEG")` | The number of times "EEG" appears in the column "type" of the current TSV file | | `exists(arg: str \| array, rule: str) -> int` | Count of files in an array that exist in the dataset. String is array with length 1. See following section for the meanings of rules. | `exists(sidecar.IntendedFor, "subject")` | True if all files in `IntendedFor` exist, relative to the subject directory. | +| `glob(arg: str) -> array` | An array of files in the dataset that match the `arg` string. Strings may include single and double stars (`*`, `**`). | `glob("sub-*")` | Non-empty array if any subject directories are present in the dataset. | | `index(arg: array, val: any) -> int` | Index of first element in an array equal to `val`, `null` if not found | `index(["i", "j", "k"], axis)` | The number, from 0-2 corresponding to the string `axis` | | `intersects(a: array, b: array) -> array \| bool` | The intersection of arrays `a` and `b`, or `false` if there are no shared values. | `intersects(dataset.modalities, ["pet", "mri"])` | Non-empty array if either PET or MRI data is found in dataset, otherwise false | | `length(arg: array) -> int` | Number of elements in an array | `length(columns.onset) > 0` | True if there is at least one value in the onset column | @@ -296,6 +297,7 @@ The following functions should be defined by an interpreter: | `substr(arg: str, start: int, end: int) -> str` | The portion of the input string spanning from start position to end position | `substr(path, 0, length(path) - 3)` | `path` with the last three characters dropped | | `type(arg: Any) -> str` | The name of the type, including `"array"`, `"object"`, `"null"` | `type(datatypes)` | Returns `"array"` | | `unique(arg: array) -> array)` | The unique values of the input array, retaining their input order. Equal float and int values are not considered distinct. | `length(unique(columns.X)) == length(columns.X)` | True if column `X` contains no duplicate values. | +| `zip(a: array, b: array, ...) -> array` | An array of arrays, where the i-th element of the result array contains the i-th element of each input array, in order. | `zip(columns.participant_id, columns.session_id)` | Pairs of participant_id and session_id entries in a TSV file. | #### The `exists()` function diff --git a/src/schema/meta/context.yaml b/src/schema/meta/context.yaml index 7b630d61f5..80949e8ff4 100644 --- a/src/schema/meta/context.yaml +++ b/src/schema/meta/context.yaml @@ -39,7 +39,6 @@ properties: - ignored - datatypes - modalities - - subjects additionalProperties: false properties: dataset_description: @@ -63,23 +62,20 @@ properties: type: array items: type: string - subjects: - description: 'Collections of subjects in dataset' + participants_tsv: + description: 'Contents of /participants.tsv, accessed by column header' type: object - required: - - sub_dirs - additionalProperties: false - properties: - sub_dirs: - description: 'Subjects as determined by sub-* directories' - type: array - items: - type: string - participant_id: - description: 'The participant_id column of participants.tsv' - type: array - items: - type: string + additionalProperties: + type: array + items: + type: string + sessions_tsv: + description: 'Contents of /sessions.tsv, accessed by column header' + type: object + additionalProperties: + type: array + items: + type: string subject: description: 'Properties and contents of the current subject' type: object diff --git a/src/schema/meta/expression_tests.yaml b/src/schema/meta/expression_tests.yaml index ff6e1ab73e..9b65a38839 100644 --- a/src/schema/meta/expression_tests.yaml +++ b/src/schema/meta/expression_tests.yaml @@ -26,6 +26,8 @@ result: false - expression: intersects(null, []) result: false +- expression: glob(null) + result: null - expression: allequal([], null) result: false - expression: allequal(null, []) @@ -48,6 +50,14 @@ result: null - expression: unique(null) result: null +- expression: zip([], null) + result: null +- expression: zip(null, []) + result: null +- expression: zip(null, [1]) + result: null +- expression: zip([1], null) + result: null - expression: type(null) result: 'null' - expression: null == false @@ -157,3 +167,7 @@ result: [1] - expression: unique([1.0, 1]) result: [1.0] +- expression: zip([1, 2, 3], [4, 5, 6]) + result: [[1, 4], [2, 5], [3, 6]] +- expression: zip([1, 2, 3], [4, 5, 6], [7, 8, 9]) + result: [[1, 4, 7], [2, 5, 8], [3, 6, 9]] diff --git a/src/schema/rules/checks/dataset.yaml b/src/schema/rules/checks/dataset.yaml index d2aca48eff..ca87e6b071 100644 --- a/src/schema/rules/checks/dataset.yaml +++ b/src/schema/rules/checks/dataset.yaml @@ -12,7 +12,7 @@ SubjectFolders: - path == '/dataset_description.json' - dataset.dataset_description.DatasetType != "study" checks: - - length(dataset.subjects.sub_dirs) > 0 + - length(glob("sub-*")) > 0 NoSubjectFolders: issue: @@ -24,7 +24,7 @@ NoSubjectFolders: - path == '/dataset_description.json' - dataset.dataset_description.DatasetType == "study" checks: - - length(dataset.subjects.sub_dirs) == 0 + - length(glob("sub-*")) == 0 # 49 ParticipantIDMismatch: @@ -38,8 +38,8 @@ ParticipantIDMismatch: - path == '/participants.tsv' checks: - | - length(intersects(unique(columns.participant_id), dataset.subjects.sub_dirs)) == - length(dataset.subjects.sub_dirs) + length(intersects(columns.participant_id, glob("sub-*"))) + == length(glob("sub-*")) # 214 SamplesTSVMissing: diff --git a/src/schema/rules/checks/general.yaml b/src/schema/rules/checks/general.yaml index f23e81b824..adb10d2886 100644 --- a/src/schema/rules/checks/general.yaml +++ b/src/schema/rules/checks/general.yaml @@ -35,3 +35,50 @@ DuplicateReadmes: - match(path, '^/README.*') checks: - exists(["README", "README.md", "README.rst", "README.txt"], "dataset") == 1 + +SegregatedSessions: + issue: + code: SEGREGATED_SESSIONS + message: | + Session files were detected outside the dataset root and + "Phenotype" additional validation has been enabled. + Aggregate session data in a root-level `sessions.tsv` file. + level: error + selectors: + - suffix == 'sessions' + - extension == '.tsv' + - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"]) + checks: + - type(entities.subject) == 'null' + +SessionRequired: + issue: + code: SESSION_REQUIRED + message: | + Multi-session data is detected in the dataset, and + "Phenotype" additional validation has been enabled. + Every data file MUST be found in a session directory. + level: error + selectors: + - type(entities.subject) != 'null' + - "!intersects([extension], ['.tsv', '.json'])" + - type(dataset.sessions_tsv) != 'null' + - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"]) + checks: + - type(entities.session) != 'null' + +SessionsTSVRequired: + issue: + code: SESSIONS_TSV_REQUIRED + message: | + Multi-session data is detected in the dataset, and + "Phenotype" additional validation has been enabled. + A sessions.tsv file MUST exist in the dataset root. + level: error + location: /sessions.tsv + selectors: + - "!intersects([extension], ['.tsv', '.json'])" + - type(entities.session) != 'null' + - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"]) + checks: + - type(dataset.sessions_tsv) != 'null' diff --git a/src/schema/rules/checks/phenotype.yaml b/src/schema/rules/checks/phenotype.yaml index 1e836508a5..c9b7471e43 100644 --- a/src/schema/rules/checks/phenotype.yaml +++ b/src/schema/rules/checks/phenotype.yaml @@ -1,6 +1,5 @@ # Rules for phenotype files --- - # 51 PhenotypeSubjectsMissing: issue: @@ -11,11 +10,84 @@ PhenotypeSubjectsMissing: level: error selectors: - datatype == 'phenotype' - - suffix == '.tsv' - - type(dataset.subjects.participant_id) != 'null' + - extension == '.tsv' + - type(dataset.participant_tsv.participant_id) != 'null' checks: - | - allequal( - sorted(intersects(columns.participant_id, dataset.subjects.participant_id)), - sorted(dataset.subjects.participant_id) - ) + length(intersects(columns.participant_id, dataset.participant_tsv.participant_id)) + == length(unique(columns.participant_id)) + +PhenotypeSessionsRequired: + issue: + code: PHENOTYPE_SESSIONS_REQUIRED + message: | + Phenotype TSV files must have a session_id column if a dataset contains multiple sessions. + level: error + selectors: + - datatype == 'phenotype' + - extension == '.tsv' + - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"]) + - type(dataset.session_tsv) != 'null' + checks: + - type(columns.session_id) != 'null' + +PhenotypeSessionFileRequired: + issue: + code: PHENOTYPE_SESSION_FILE_REQUIRED + message: | + A phenotype TSV file contains a session_id column but no sessions.tsv file + was found at the dataset root. A sessions.tsv file is required to define + valid session labels. + level: error + selectors: + - datatype == 'phenotype' + - extension == '.tsv' + - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"]) + - type(columns.session_id) != 'null' + checks: + - exists('sessions.tsv', 'dataset') + +PhenotypeSessionAggregation: + issue: + code: PHENOTYPE_SESSION_AGGREGATION + message: | + Participant-level sessions files (sub-