BEP36 rules

src/metaschema.json

@@ -326,8 +326,11 @@
                   "type": "object",
                   "properties": {
                     "issue": {
-                      "allOf": [{ "$ref": "#/definitions/ruleTypes/issue" }],
-                      "required": ["level"]
+                      "allOf": [
+                        { "$ref": "#/definitions/ruleTypes/issue" },
+                        { "required": ["level"] }
+                      ],
+                      "unevaluatedProperties": false
                     },
                     "selectors": {
                       "$ref": "#/definitions/ruleTypes/expressionList"
@@ -473,17 +476,18 @@
           "type": "object",
           "patternProperties": {
             "^[a-zA-Z0-9_]+$": {
-              "type": "object",
-              "properties": {
-                "code": { "type": "string" },
-                "message": { "type": "string" },
-                "level": { "enum": ["error", "warning"] },
-                "selectors": {
-                  "$ref": "#/definitions/ruleTypes/expressionList"
+              "allOf": [
+                { "$ref": "#/definitions/ruleTypes/issue" },
+                {
+                  "properties": {
+                    "selectors": {
+                      "$ref": "#/definitions/ruleTypes/expressionList"
+                    }
+                  },
+                  "required": ["level"]
                 }
-              },
-              "required": ["message", "level"],
-              "additionalProperties": false
+              ],
+              "unevaluatedProperties": false
             }
           },
           "additionalProperties": false
@@ -595,7 +599,10 @@
               "level": { "$ref": "#/definitions/enums/requirement_level" },
               "level_addendum": { "type": "string" },
               "description_addendum": { "type": "string" },
-              "issue": { "$ref": "#/definitions/ruleTypes/issue" }
+              "issue": {
+                "$ref": "#/definitions/ruleTypes/issue",
+                "unevaluatedProperties": false
+              }
             },
             "required": ["level"],
             "additionalProperties": false
@@ -606,11 +613,13 @@
         "type": "object",
         "properties": {
           "code": { "type": "string" },
+          "subCode": { "type": "string" },
+          "level": { "enum": ["error", "warning"] },
+          "location": { "type": "string" },
           "message": { "type": "string" },
-          "level": { "enum": ["error", "warning"] }
+          "suggestion": { "type": "string" }
         },
-        "required": ["code", "message"],
-        "additionalProperties": false
+        "required": ["code", "message"]
       },
       "expressionList": {
         "type": "array",

src/schema/README.md

@@ -286,6 +286,7 @@ The following functions should be defined by an interpreter:
 | `allequal(a: array, b: array) -> bool`            | `true` if arrays have the same length and paired elements are equal                                                                       | `allequal(sorted(columns.onset, "numeric"), columns.onset)` | True if the array columns.onset is sorted numerically.                         |
 | `count(arg: array, val: any) -> int`              | Number of elements in an array equal to `val`                                                                                             | `count(columns.type, "EEG")`                                | The number of times "EEG" appears in the column "type" of the current TSV file |
 | `exists(arg: str \| array, rule: str) -> int`     | Count of files in an array that exist in the dataset. String is array with length 1. See following section for the meanings of rules.     | `exists(sidecar.IntendedFor, "subject")`                    | True if all files in `IntendedFor` exist, relative to the subject directory.   |
+| `glob(arg: str) -> array`                         | An array of files in the dataset that match the `arg` string. Strings may include single and double stars (`*`, `**`).                    | `glob("sub-*")`                                             | Non-empty array if any subject directories are present in the dataset.         |
 | `index(arg: array, val: any) -> int`              | Index of first element in an array equal to `val`, `null` if not found                                                                    | `index(["i", "j", "k"], axis)`                              | The number, from 0-2 corresponding to the string `axis`                        |
 | `intersects(a: array, b: array) -> array \| bool` | The intersection of arrays `a` and `b`, or `false` if there are no shared values.                                                         | `intersects(dataset.modalities, ["pet", "mri"])`            | Non-empty array if either PET or MRI data is found in dataset, otherwise false |
 | `length(arg: array) -> int`                       | Number of elements in an array                                                                                                            | `length(columns.onset) > 0`                                 | True if there is at least one value in the onset column                        |
@@ -296,6 +297,7 @@ The following functions should be defined by an interpreter:
 | `substr(arg: str, start: int, end: int) -> str`   | The portion of the input string spanning from start position to end position                                                              | `substr(path, 0, length(path) - 3)`                         | `path` with the last three characters dropped                                  |
 | `type(arg: Any) -> str`                           | The name of the type, including `"array"`, `"object"`, `"null"`                                                                           | `type(datatypes)`                                           | Returns `"array"`                                                              |
 | `unique(arg: array) -> array)`                    | The unique values of the input array, retaining their input order. Equal float and int values are not considered distinct.                | `length(unique(columns.X)) == length(columns.X)`            | True if column `X` contains no duplicate values.                               |
+| `zip(a: array, b: array, ...) -> array`           | An array of arrays, where the i-th element of the result array contains the i-th element of each input array, in order.                   | `zip(columns.participant_id, columns.session_id)`           | Pairs of participant_id and session_id entries in a TSV file.                  |
 
 #### The `exists()` function
 

src/schema/meta/context.yaml

@@ -39,7 +39,6 @@ properties:
       - ignored
       - datatypes
       - modalities
-      - subjects
     additionalProperties: false
     properties:
       dataset_description:
@@ -63,23 +62,20 @@ properties:
         type: array
         items:
           type: string
-      subjects:
-        description: 'Collections of subjects in dataset'
+      participants_tsv:
+        description: 'Contents of /participants.tsv, accessed by column header'
         type: object
-        required:
-          - sub_dirs
-        additionalProperties: false
-        properties:
-          sub_dirs:
-            description: 'Subjects as determined by sub-* directories'
-            type: array
-            items:
-              type: string
-          participant_id:
-            description: 'The participant_id column of participants.tsv'
-            type: array
-            items:
-              type: string
+        additionalProperties:
+          type: array
+          items:
+            type: string
+      sessions_tsv:
+        description: 'Contents of /sessions.tsv, accessed by column header'
+        type: object
+        additionalProperties:
+          type: array
+          items:
+            type: string
   subject:
     description: 'Properties and contents of the current subject'
     type: object

src/schema/meta/expression_tests.yaml

@@ -26,6 +26,8 @@
   result: false
 - expression: intersects(null, [])
   result: false
+- expression: glob(null)
+  result: null
 - expression: allequal([], null)
   result: false
 - expression: allequal(null, [])
@@ -48,6 +50,14 @@
   result: null
 - expression: unique(null)
   result: null
+- expression: zip([], null)
+  result: null
+- expression: zip(null, [])
+  result: null
+- expression: zip(null, [1])
+  result: null
+- expression: zip([1], null)
+  result: null
 - expression: type(null)
   result: 'null'
 - expression: null == false
@@ -157,3 +167,7 @@
   result: [1]
 - expression: unique([1.0, 1])
   result: [1.0]
+- expression: zip([1, 2, 3], [4, 5, 6])
+  result: [[1, 4], [2, 5], [3, 6]]
+- expression: zip([1, 2, 3], [4, 5, 6], [7, 8, 9])
+  result: [[1, 4, 7], [2, 5, 8], [3, 6, 9]]

src/schema/rules/checks/dataset.yaml

@@ -12,7 +12,7 @@ SubjectFolders:
     - path == '/dataset_description.json'
     - dataset.dataset_description.DatasetType != "study"
   checks:
-    - length(dataset.subjects.sub_dirs) > 0
+    - length(glob("sub-*")) > 0
 
 NoSubjectFolders:
   issue:
@@ -24,7 +24,7 @@ NoSubjectFolders:
     - path == '/dataset_description.json'
     - dataset.dataset_description.DatasetType == "study"
   checks:
-    - length(dataset.subjects.sub_dirs) == 0
+    - length(glob("sub-*")) == 0
 
 # 49
 ParticipantIDMismatch:
@@ -38,8 +38,8 @@ ParticipantIDMismatch:
     - path == '/participants.tsv'
   checks:
     - |
-      length(intersects(unique(columns.participant_id), dataset.subjects.sub_dirs)) ==
-      length(dataset.subjects.sub_dirs)
+      length(intersects(columns.participant_id, glob("sub-*")))
+      == length(glob("sub-*"))
 
 # 214
 SamplesTSVMissing:

src/schema/rules/checks/general.yaml

@@ -35,3 +35,50 @@ DuplicateReadmes:
     - match(path, '^/README.*')
   checks:
     - exists(["README", "README.md", "README.rst", "README.txt"], "dataset") == 1
+
+SegregatedSessions:
+  issue:
+    code: SEGREGATED_SESSIONS
+    message: |
+      Session files were detected outside the dataset root and
+      "Phenotype" additional validation has been enabled.
+      Aggregate session data in a root-level `sessions.tsv` file.
+    level: error
+  selectors:
+    - suffix == 'sessions'
+    - extension == '.tsv'
+    - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"])
+  checks:
+    - type(entities.subject) == 'null'
+
+SessionRequired:
+  issue:
+    code: SESSION_REQUIRED
+    message: |
+      Multi-session data is detected in the dataset, and
+      "Phenotype" additional validation has been enabled.
+      Every data file MUST be found in a session directory.
+    level: error
+  selectors:
+    - type(entities.subject) != 'null'
+    - "!intersects([extension], ['.tsv', '.json'])"
+    - type(dataset.sessions_tsv) != 'null'
+    - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"])
+  checks:
+    - type(entities.session) != 'null'
+
+SessionsTSVRequired:
+  issue:
+    code: SESSIONS_TSV_REQUIRED
+    message: |
+      Multi-session data is detected in the dataset, and
+      "Phenotype" additional validation has been enabled.
+      A sessions.tsv file MUST exist in the dataset root.
+    level: error
+    location: /sessions.tsv
+  selectors:
+    - "!intersects([extension], ['.tsv', '.json'])"
+    - type(entities.session) != 'null'
+    - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"])
+  checks:
+    - type(dataset.sessions_tsv) != 'null'

src/schema/rules/checks/phenotype.yaml

@@ -1,6 +1,5 @@
 # Rules for phenotype files
 ---
-
 # 51
 PhenotypeSubjectsMissing:
   issue:
@@ -11,11 +10,83 @@ PhenotypeSubjectsMissing:
     level: error
   selectors:
     - datatype == 'phenotype'
-    - suffix == '.tsv'
-    - type(dataset.subjects.participant_id) != 'null'
+    - extension == '.tsv'
+    - type(dataset.participant_tsv.participant_id) != 'null'
   checks:
     - |
-      allequal(
-        sorted(intersects(columns.participant_id, dataset.subjects.participant_id)),
-        sorted(dataset.subjects.participant_id)
-      )
+      length(intersects(columns.participant_id, dataset.participant_tsv.participant_id))
+      == length(unique(columns.participant_id))
+
+PhenotypeSessionsRequired:
+  issue:
+    code: PHENOTYPE_SESSIONS_REQUIRED
+    message: |
+      Phenotype TSV files must have a session_id column if a dataset contains multiple sessions.
+    level: error
+  selectors:
+    - datatype == 'phenotype'
+    - extension == '.tsv'
+    - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"])
+    - type(dataset.session_tsv) != 'null'
+  checks:
+    - type(columns.session_id) != 'null'
+
+PhenotypeSessionFileRequired:
+  issue:
+    code: PHENOTYPE_SESSION_FILE_REQUIRED
+    message: |
+      A phenotype TSV file contains a session_id column but no sessions.tsv file
+      was found at the dataset root. A sessions.tsv file is required to define
+      valid session labels.
+    level: error
+  selectors:
+    - datatype == 'phenotype'
+    - extension == '.tsv'
+    - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"])
+    - type(columns.session_id) != 'null'
+  checks:
+    - exists('sessions.tsv', 'dataset')
+
+PhenotypeSessionAggregation:
+  issue:
+    code: PHENOTYPE_SESSION_AGGREGATION
+    message: |
+      Participant-level sessions files (sub-<label>_sessions.tsv) must not exist
+      when a root sessions.tsv file is present. Session information should be
+      aggregated in the root sessions.tsv file.
+    level: error
+  selectors:
+    - suffix == 'sessions'
+    - extension == '.tsv'
+    - type(entities.subject) != 'null'
+  checks:
+    - "!exists('sessions.tsv', 'dataset')"
+
+PhenotypeSessionsMissing:
+  issue:
+    code: PHENOTYPE_SESSIONS_MISSING
+    message: |
+      Session directories were found in this dataset but no sessions.tsv file
+      exists at the dataset root. A sessions.tsv file is required when session
+      directories are present and Phenotype validation is enabled.
+    level: error
+  selectors:
+    - path == '/dataset_description.json'
+    - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"])
+    - length(glob("sub-*/ses-*")) > 0
+  checks:
+    - exists('sessions.tsv', 'dataset')
+
+PhenotypeSessionLevels:
+  issue:
+    code: PHENOTYPE_SESSION_LEVELS
+    message: |
+      The sessions.tsv sidecar must include a Levels field for session_id
+      to define valid session labels and their descriptions.
+    level: error
+  selectors:
+    - suffix == 'sessions'
+    - extension == '.tsv'
+    - intersects(dataset.dataset_description.AdditionalValidation, ["Phenotype"])
+  checks:
+    - type(sidecar.session_id.Levels) != 'null'

tools/schemacode/tests/test_context_types.py

@@ -20,16 +20,12 @@ def test_assignability() -> None:
 
         uvx --with=. mypy tests
     """
-    subjects: p.Subjects = ctx.Subjects([])
-    subjects = ctx.Subjects([], [])
-
     dataset: p.Dataset = ctx.Dataset(
         dataset_description={},
         tree={},
         ignored=[],
         datatypes=[],
         modalities=[],
-        subjects=subjects,
     )
 
     magnitude: p.Magnitude = ctx.Magnitude("path")