rf(schema): Make participants/sessions.tsv content available, add glob() and zip() functions

src/schema/README.md

@@ -286,6 +286,7 @@ The following functions should be defined by an interpreter:
 | `allequal(a: array, b: array) -> bool`            | `true` if arrays have the same length and paired elements are equal                                                                       | `allequal(sorted(columns.onset, "numeric"), columns.onset)` | True if the array columns.onset is sorted numerically.                         |
 | `count(arg: array, val: any) -> int`              | Number of elements in an array equal to `val`                                                                                             | `count(columns.type, "EEG")`                                | The number of times "EEG" appears in the column "type" of the current TSV file |
 | `exists(arg: str \| array, rule: str) -> int`     | Count of files in an array that exist in the dataset. String is array with length 1. See following section for the meanings of rules.     | `exists(sidecar.IntendedFor, "subject")`                    | True if all files in `IntendedFor` exist, relative to the subject directory.   |
+| `glob(arg: str) -> array`                         | An array of files in the dataset that match the `arg` string. Strings may include single and double stars (`*`, `**`).                    | `glob("sub-*")`                                             | Non-empty array if any subject directories are present in the dataset.         |
 | `index(arg: array, val: any) -> int`              | Index of first element in an array equal to `val`, `null` if not found                                                                    | `index(["i", "j", "k"], axis)`                              | The number, from 0-2 corresponding to the string `axis`                        |
 | `intersects(a: array, b: array) -> array \| bool` | The intersection of arrays `a` and `b`, or `false` if there are no shared values.                                                         | `intersects(dataset.modalities, ["pet", "mri"])`            | Non-empty array if either PET or MRI data is found in dataset, otherwise false |
 | `length(arg: array) -> int`                       | Number of elements in an array                                                                                                            | `length(columns.onset) > 0`                                 | True if there is at least one value in the onset column                        |
@@ -296,6 +297,7 @@ The following functions should be defined by an interpreter:
 | `substr(arg: str, start: int, end: int) -> str`   | The portion of the input string spanning from start position to end position                                                              | `substr(path, 0, length(path) - 3)`                         | `path` with the last three characters dropped                                  |
 | `type(arg: Any) -> str`                           | The name of the type, including `"array"`, `"object"`, `"null"`                                                                           | `type(datatypes)`                                           | Returns `"array"`                                                              |
 | `unique(arg: array) -> array)`                    | The unique values of the input array, retaining their input order. Equal float and int values are not considered distinct.                | `length(unique(columns.X)) == length(columns.X)`            | True if column `X` contains no duplicate values.                               |
+| `zip(a: array, b: array, ...) -> array`           | An array of arrays, where the i-th element of the result array contains the i-th element of each input array, in order.                   | `zip(columns.participant_id, columns.session_id)`           | Pairs of participant_id and session_id entries in a TSV file.                  |
 
 #### The `exists()` function
 

src/schema/meta/context.yaml

@@ -39,7 +39,6 @@ properties:
       - ignored
       - datatypes
       - modalities
-      - subjects
     additionalProperties: false
     properties:
       dataset_description:
@@ -63,23 +62,20 @@ properties:
         type: array
         items:
           type: string
-      subjects:
-        description: 'Collections of subjects in dataset'
+      participants_tsv:
+        description: 'Contents of /participants.tsv, accessed by column header'
         type: object
-        required:
-          - sub_dirs
-        additionalProperties: false
-        properties:
-          sub_dirs:
-            description: 'Subjects as determined by sub-* directories'
-            type: array
-            items:
-              type: string
-          participant_id:
-            description: 'The participant_id column of participants.tsv'
-            type: array
-            items:
-              type: string
+        additionalProperties:
+          type: array
+          items:
+            type: string
+      sessions_tsv:
+        description: 'Contents of /sessions.tsv, accessed by column header'
+        type: object
+        additionalProperties:
+          type: array
+          items:
+            type: string
   subject:
     description: 'Properties and contents of the current subject'
     type: object

src/schema/meta/expression_tests.yaml

@@ -26,6 +26,8 @@
   result: false
 - expression: intersects(null, [])
   result: false
+- expression: glob(null)
+  result: null
 - expression: allequal([], null)
   result: false
 - expression: allequal(null, [])
@@ -48,6 +50,14 @@
   result: null
 - expression: unique(null)
   result: null
+- expression: zip([], null)
+  result: null
+- expression: zip(null, [])
+  result: null
+- expression: zip(null, [1])
+  result: null
+- expression: zip([1], null)
+  result: null
 - expression: type(null)
   result: 'null'
 - expression: null == false
@@ -157,3 +167,7 @@
   result: [1]
 - expression: unique([1.0, 1])
   result: [1.0]
+- expression: zip([1, 2, 3], [4, 5, 6])
+  result: [[1, 4], [2, 5], [3, 6]]
+- expression: zip([1, 2, 3], [4, 5, 6], [7, 8, 9])
+  result: [[1, 4, 7], [2, 5, 8], [3, 6, 9]]

src/schema/rules/checks/dataset.yaml

@@ -12,7 +12,7 @@ SubjectFolders:
     - path == '/dataset_description.json'
     - dataset.dataset_description.DatasetType != "study"
   checks:
-    - length(dataset.subjects.sub_dirs) > 0
+    - length(glob("sub-*")) > 0
 
 NoSubjectFolders:
   issue:
@@ -24,7 +24,7 @@ NoSubjectFolders:
     - path == '/dataset_description.json'
     - dataset.dataset_description.DatasetType == "study"
   checks:
-    - length(dataset.subjects.sub_dirs) == 0
+    - length(glob("sub-*")) == 0
 
 # 49
 ParticipantIDMismatch:
@@ -38,10 +38,8 @@ ParticipantIDMismatch:
     - path == '/participants.tsv'
   checks:
     - |
-      allequal(
-        sorted(intersects(columns.participant_id, dataset.subjects.sub_dirs)),
-        sorted(dataset.subjects.sub_dirs)
-      )
+      length(intersects(columns.participant_id, glob("sub-*")))
+      == length(glob("sub-*"))
 
 # 214
 SamplesTSVMissing:

src/schema/rules/checks/phenotype.yaml

@@ -12,10 +12,8 @@ PhenotypeSubjectsMissing:
   selectors:
     - datatype == 'phenotype'
     - suffix == '.tsv'
-    - type(dataset.subjects.participant_id) != 'null'
+    - type(dataset.participant_tsv.participant_id) != 'null'
   checks:
     - |
-      allequal(
-        sorted(intersects(columns.participant_id, dataset.subjects.participant_id)),
-        sorted(dataset.subjects.participant_id)
-      )
+      length(intersects(columns.participant_id, dataset.participant_tsv.participant_id))
+      == length(dataset.participant_tsv.participant_id)

tools/schemacode/tests/test_context_types.py

@@ -20,16 +20,12 @@ def test_assignability() -> None:
 
         uvx --with=. mypy tests
     """
-    subjects: p.Subjects = ctx.Subjects([])
-    subjects = ctx.Subjects([], [])
-
     dataset: p.Dataset = ctx.Dataset(
         dataset_description={},
         tree={},
         ignored=[],
         datatypes=[],
         modalities=[],
-        subjects=subjects,
     )
 
     magnitude: p.Magnitude = ctx.Magnitude("path")