From 1b7234f381f6b24a44d7bec8ad8bf0f48695e91b Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Fri, 17 May 2024 12:33:32 +0200 Subject: [PATCH 01/33] [FIX] Minor YAML formatting --- src/schema/objects/columns.yaml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml index 5010d2453f..742cf928db 100644 --- a/src/schema/objects/columns.yaml +++ b/src/schema/objects/columns.yaml @@ -47,8 +47,7 @@ alpha_rotation: display_name: Alpha rotation description: | Euler angle to match probe extension dimensions (width, height, depth) to global x, y, z coordinates. - unit: - degree + unit: degree associated_brain_region: name: associated_brain_region display_name: Associated brain region @@ -67,16 +66,15 @@ associated_brain_region_quality_type: name: associated_brain_region_quality_type display_name: Associated brain region quality type description: | - The method used to identify the associated brain region (estimated|proof) - depending on anatomical pictures proofing the location or indirect estimation of the location. + The method used to identify the associated brain region (estimated|proof) + depending on anatomical pictures proofing the location or indirect estimation of the location. type: string beta_rotation: name: beta_rotation display_name: Beta rotation description: | Euler angle to match probe extension dimensions (width, height, depth) to global x, y, z coordinates. - unit: - degree + unit: degree cardiac: name: cardiac display_name: Cardiac measurement @@ -231,8 +229,7 @@ gamma_rotation: display_name: Gamma rotation description: | Euler angle to match probe extension dimensions (width, height, depth) to global x, y, z coordinates. - unit: - degree + unit: degree group__channel: name: group display_name: Channel group From 47c2a034c5a88c7402ff67ff3e27ba237cbdb709 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Fri, 17 May 2024 12:36:27 +0200 Subject: [PATCH 02/33] [ENH] Added additional birthdate column for participants.tsv --- src/schema/objects/columns.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml index 742cf928db..dc3fbb2488 100644 --- a/src/schema/objects/columns.yaml +++ b/src/schema/objects/columns.yaml @@ -69,6 +69,13 @@ associated_brain_region_quality_type: The method used to identify the associated brain region (estimated|proof) depending on anatomical pictures proofing the location or indirect estimation of the location. type: string +birthdate: + name: birthdate + display_name: Birthdate + description: | + Day of birth of the participant. Datetime format and their anonymization are described in [Units](SPEC_ROOT/common-principles.md#units). + type: string + format: datetime beta_rotation: name: beta_rotation display_name: Beta rotation From cfc163e1ff8643f24386093e5b79541ecf9bcb92 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Fri, 17 May 2024 12:37:57 +0200 Subject: [PATCH 03/33] [ENH] Added age_category, age_type and corresponding enums --- src/schema/objects/columns.yaml | 17 +++++++++++++++++ src/schema/objects/enums.yaml | 21 +++++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml index dc3fbb2488..85ecddf36c 100644 --- a/src/schema/objects/columns.yaml +++ b/src/schema/objects/columns.yaml @@ -42,6 +42,23 @@ age: for privacy purposes. type: number unit: year +age_category: + name: age_category + display_name: Age category + description: | + For non-numeric description of the age (e.g. developmental stage) a string can be provided. + We RECOMMEND to use the [UBERON life cycle stage](http://purl.obolibrary.org/obo/UBERON_0000105) to specify it. + type: string +age_type: + name: age_type + display_name: Age type + description: | + Type of age measurement (e.g. `postnatal`, `fetal`, `gestational`). If not specified, `postnatal` is assumed. + type: string + enum: + - $ref: objects.enums.postnatal.value + - $ref: objects.enums.fetal.value + - $ref: objects.enums.gestational.value alpha_rotation: name: alpha_rotation display_name: Alpha rotation diff --git a/src/schema/objects/enums.yaml b/src/schema/objects/enums.yaml index f01d16cdc6..24dccdfde2 100644 --- a/src/schema/objects/enums.yaml +++ b/src/schema/objects/enums.yaml @@ -1405,3 +1405,24 @@ microvascular: display_name: microvascular description: | The origin of a tissue: microvascular +postnatal: + value: postnatal + display_name: postnatal + description: | + This is the actual time that has passed since a subject's birth, + also known as “chronological age”. + If age_type is omitted, age is assumed to be this type. +fetal: + value: fetal + display_name: fetal + description: | + Also known as "embryonic age," this measures the age of a + fetus or embryo from the time of conception. It's typically about two weeks + less than the gestational age. +gestational: + value: gestational + display_name: gestational + description: | + This measures the duration of pregnancy from the first day of the + mother's last menstrual period to the current date. This should be included + for experiments where some subjects have not been born. From b9894f5a6ae23e20fbdcc30cd0ce05d53e4bd458 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Fri, 17 May 2024 12:39:24 +0200 Subject: [PATCH 04/33] [ENH] Added surgery_date column --- src/schema/objects/columns.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml index 85ecddf36c..4aa8c9f66e 100644 --- a/src/schema/objects/columns.yaml +++ b/src/schema/objects/columns.yaml @@ -720,6 +720,14 @@ strain_rrid: of the strain of the species, for example: `RRID:IMSR_JAX:000664`. type: string format: rrid +# TODO: Clarify if column values can be arrays. +surgery_date: + name: surgery_date + display_name: Surgery date + description: | + Date(s) of surgery. Datetime format and their anonymization are described in [Units](SPEC_ROOT/common-principles.md#units). The details of the surgery should be described in the sidecar json file. + type: string + format: datetime time: name: time display_name: Time From 414f27298eeeba6b108b52a887ad8adbe74708db Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 May 2024 10:53:14 +0000 Subject: [PATCH 05/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/schema/objects/enums.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/schema/objects/enums.yaml b/src/schema/objects/enums.yaml index 24dccdfde2..6f430e17e4 100644 --- a/src/schema/objects/enums.yaml +++ b/src/schema/objects/enums.yaml @@ -1409,20 +1409,20 @@ postnatal: value: postnatal display_name: postnatal description: | - This is the actual time that has passed since a subject's birth, - also known as “chronological age”. + This is the actual time that has passed since a subject's birth, + also known as “chronological age”. If age_type is omitted, age is assumed to be this type. fetal: value: fetal display_name: fetal description: | - Also known as "embryonic age," this measures the age of a - fetus or embryo from the time of conception. It's typically about two weeks + Also known as "embryonic age," this measures the age of a + fetus or embryo from the time of conception. It's typically about two weeks less than the gestational age. gestational: value: gestational display_name: gestational description: | - This measures the duration of pregnancy from the first day of the - mother's last menstrual period to the current date. This should be included + This measures the duration of pregnancy from the first day of the + mother's last menstrual period to the current date. This should be included for experiments where some subjects have not been born. From 3833e28a128bfad189eaf7f38a3631235dcd7616 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Mon, 27 May 2024 13:48:26 +0200 Subject: [PATCH 06/33] Reverted changes that better suit in a PR against master --- src/schema/objects/columns.yaml | 24 ------------------------ src/schema/objects/enums.yaml | 22 +--------------------- 2 files changed, 1 insertion(+), 45 deletions(-) diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml index 4aa8c9f66e..45e4668298 100644 --- a/src/schema/objects/columns.yaml +++ b/src/schema/objects/columns.yaml @@ -42,23 +42,6 @@ age: for privacy purposes. type: number unit: year -age_category: - name: age_category - display_name: Age category - description: | - For non-numeric description of the age (e.g. developmental stage) a string can be provided. - We RECOMMEND to use the [UBERON life cycle stage](http://purl.obolibrary.org/obo/UBERON_0000105) to specify it. - type: string -age_type: - name: age_type - display_name: Age type - description: | - Type of age measurement (e.g. `postnatal`, `fetal`, `gestational`). If not specified, `postnatal` is assumed. - type: string - enum: - - $ref: objects.enums.postnatal.value - - $ref: objects.enums.fetal.value - - $ref: objects.enums.gestational.value alpha_rotation: name: alpha_rotation display_name: Alpha rotation @@ -86,13 +69,6 @@ associated_brain_region_quality_type: The method used to identify the associated brain region (estimated|proof) depending on anatomical pictures proofing the location or indirect estimation of the location. type: string -birthdate: - name: birthdate - display_name: Birthdate - description: | - Day of birth of the participant. Datetime format and their anonymization are described in [Units](SPEC_ROOT/common-principles.md#units). - type: string - format: datetime beta_rotation: name: beta_rotation display_name: Beta rotation diff --git a/src/schema/objects/enums.yaml b/src/schema/objects/enums.yaml index 6f430e17e4..76a9604f6d 100644 --- a/src/schema/objects/enums.yaml +++ b/src/schema/objects/enums.yaml @@ -1405,24 +1405,4 @@ microvascular: display_name: microvascular description: | The origin of a tissue: microvascular -postnatal: - value: postnatal - display_name: postnatal - description: | - This is the actual time that has passed since a subject's birth, - also known as “chronological age”. - If age_type is omitted, age is assumed to be this type. -fetal: - value: fetal - display_name: fetal - description: | - Also known as "embryonic age," this measures the age of a - fetus or embryo from the time of conception. It's typically about two weeks - less than the gestational age. -gestational: - value: gestational - display_name: gestational - description: | - This measures the duration of pregnancy from the first day of the - mother's last menstrual period to the current date. This should be included - for experiments where some subjects have not been born. + From 7079c1823ed2c9518e1dd8931de76ec2dd02e13c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 11:49:06 +0000 Subject: [PATCH 07/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/schema/objects/enums.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/src/schema/objects/enums.yaml b/src/schema/objects/enums.yaml index 76a9604f6d..f01d16cdc6 100644 --- a/src/schema/objects/enums.yaml +++ b/src/schema/objects/enums.yaml @@ -1405,4 +1405,3 @@ microvascular: display_name: microvascular description: | The origin of a tissue: microvascular - From 11bffa07b51307bba69c8035930854a61616c327 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Mon, 27 May 2024 17:14:56 +0200 Subject: [PATCH 08/33] [ENH] Added tabular_data file and columns for _probes.tsv --- src/schema/objects/columns.yaml | 87 +++++++++++++++++++++--- src/schema/rules/tabular_data/ephys.yaml | 37 ++++++++++ 2 files changed, 115 insertions(+), 9 deletions(-) create mode 100644 src/schema/rules/tabular_data/ephys.yaml diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml index 45e4668298..29ceccc92e 100644 --- a/src/schema/objects/columns.yaml +++ b/src/schema/objects/columns.yaml @@ -123,13 +123,22 @@ contact_count: description: | Number of miscellaneous analog contacts for auxiliary signals. type: number +coordinate_reference_point: + name: coordinate_reference_point + display_name: Coordinate reference point + description: | + Point of the probe that is described by the probe coordinates and + on which the alpha, beta and gamma rotations are applied. + type: string depth__probes: - name: height - display_name: Height + name: depth + display_name: Depth description: | - Physical height of the probe. + Physical depth of the probe, e.g. 0.3. + This dimension should be omitted or set to 0 for two-dimensional (shank-type) probes. + This dimension corresponds to the z’axis of the Euler transformation defined by + alpha, beta and gamma rotations values. type: number - unit: mm detector__channels: name: detector display_name: Detector Name @@ -202,6 +211,14 @@ dimension: Size of the group (grid/strip/probe) that this electrode belongs to. Must be of form `[AxB]` with the smallest dimension first (for example, `[1x8]`). type: string +# TODO: Add unit enums if everyone agrees to keep this column. +dimension_unit: + name: dimension_unit + display_name: Dimension unit + description: | + Units of the physical dimensions 'width', 'height' and 'depth' of the probe. + For example, `mm`. + type: string duration: name: duration display_name: Event duration @@ -217,6 +234,12 @@ duration: - type: string enum: - n/a +electrode_count: + name: electrode_count + display_name: Electrode count + description: | + Number of miscellaneous analog electrodes for auxiliary signals (e.g. 2). + type: number filename: name: filename display_name: Filename @@ -275,9 +298,10 @@ height__probes: name: height display_name: Height description: | - Physical height of the probe. + Physical height of the probe, e.g. 5. + This dimension corresponds to the y’axis of the Euler transformation defined by + alpha, beta and gamma rotations values below. type: number - unit: mm hemisphere: name: hemisphere display_name: Electrode hemisphere @@ -287,6 +311,16 @@ hemisphere: enum: - $ref: objects.enums.left_hemisphere.value - $ref: objects.enums.right_hemisphere.value +# TODO: Should the hemisphere details be unified for electrodes and probes or will it break other modality? +hemisphere__probes: + name: hemisphere + display_name: Hemisphere + description: | + Hemisphere in which the probe is placed. + type: string + enum: + - $ref: objects.enums.left_hemisphere.value + - $ref: objects.enums.right_hemisphere.value high_cutoff: name: high_cutoff display_name: High cutoff @@ -341,6 +375,13 @@ manufacturer: The manufacturer for each electrode. Can be used if electrodes were manufactured by more than one company. type: string +# TODO: Should the manufacturer details be unified for electrodes and probes? +manufacturer__probes: + name: manufacturer + display_name: Manufacturer + description: | + Manufacturer of the probes system (e.g. 'openephys', 'alphaomega','blackrock'). + type: string mapping: name: mapping display_name: Label mapping @@ -463,6 +504,13 @@ plasma_radioactivity: Radioactivity in plasma, in unit of plasma radioactivity (for example, `kBq/mL`). type: number # reference column for channels.tsv files for EEG data +probe_id: + name: probe_id + display_name: Probe ID + description: | + A unique identifier of the probe, can be identical with the `device_serial_number`. + (expected to match probe_ids listed in `*_eletrodes.tsv`). + type: string reference__eeg: name: reference display_name: Electrode reference @@ -487,7 +535,8 @@ reference_atlas: name: reference_atlas display_name: Reference atlas description: | - Name of reference atlas used for associated brain region identification, preferably an ebrains supported atlas. + Name of reference atlas used for associated brain region identification, + preferably an [ebrains supported atlas](https://ebrains.eu/services/atlases#services). type: string reference_frame: name: reference_frame @@ -901,9 +950,10 @@ width__probes: name: width display_name: Width description: | - Physical width of the probe. + Physical width of the probe, e.g. 5. + This dimension corresponds to the x’axis of the Euler transformation defined by + alpha, beta and gamma rotations values. type: number - unit: mm x: name: x display_name: X position @@ -959,6 +1009,25 @@ z__optodes: - type: string enum: - n/a +#TODO: How are the units specified for the probe x, y and z fields? +x__probes: + name: x + display_name: Probe X position + description: | + Probe position along the global coordinate system x-axis. + type: number +y__probes: + name: y + display_name: Probe Y position + description: | + Probe position along the global coordinate system y-axis. + type: number +z__probes: + name: z + display_name: Probe Z position + description: | + Probe position along the global coordinate system z-axis. + type: number template_x: name: template_x display_name: X template position diff --git a/src/schema/rules/tabular_data/ephys.yaml b/src/schema/rules/tabular_data/ephys.yaml new file mode 100644 index 0000000000..640ab480f0 --- /dev/null +++ b/src/schema/rules/tabular_data/ephys.yaml @@ -0,0 +1,37 @@ +ephysProbes: + selectors: + - datatype == "ephys" + - suffix == "probes" + - extension == ".tsv" + initial_columns: + - probe_id + - type__probes + - x__probes + - y__probes + - z__probes + columns: + probe_id: required + type__probes: required + x__probes: recommended + y__probes: recommended + z__probes: recommended + manufacturer__probes: recommended + device_serial_number: recommended + electrode_count: optional + width__probes: optional + height__probes: optional + depth__probes: optional + dimension_unit: optional + alpha_rotation: recommended + beta_rotation: recommended + gamma_rotation: recommended + coordinate_reference_point: recommended + hemisphere__probes: recommended + associated_brain_region: recommended + associated_brain_region_id: recommended + associated_brain_region_quality_type: recommended + reference_atlas: recommended + material__probes: optional + index_columns: [probe_id] + additional_columns: allowed_if_defined + \ No newline at end of file From ab66a633a0db6651e338957b95f71f5f94749a00 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 15:16:08 +0000 Subject: [PATCH 09/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/schema/objects/columns.yaml | 22 +++++++++++----------- src/schema/rules/tabular_data/ephys.yaml | 1 - 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml index 29ceccc92e..297c44606c 100644 --- a/src/schema/objects/columns.yaml +++ b/src/schema/objects/columns.yaml @@ -127,16 +127,16 @@ coordinate_reference_point: name: coordinate_reference_point display_name: Coordinate reference point description: | - Point of the probe that is described by the probe coordinates and + Point of the probe that is described by the probe coordinates and on which the alpha, beta and gamma rotations are applied. type: string depth__probes: name: depth display_name: Depth description: | - Physical depth of the probe, e.g. 0.3. - This dimension should be omitted or set to 0 for two-dimensional (shank-type) probes. - This dimension corresponds to the z’axis of the Euler transformation defined by + Physical depth of the probe, e.g. 0.3. + This dimension should be omitted or set to 0 for two-dimensional (shank-type) probes. + This dimension corresponds to the z’axis of the Euler transformation defined by alpha, beta and gamma rotations values. type: number detector__channels: @@ -211,7 +211,7 @@ dimension: Size of the group (grid/strip/probe) that this electrode belongs to. Must be of form `[AxB]` with the smallest dimension first (for example, `[1x8]`). type: string -# TODO: Add unit enums if everyone agrees to keep this column. +# TODO: Add unit enums if everyone agrees to keep this column. dimension_unit: name: dimension_unit display_name: Dimension unit @@ -298,8 +298,8 @@ height__probes: name: height display_name: Height description: | - Physical height of the probe, e.g. 5. - This dimension corresponds to the y’axis of the Euler transformation defined by + Physical height of the probe, e.g. 5. + This dimension corresponds to the y’axis of the Euler transformation defined by alpha, beta and gamma rotations values below. type: number hemisphere: @@ -535,7 +535,7 @@ reference_atlas: name: reference_atlas display_name: Reference atlas description: | - Name of reference atlas used for associated brain region identification, + Name of reference atlas used for associated brain region identification, preferably an [ebrains supported atlas](https://ebrains.eu/services/atlases#services). type: string reference_frame: @@ -950,8 +950,8 @@ width__probes: name: width display_name: Width description: | - Physical width of the probe, e.g. 5. - This dimension corresponds to the x’axis of the Euler transformation defined by + Physical width of the probe, e.g. 5. + This dimension corresponds to the x’axis of the Euler transformation defined by alpha, beta and gamma rotations values. type: number x: @@ -1009,7 +1009,7 @@ z__optodes: - type: string enum: - n/a -#TODO: How are the units specified for the probe x, y and z fields? +#TODO: How are the units specified for the probe x, y and z fields? x__probes: name: x display_name: Probe X position diff --git a/src/schema/rules/tabular_data/ephys.yaml b/src/schema/rules/tabular_data/ephys.yaml index 640ab480f0..1cb9309aa0 100644 --- a/src/schema/rules/tabular_data/ephys.yaml +++ b/src/schema/rules/tabular_data/ephys.yaml @@ -34,4 +34,3 @@ ephysProbes: material__probes: optional index_columns: [probe_id] additional_columns: allowed_if_defined - \ No newline at end of file From b3c2b433d34761c19409e58ae195065ca0db5c7b Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Mon, 27 May 2024 17:49:46 +0200 Subject: [PATCH 10/33] [ENH] Added tabular_data rules and columns for _electrodes.tsv --- src/schema/objects/columns.yaml | 68 ++++++++++++++++++++++++ src/schema/rules/tabular_data/ephys.yaml | 33 +++++++++++- 2 files changed, 100 insertions(+), 1 deletion(-) diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml index 29ceccc92e..0731b625d6 100644 --- a/src/schema/objects/columns.yaml +++ b/src/schema/objects/columns.yaml @@ -240,6 +240,19 @@ electrode_count: description: | Number of miscellaneous analog electrodes for auxiliary signals (e.g. 2). type: number +electrode_id: + name: electrode_id + display_name: Electrode ID + description: | + A unique identifier of the electrode. + (expected to match electrode_ids listed in `*_channels.tsv`). + type: string +electrode_shape: + name: electrode_shape + display_name: Electrode shape + description: | + Description of the shape of the electrode (for example, `square`, `circle`). + type: string filename: name: filename display_name: Filename @@ -356,6 +369,12 @@ index: description: | The label integer index. type: integer +location: + name: location + display_name: Location + description: | + An indication on the location of the electrode (for example, `cortical layer 3`, `CA1`). + type: string low_cutoff: name: low_cutoff display_name: Low cutoff @@ -489,6 +508,27 @@ participant_id: matching a participant entity found in the dataset. type: string pattern: ^sub-[0-9a-zA-Z]+$ +pipette_solution: + name: pipette_solution + display_name: Pipette solution + description: | + The solution used to fill the pipette. + See also [openMINDS Pipette](https://github.com/openMetadataInitiative/openMINDS_ephys/blob/v1/schemas/device/pipetteUsage.schema.tpl.json). + type: string +internal_pipette_diameter: + name: internal_pipette_diameter + display_name: Internal pipette diameter + description: | + The internal diameter of the pipette in micrometers. + type: number + unit: mm +external_pipette_diameter: + name: external_pipette_diameter + display_name: External pipette diameter + description: | + The external diameter of the pipette in micrometers. + type: number + unit: mm placement__motion: name: placement display_name: Placement @@ -639,6 +679,13 @@ sex: - OTHER - Other - n/a +shank_id: + name: shank_id + display_name: Shank ID + description: | + A unique identifier to specify which shank of the probe the electrode is on. + This is useful for spike sorting when the electrodes are on a multi-shank probe. + type: string short_channel: name: short_channel display_name: Short Channel @@ -1028,6 +1075,27 @@ z__probes: description: | Probe position along the global coordinate system z-axis. type: number +x__electrodes: + name: x + display_name: Electrode X position + description: | + Recorded position along the local width-axis relative + to the probe origin and rotation (see `_probes.tsv`) + type: number +y__electrodes: + name: y + display_name: Electrode Y position + description: | + Recorded position along the local height-axis relative + to the probe origin and rotation (see `_probes.tsv`) + type: number +z__electrodes: + name: z + display_name: Electrode Z position + description: | + Recorded position along the local depth-axis relative + to the probe origin and rotation (see `_probes.tsv`) + type: number template_x: name: template_x display_name: X template position diff --git a/src/schema/rules/tabular_data/ephys.yaml b/src/schema/rules/tabular_data/ephys.yaml index 640ab480f0..c3ffde7e5c 100644 --- a/src/schema/rules/tabular_data/ephys.yaml +++ b/src/schema/rules/tabular_data/ephys.yaml @@ -34,4 +34,35 @@ ephysProbes: material__probes: optional index_columns: [probe_id] additional_columns: allowed_if_defined - \ No newline at end of file + + +ephysElectrodes: + selectors: + - datatype == "ephys" + - suffix == "electrodes" + - extension == ".tsv" + initial_columns: + - electrode_id + - probe_id + - hemisphere + - x__electrodes + - y__electrodes + - z__electrodes + columns: + electrode_id: required + probe_id: required + hemisphere: recommended + x__electrodes: recommended + y__electrodes: recommended + z__electrodes: recommended + impedance: recommended + shank_id: optional + size: optional + electrode_shape: optional + material: optional + location: recommended + pipette_solution: optional + internal_pipette_diameter: optional + external_pipette_diameter: optional + index_columns: [electrode_id] + additional_columns: allowed_if_defined \ No newline at end of file From 29704cec7f4c1212c1eb00fa41c6789b667888e6 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 15:51:30 +0000 Subject: [PATCH 11/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/schema/objects/columns.yaml | 10 +++++----- src/schema/rules/tabular_data/ephys.yaml | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml index 44a67adb1b..3554f243ab 100644 --- a/src/schema/objects/columns.yaml +++ b/src/schema/objects/columns.yaml @@ -512,7 +512,7 @@ pipette_solution: name: pipette_solution display_name: Pipette solution description: | - The solution used to fill the pipette. + The solution used to fill the pipette. See also [openMINDS Pipette](https://github.com/openMetadataInitiative/openMINDS_ephys/blob/v1/schemas/device/pipetteUsage.schema.tpl.json). type: string internal_pipette_diameter: @@ -683,7 +683,7 @@ shank_id: name: shank_id display_name: Shank ID description: | - A unique identifier to specify which shank of the probe the electrode is on. + A unique identifier to specify which shank of the probe the electrode is on. This is useful for spike sorting when the electrodes are on a multi-shank probe. type: string short_channel: @@ -1079,21 +1079,21 @@ x__electrodes: name: x display_name: Electrode X position description: | - Recorded position along the local width-axis relative + Recorded position along the local width-axis relative to the probe origin and rotation (see `_probes.tsv`) type: number y__electrodes: name: y display_name: Electrode Y position description: | - Recorded position along the local height-axis relative + Recorded position along the local height-axis relative to the probe origin and rotation (see `_probes.tsv`) type: number z__electrodes: name: z display_name: Electrode Z position description: | - Recorded position along the local depth-axis relative + Recorded position along the local depth-axis relative to the probe origin and rotation (see `_probes.tsv`) type: number template_x: diff --git a/src/schema/rules/tabular_data/ephys.yaml b/src/schema/rules/tabular_data/ephys.yaml index 640ab480f0..1cb9309aa0 100644 --- a/src/schema/rules/tabular_data/ephys.yaml +++ b/src/schema/rules/tabular_data/ephys.yaml @@ -34,4 +34,3 @@ ephysProbes: material__probes: optional index_columns: [probe_id] additional_columns: allowed_if_defined - \ No newline at end of file From 73e600ec00c6596298562dbccdbb893483def020 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Mon, 27 May 2024 17:54:25 +0200 Subject: [PATCH 12/33] [ENH] Added tabular_data rules for _electrodes.tsv --- src/schema/rules/tabular_data/ephys.yaml | 31 ++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/src/schema/rules/tabular_data/ephys.yaml b/src/schema/rules/tabular_data/ephys.yaml index 1cb9309aa0..a5463b69b0 100644 --- a/src/schema/rules/tabular_data/ephys.yaml +++ b/src/schema/rules/tabular_data/ephys.yaml @@ -34,3 +34,34 @@ ephysProbes: material__probes: optional index_columns: [probe_id] additional_columns: allowed_if_defined + +ephysElectrodes: + selectors: + - datatype == "ephys" + - suffix == "electrodes" + - extension == ".tsv" + initial_columns: + - electrode_id + - probe_id + - hemisphere + - x__electrodes + - y__electrodes + - z__electrodes + columns: + electrode_id: required + probe_id: required + hemisphere: recommended + x__electrodes: recommended + y__electrodes: recommended + z__electrodes: recommended + impedance: recommended + shank_id: optional + size: optional + electrode_shape: optional + material: optional + location: recommended + pipette_solution: optional + internal_pipette_diameter: optional + external_pipette_diameter: optional + index_columns: [electrode_id] + additional_columns: allowed_if_defined From e67f946a9eeff8d9a2079791bb9a022e6add0d78 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Mon, 27 May 2024 19:01:09 +0200 Subject: [PATCH 13/33] [ENH] Added tabular_data rules and columns for _channels.tsv --- src/schema/objects/columns.yaml | 103 +++++++++++++++++++++++ src/schema/rules/tabular_data/ephys.yaml | 33 ++++++++ 2 files changed, 136 insertions(+) diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml index 3554f243ab..61d201a579 100644 --- a/src/schema/objects/columns.yaml +++ b/src/schema/objects/columns.yaml @@ -93,6 +93,24 @@ channel: In the absence of a delimiter, tools MUST interpret any character as being part of a channel name. type: string +channel_id: + name: channel_id + display_name: Channel ID + description: | + Unique identifier of the channel, only containing letters and numbers. + This id must be reflected in the data files, e.g. as signal annotations in the nix file. + When inserting existing nwb files into a BIDS structure, one can combine + the nwb record_id and channel_index to create this unique identifier. + type: string + pattern: ^[0-9a-zA-Z]+$ +channel_label: + name: channel_label + display_name: Channel label + description: | + Human readable identifier. Use this name to specify the + content of signals not generated by electrodes, + e.g. DAQ internal synchronization signals, behavioral signals, behavioral cues, etc. + type: string color: name: color display_name: Color label @@ -260,12 +278,29 @@ filename: Relative paths to files. type: string format: participant_relative +# TODO: Figure out and add rule for default values. +gain: + name: gain + display_name: Gain + description: | + Amplification factor applied from signal detection at the electrode + to the signal stored in the data file. + If no gain factor is provided it is assumed to be 1. + type: number gamma_rotation: name: gamma_rotation display_name: Gamma rotation description: | Euler angle to match probe extension dimensions (width, height, depth) to global x, y, z coordinates. unit: degree +#TODO: Clarify if the optionality of the column be defined here or as an addendum in the rules. +ground: + name: ground + display_name: Ground + description: | + Information on the ground. For example, 'chamber screw', 'head post', 'ear clip'. + Only should be used to optionally override the global ground in the `_ephys.json` file. + type: string group__channel: name: group display_name: Channel group @@ -307,6 +342,18 @@ handedness: - AMBIDEXTROUS - Ambidextrous - n/a +hardware_filters: + name: hardware_filters + display_name: Hardware filters + description: | + List of hardware (amplifier) filter keys applied. + Note that parameters should be defined in the general ephys sidecar .json file. + Indicate `n/a` in the absence of hardware filters applied. + anyOf: + - type: string + - type: string + enum: + - n/a height__probes: name: height display_name: Height @@ -551,6 +598,12 @@ probe_id: A unique identifier of the probe, can be identical with the `device_serial_number`. (expected to match probe_ids listed in `*_eletrodes.tsv`). type: string +recording_mode: + name: recording_mode + display_name: Recording mode + description: | + The mode of recording for patch clamp datasets (for example, `voltage clamp`, `current clamp`). + type: string reference__eeg: name: reference display_name: Electrode reference @@ -559,6 +612,13 @@ reference__eeg: This column is not needed when it is common to all channels. In that case the reference electrode(s) can be specified in `*_eeg.json` as `EEGReference`). type: string +reference__ephys: + name: reference + display_name: Reference + description: | + Name of the electrode used as physical reference. + For example, electrode_id, physical location (subdural, chamber screw). + type: string # reference column for channels.tsv files for iEEG data reference__ieeg: name: reference @@ -714,6 +774,20 @@ software_filters: - type: string enum: - n/a +# TODO: This seems very confusing. Should be merge with the column above? +software_filters__channels: + name: software_filters + display_name: Software filters + description: | + List of temporal and/or spatial software filters applied + (for example, `SSS`, `SpatialCompensation`). + Note that parameters should be defined in the general ephys sidecar .json file. + Indicate `n/a` in the absence of software filters applied. + anyOf: + - type: string + - type: string + enum: + - n/a source__channels: name: source display_name: Source name @@ -792,6 +866,12 @@ strain_rrid: of the strain of the species, for example: `RRID:IMSR_JAX:000664`. type: string format: rrid +stream_id: + name: stream_id + display_name: Stream ID + description: | + Data stream of the recording the signal. + type: string # TODO: Clarify if column values can be arrays. surgery_date: name: surgery_date @@ -808,6 +888,19 @@ time: For example, 5. type: number unit: s +time_offset: + name: time_offset + display_name: Time offset + description: | + Time shift between signal of this channel to a reference channel in seconds. + type: number + unit: s +time_reference_channel: + name: time_reference_channel + display_name: Time reference channel + description: | + Name of the channel that is used for time alignment of signals. + type: string tracked_point__channels: name: tracked_point display_name: Tracked point channel @@ -831,6 +924,7 @@ trigger: description: | continuous measurement of the scanner trigger signal type: number +# TODO: Populate enums that make sense for channels in the ephys datasets. # type column in channels.tsv files type__channels: name: type @@ -920,6 +1014,15 @@ units: (see [Units](SPEC_ROOT/common-principles.md#units)). type: string format: unit +units__channels: + name: units + display_name: Units + description: | + Physical unit of the value represented in this channel, + for example, `V` for Volt, or `uV` for micro Volt + (see [Units](SPEC_ROOT/common-principles.md#units)). + type: string + format: unit units__nirs: name: units display_name: Units diff --git a/src/schema/rules/tabular_data/ephys.yaml b/src/schema/rules/tabular_data/ephys.yaml index a5463b69b0..4e2c28b6d8 100644 --- a/src/schema/rules/tabular_data/ephys.yaml +++ b/src/schema/rules/tabular_data/ephys.yaml @@ -65,3 +65,36 @@ ephysElectrodes: external_pipette_diameter: optional index_columns: [electrode_id] additional_columns: allowed_if_defined + +ephysChannels: + selectors: + - datatype == "ephys" + - suffix == "channels" + - extension == ".tsv" + initial_columns: + - channel_id + - reference__ephys + - type__channels + - units__channels + - sampling_frequency + columns: + - channel_id: required + - reference__ephys: required + - type__channels: required + - units__channels: required + - sampling_frequency: optional + - channel_label: optional + - stream_id: optional + - description: optional + - hardware_filters: recommended + - software_filters: recommended + - status: optional + - status_description: optional + - gain: recommended + - time_offset: optional + - time_reference_channel: optional + - ground: optional + - recording_mode: recommended + index_columns: [channel_id] + additional_columns: allowed_if_defined + From 1622986879f994a3235b6d40faa1a87b85f2eb83 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 27 May 2024 17:01:35 +0000 Subject: [PATCH 14/33] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- src/schema/objects/columns.yaml | 16 ++++++++-------- src/schema/rules/tabular_data/ephys.yaml | 3 +-- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml index 61d201a579..197a3faf3f 100644 --- a/src/schema/objects/columns.yaml +++ b/src/schema/objects/columns.yaml @@ -97,9 +97,9 @@ channel_id: name: channel_id display_name: Channel ID description: | - Unique identifier of the channel, only containing letters and numbers. - This id must be reflected in the data files, e.g. as signal annotations in the nix file. - When inserting existing nwb files into a BIDS structure, one can combine + Unique identifier of the channel, only containing letters and numbers. + This id must be reflected in the data files, e.g. as signal annotations in the nix file. + When inserting existing nwb files into a BIDS structure, one can combine the nwb record_id and channel_index to create this unique identifier. type: string pattern: ^[0-9a-zA-Z]+$ @@ -107,9 +107,9 @@ channel_label: name: channel_label display_name: Channel label description: | - Human readable identifier. Use this name to specify the - content of signals not generated by electrodes, - e.g. DAQ internal synchronization signals, behavioral signals, behavioral cues, etc. + Human readable identifier. Use this name to specify the + content of signals not generated by electrodes, + e.g. DAQ internal synchronization signals, behavioral signals, behavioral cues, etc. type: string color: name: color @@ -283,8 +283,8 @@ gain: name: gain display_name: Gain description: | - Amplification factor applied from signal detection at the electrode - to the signal stored in the data file. + Amplification factor applied from signal detection at the electrode + to the signal stored in the data file. If no gain factor is provided it is assumed to be 1. type: number gamma_rotation: diff --git a/src/schema/rules/tabular_data/ephys.yaml b/src/schema/rules/tabular_data/ephys.yaml index 4e2c28b6d8..a75d1c009f 100644 --- a/src/schema/rules/tabular_data/ephys.yaml +++ b/src/schema/rules/tabular_data/ephys.yaml @@ -96,5 +96,4 @@ ephysChannels: - ground: optional - recording_mode: recommended index_columns: [channel_id] - additional_columns: allowed_if_defined - + additional_columns: allowed_if_defined From 749a7c9606c721076eb0343d26a30578544a725b Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Tue, 18 Jun 2024 19:24:14 +0200 Subject: [PATCH 15/33] [FIX] Fixed pre-commit errors --- src/schema/objects/columns.yaml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml index 197a3faf3f..82a51eb348 100644 --- a/src/schema/objects/columns.yaml +++ b/src/schema/objects/columns.yaml @@ -293,7 +293,7 @@ gamma_rotation: description: | Euler angle to match probe extension dimensions (width, height, depth) to global x, y, z coordinates. unit: degree -#TODO: Clarify if the optionality of the column be defined here or as an addendum in the rules. +# TODO: Clarify if the optionality of the column be defined here or as an addendum in the rules. ground: name: ground display_name: Ground @@ -560,7 +560,8 @@ pipette_solution: display_name: Pipette solution description: | The solution used to fill the pipette. - See also [openMINDS Pipette](https://github.com/openMetadataInitiative/openMINDS_ephys/blob/v1/schemas/device/pipetteUsage.schema.tpl.json). + See also [openMINDS Pipette] + (https://github.com/openMetadataInitiative/openMINDS_ephys/blob/v1/schemas/device/pipetteUsage.schema.tpl.json). type: string internal_pipette_diameter: name: internal_pipette_diameter @@ -877,7 +878,9 @@ surgery_date: name: surgery_date display_name: Surgery date description: | - Date(s) of surgery. Datetime format and their anonymization are described in [Units](SPEC_ROOT/common-principles.md#units). The details of the surgery should be described in the sidecar json file. + Date(s) of surgery. Datetime format and their anonymization + are described in [Units](SPEC_ROOT/common-principles.md#units). + The details of the surgery should be described in the sidecar json file. type: string format: datetime time: @@ -1159,7 +1162,7 @@ z__optodes: - type: string enum: - n/a -#TODO: How are the units specified for the probe x, y and z fields? +# TODO: How are the units specified for the probe x, y and z fields? x__probes: name: x display_name: Probe X position From 329b1e2b325587e819fd35fa769db9e7a29a2d10 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Tue, 18 Jun 2024 19:26:31 +0200 Subject: [PATCH 16/33] [ENH] Added additional ephys columns for events.tsv --- src/schema/objects/columns.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml index 82a51eb348..ca6d36a695 100644 --- a/src/schema/objects/columns.yaml +++ b/src/schema/objects/columns.yaml @@ -148,6 +148,14 @@ coordinate_reference_point: Point of the probe that is described by the probe coordinates and on which the alpha, beta and gamma rotations are applied. type: string +data_entity_id: + name: data_entity_id + display_name: Data entity ID + description: | + The corresponding (UUID) identifier of the recording trace in the data file. + The same `data_entity_id` can be listed multiple times with different + `onset` & `duration` values to select a subset of a recording trace. + type: string depth__probes: name: depth display_name: Depth @@ -252,6 +260,12 @@ duration: - type: string enum: - n/a +duration_index: + name: duration_index + display_name: Duration index + description: | + Number of recording samples included in the event. + type: number electrode_count: name: electrode_count display_name: Electrode count @@ -538,6 +552,12 @@ onset: acquired data point. type: number unit: s +onset_index: + name: onset_index + display_name: Onset index + description: | + Index of the onset recording sample (0 based) within the data entity. + type: integer pathology: name: pathology display_name: Pathology @@ -911,6 +931,16 @@ tracked_point__channels: Label of the point that is being tracked, for example, label of a tracker or a marker (for example,`"LeftFoot"`, `"RightWrist"`). type: string +# TODO: Add relative links to `trial_type` column and events.tsv description. +trial_id: + name: trial_id + display_name: Trial ID + description: | + If the data entity belongs to set of data_entities forming a trial, + they have to be linked to the same trial_id. This can be used to e.g. group stimulation and response records. + The optional `trial_type` events.tsv column can be used to provide the further information on the trial like + `electric stimulation`,`rest`, or `pharmaceutical stimulation`, etc. + type: string trial_type: name: trial_type display_name: Trial type From 8a0df20877978f4554d5aa1a6bdc49d24adf805d Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Tue, 18 Jun 2024 19:30:29 +0200 Subject: [PATCH 17/33] [ENH] Added tabular data rules for events.tsv to include ephys related columns --- src/schema/rules/tabular_data/task.yaml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/schema/rules/tabular_data/task.yaml b/src/schema/rules/tabular_data/task.yaml index 0ca01afe25..405a6d674b 100644 --- a/src/schema/rules/tabular_data/task.yaml +++ b/src/schema/rules/tabular_data/task.yaml @@ -15,6 +15,27 @@ TaskEvents: description_addendum: | Note that this column only applies to data types where channels are specified, such as EEG, iEEG, MEG or NIRS. +# TODO: Make the addendums more clear and descriptive + data_entity_id: + level: optional + description_addendum: | + Note that this column only applies to + microelectrode eleectrophysiology data type. + duration_index: + level: optional + description_addendum: | + Note that this column only applies to + microelectrode eleectrophysiology data type. + onset_index: + level: optional + description_addendum: | + Note that this column only applies to + microelectrode eleectrophysiology data type. + trial_id: + level: optional + description_addendum: | + Note that this column only applies to + microelectrode eleectrophysiology data type. additional_columns: allowed initial_columns: - onset From 8caaaf204a54dd226c3a0d88a2c7df4f3e2ca44d Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Tue, 18 Jun 2024 20:48:49 +0200 Subject: [PATCH 18/33] [ENH] Added metadata entities for ephys setup metadata --- src/schema/objects/metadata.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/schema/objects/metadata.yaml b/src/schema/objects/metadata.yaml index 57515dd6f0..cbd41232c1 100644 --- a/src/schema/objects/metadata.yaml +++ b/src/schema/objects/metadata.yaml @@ -1970,6 +1970,12 @@ ManufacturersModelName: description: | Manufacturer's model name of the equipment that produced the measurements. type: string +ManufacturersModelVersion: + name: ManufacturersModelVersion + display_name: Manufacturers Model Version + description: | + Manufacturer's model version of the equipment that produced the measurements. + type: string MatrixCoilMode: name: MatrixCoilMode display_name: Matrix Coil Mode @@ -2657,6 +2663,12 @@ RecordingDuration: Length of the recording in seconds (for example, `3600`). type: number unit: s +RecordingSetupName: + name: RecordingSetupName + display_name: Recording Setup Name + description: | + Custom name of the recording setup. + type: string RecordingType: name: RecordingType display_name: Recording Type From 0b75398bd477d355323553dca22234f662961cd4 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Tue, 18 Jun 2024 20:52:01 +0200 Subject: [PATCH 19/33] [ENH] Created sidecar rule file for ephys metadata and added data origin and setup fields --- src/schema/rules/sidecars/ephys.yaml | 47 ++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 src/schema/rules/sidecars/ephys.yaml diff --git a/src/schema/rules/sidecars/ephys.yaml b/src/schema/rules/sidecars/ephys.yaml new file mode 100644 index 0000000000..714a7ef44f --- /dev/null +++ b/src/schema/rules/sidecars/ephys.yaml @@ -0,0 +1,47 @@ +# +# Groups of related metadata fields +# +# Assumptions: never need disjunction of selectors +# Assumptions: top-to-bottom overrides is sufficient logic + +--- +# Groups are defined as they appear in the proposal + +ephysInstitutionInformation: + selectors: + - datatype == "ephys" + - suffix == "ephys" + fields: + InstitutionName: recommended + InstitutionAddress: recommended + InstitutionalDepartmentName: recommended + +ephysSetup: + selectors: + - datatype == "ephys" + - suffix == "ephys" + fields: + PowerLineFrequency: required + Manufacturer: + level: recommended + description_addendum: For example, `"TDT"`, `"Blackrock"`. + ManufacturersModelName: recommended + ManufacturersModelVersion: recommended + RecordingSetupName: recommended + SamplingFrequency: + level: required + description_addendum: | + Internal (maximum) sampling frequency (in Hz) + of the recording (for example, "24000"). + DeviceSerialNumber: + level: recommended + description_addendum: | + The serial number of the components of the setup, + recommended to add serial numbers and versions of + ALL components constituting the setup. + SoftwareName: + level: recommended + description_addendum: | + The name of the software suite used to + record the data. + SoftwareVersions: recommended From 3626004accecb01f7953469706c37dbafc56ff2b Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Wed, 17 Jul 2024 13:49:03 +0200 Subject: [PATCH 20/33] Made schema changes to match microephys data type --- src/schema/objects/columns.yaml | 10 +++++----- src/schema/rules/sidecars/ephys.yaml | 10 +++++----- src/schema/rules/tabular_data/ephys.yaml | 16 ++++++++-------- 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml index ca6d36a695..44a8ef645b 100644 --- a/src/schema/objects/columns.yaml +++ b/src/schema/objects/columns.yaml @@ -313,7 +313,7 @@ ground: display_name: Ground description: | Information on the ground. For example, 'chamber screw', 'head post', 'ear clip'. - Only should be used to optionally override the global ground in the `_ephys.json` file. + Only should be used to optionally override the global ground in the `_ecephys.json` or `_icephys.json` file. type: string group__channel: name: group @@ -361,7 +361,7 @@ hardware_filters: display_name: Hardware filters description: | List of hardware (amplifier) filter keys applied. - Note that parameters should be defined in the general ephys sidecar .json file. + Note that parameters should be defined in the general microephys sidecar .json file. Indicate `n/a` in the absence of hardware filters applied. anyOf: - type: string @@ -633,7 +633,7 @@ reference__eeg: This column is not needed when it is common to all channels. In that case the reference electrode(s) can be specified in `*_eeg.json` as `EEGReference`). type: string -reference__ephys: +reference__microephys: name: reference display_name: Reference description: | @@ -802,7 +802,7 @@ software_filters__channels: description: | List of temporal and/or spatial software filters applied (for example, `SSS`, `SpatialCompensation`). - Note that parameters should be defined in the general ephys sidecar .json file. + Note that parameters should be defined in the general microephys sidecar .json file. Indicate `n/a` in the absence of software filters applied. anyOf: - type: string @@ -957,7 +957,7 @@ trigger: description: | continuous measurement of the scanner trigger signal type: number -# TODO: Populate enums that make sense for channels in the ephys datasets. +# TODO: Populate enums that make sense for channels in the microephys datasets. # type column in channels.tsv files type__channels: name: type diff --git a/src/schema/rules/sidecars/ephys.yaml b/src/schema/rules/sidecars/ephys.yaml index 714a7ef44f..0f9bd99128 100644 --- a/src/schema/rules/sidecars/ephys.yaml +++ b/src/schema/rules/sidecars/ephys.yaml @@ -7,10 +7,10 @@ --- # Groups are defined as they appear in the proposal -ephysInstitutionInformation: +microephysInstitutionInformation: selectors: - - datatype == "ephys" - - suffix == "ephys" + - datatype == ["ecephys", "icephys"] + - suffix == ["ecephys", "icephys"] fields: InstitutionName: recommended InstitutionAddress: recommended @@ -18,8 +18,8 @@ ephysInstitutionInformation: ephysSetup: selectors: - - datatype == "ephys" - - suffix == "ephys" + - datatype == ["ecephys", "icephys"] + - suffix == ["ecephys", "icephys"] fields: PowerLineFrequency: required Manufacturer: diff --git a/src/schema/rules/tabular_data/ephys.yaml b/src/schema/rules/tabular_data/ephys.yaml index a75d1c009f..162acf0696 100644 --- a/src/schema/rules/tabular_data/ephys.yaml +++ b/src/schema/rules/tabular_data/ephys.yaml @@ -1,6 +1,6 @@ -ephysProbes: +microephysProbes: selectors: - - datatype == "ephys" + - datatype == ["ecephys", "icephys"] - suffix == "probes" - extension == ".tsv" initial_columns: @@ -35,9 +35,9 @@ ephysProbes: index_columns: [probe_id] additional_columns: allowed_if_defined -ephysElectrodes: +microephysElectrodes: selectors: - - datatype == "ephys" + - datatype == ["ecephys", "icephys"] - suffix == "electrodes" - extension == ".tsv" initial_columns: @@ -66,20 +66,20 @@ ephysElectrodes: index_columns: [electrode_id] additional_columns: allowed_if_defined -ephysChannels: +microephysChannels: selectors: - - datatype == "ephys" + - datatype == ["ecephys", "icephys"] - suffix == "channels" - extension == ".tsv" initial_columns: - channel_id - - reference__ephys + - reference__microephys - type__channels - units__channels - sampling_frequency columns: - channel_id: required - - reference__ephys: required + - reference__microephys: required - type__channels: required - units__channels: required - sampling_frequency: optional From 2ff94e0524fe53ad57ceb30026e0db00339c7d0c Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Wed, 17 Jul 2024 13:50:47 +0200 Subject: [PATCH 21/33] Renamed rule files from ephys to microephys --- src/schema/rules/sidecars/{ephys.yaml => microephys.yaml} | 0 src/schema/rules/tabular_data/{ephys.yaml => microephys.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename src/schema/rules/sidecars/{ephys.yaml => microephys.yaml} (100%) rename src/schema/rules/tabular_data/{ephys.yaml => microephys.yaml} (100%) diff --git a/src/schema/rules/sidecars/ephys.yaml b/src/schema/rules/sidecars/microephys.yaml similarity index 100% rename from src/schema/rules/sidecars/ephys.yaml rename to src/schema/rules/sidecars/microephys.yaml diff --git a/src/schema/rules/tabular_data/ephys.yaml b/src/schema/rules/tabular_data/microephys.yaml similarity index 100% rename from src/schema/rules/tabular_data/ephys.yaml rename to src/schema/rules/tabular_data/microephys.yaml From 1a1d3c64f90406404aea1de8ac3930ccf83a2671 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Wed, 17 Jul 2024 14:34:39 +0200 Subject: [PATCH 22/33] Made schema changes to match microephys data type --- src/schema/rules/sidecars/microephys.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schema/rules/sidecars/microephys.yaml b/src/schema/rules/sidecars/microephys.yaml index 0f9bd99128..8883a84525 100644 --- a/src/schema/rules/sidecars/microephys.yaml +++ b/src/schema/rules/sidecars/microephys.yaml @@ -16,7 +16,7 @@ microephysInstitutionInformation: InstitutionAddress: recommended InstitutionalDepartmentName: recommended -ephysSetup: +microephysSetup: selectors: - datatype == ["ecephys", "icephys"] - suffix == ["ecephys", "icephys"] From 30974f8ed047989f92fbe6d092974669aa72a351 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Wed, 17 Jul 2024 15:07:33 +0200 Subject: [PATCH 23/33] [ENH] Added objects and rules for coordinate system sidecar file --- src/schema/objects/metadata.yaml | 48 +++++++++++++++++++++++ src/schema/rules/sidecars/microephys.yaml | 12 ++++++ 2 files changed, 60 insertions(+) diff --git a/src/schema/objects/metadata.yaml b/src/schema/objects/metadata.yaml index cbd41232c1..2a6a5adcff 100644 --- a/src/schema/objects/metadata.yaml +++ b/src/schema/objects/metadata.yaml @@ -1853,6 +1853,54 @@ MEGREFChannelCount: `MEGREFChannelCount` should be set to `0`. type: integer minimum: 0 +# TODO: Change if needs to be broken down into ecephys and icephys +# TODO: Add links to the 2D coordinate system, Pixels and glossary entry. +# TODO: Add enums for the valid values specific to microephys +MicroephysCoordinateSystem: + name: MicroephysCoordinateSystem + display_name: Microephys Coordinate System + description: | + Defines the coordinate system for the microelectrode probes. + See the + [Coordinate Systems Appendix](SPEC_ROOT/appendices/coordinate-systems.md) + for a list of restricted keywords for coordinate systems. + If `"Other"`, provide definition of the coordinate system in + `"MicroephysCoordinateSystemDescription"`. If positions correspond to + pixel indices in a 2D image (of either a volume-rendering, + surface-rendering, operative photo, or operative drawing), + this MUST be "Pixels". For more information, see the section + on 2D coordinate systems. + For a list of valid values for this field, see the associated glossary entry. + anyOf: + # - $ref: objects.enums._MicroephysCoordSys + - $ref: objects.enums._StandardTemplateCoordSys +MicroephysCoordinateSystemDescription: + name: MicroephysCoordinateSystemDescription + display_name: Microephys Coordinate System Description + description: | + Free-form text description of the coordinate system. + May also include a link to a documentation page or paper describing the + system in greater detail. + type: string +MicroephysCoordinateUnits: + name: MicroephysCoordinateUnits + display_name: Microephys Coordinate Units + description: | + Units of the coordinates of `"MicroephysCoordinateSystem"`. + type: string +# TODO: Clarify if units need to be used or enums, if units, define pixels + enum: + - m + - mm + - cm + - pixels +MicroephysCoordinateSystemPhoto: + name: MicroephysCoordinateSystemPhoto + display_name: Microephys Coordinate System Photo + description: | + A link to a photo or drawing of the microelectrode probe system. + type: string + format: uri MISCChannelCount: name: MISCChannelCount display_name: Miscellaneous channel count diff --git a/src/schema/rules/sidecars/microephys.yaml b/src/schema/rules/sidecars/microephys.yaml index 8883a84525..e946ceaf6a 100644 --- a/src/schema/rules/sidecars/microephys.yaml +++ b/src/schema/rules/sidecars/microephys.yaml @@ -45,3 +45,15 @@ microephysSetup: The name of the software suite used to record the data. SoftwareVersions: recommended + +# TODO: Check how to conditionally change level to required for photo if +# MicroephysCoordinateSystem is Pixels +microephysCoordsystemGeneral: + selectors: + - datatype == ["ecephys", "icephys"] + - suffix == "coordsystem" + fields: + MicroephysCoordinateSystem: required + MicroephysCoordinateSystemDescription: recommended + MicroephysCoordinateUnits: required + MicroephysCoordinateSystemPhoto: optional From f524dfeaec3ad3b768204af381e6a3c3f6bae5dc Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Wed, 17 Jul 2024 15:23:45 +0200 Subject: [PATCH 24/33] [ENH] Added processing microephys metadata field in rules --- src/schema/rules/sidecars/microephys.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/schema/rules/sidecars/microephys.yaml b/src/schema/rules/sidecars/microephys.yaml index e946ceaf6a..4542fb59e5 100644 --- a/src/schema/rules/sidecars/microephys.yaml +++ b/src/schema/rules/sidecars/microephys.yaml @@ -46,6 +46,14 @@ microephysSetup: record the data. SoftwareVersions: recommended +microephysProcessing: + selectors: + - datatype == ["ecephys", "icephys"] + - suffix == ["ecephys", "icephys"] + fields: + SoftwareFilters: required + HardwareFilters: recommended + # TODO: Check how to conditionally change level to required for photo if # MicroephysCoordinateSystem is Pixels microephysCoordsystemGeneral: From 686b5599b3fb9b89602758e7c1f4a04bb4c1e12f Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Wed, 17 Jul 2024 15:32:30 +0200 Subject: [PATCH 25/33] [ENH] Added pharmaceuticals microephys metadata field in rules --- src/schema/rules/sidecars/microephys.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/schema/rules/sidecars/microephys.yaml b/src/schema/rules/sidecars/microephys.yaml index 4542fb59e5..422bf78107 100644 --- a/src/schema/rules/sidecars/microephys.yaml +++ b/src/schema/rules/sidecars/microephys.yaml @@ -54,6 +54,17 @@ microephysProcessing: SoftwareFilters: required HardwareFilters: recommended +microephysPharmaceuticals: + selectors: + - datatype == ["ecephys", "icephys"] + - suffix == ["ecephys", "icephys"] + fields: + PharmaceuticalName: recommended + PharmaceuticalDoseAmount: recommended + PharmaceuticalDoseUnits: recommended + PharmaceuticalDoseRegimen: recommended + PharmaceuticalDoseTime: recommended + # TODO: Check how to conditionally change level to required for photo if # MicroephysCoordinateSystem is Pixels microephysCoordsystemGeneral: From c9e2eb33cfaf15a06674ed072b67d6fb4523862d Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Wed, 17 Jul 2024 15:41:24 +0200 Subject: [PATCH 26/33] [ENH] Added supplementary microephys metadata field and rules --- src/schema/objects/metadata.yaml | 7 +++++++ src/schema/rules/sidecars/microephys.yaml | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/src/schema/objects/metadata.yaml b/src/schema/objects/metadata.yaml index 2a6a5adcff..38c944c79d 100644 --- a/src/schema/objects/metadata.yaml +++ b/src/schema/objects/metadata.yaml @@ -3452,6 +3452,13 @@ SubjectArtefactDescription: If this field is set to `"n/a"`, it will be interpreted as absence of major source of artifacts except cardiac and blinks. type: string +SupplementarySignals: + name: SupplementarySignals + display_name: Supplementary Signals + description: | + Description of the supplementary signal (additional modalities) recorded + in parallel and are also stored in the data file. + type: string TaskDescription: name: TaskDescription display_name: Task Description diff --git a/src/schema/rules/sidecars/microephys.yaml b/src/schema/rules/sidecars/microephys.yaml index 422bf78107..173dafbe1f 100644 --- a/src/schema/rules/sidecars/microephys.yaml +++ b/src/schema/rules/sidecars/microephys.yaml @@ -76,3 +76,10 @@ microephysCoordsystemGeneral: MicroephysCoordinateSystemDescription: recommended MicroephysCoordinateUnits: required MicroephysCoordinateSystemPhoto: optional + +microephysSupplementary: + selectors: + - datatype == ["ecephys", "icephys"] + - suffix == ["ecephys", "icephys"] + fields: + SupplementarySignals: optional From c2a251a32523569714ef4fc6993dfd2693a452d2 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Wed, 17 Jul 2024 15:44:29 +0200 Subject: [PATCH 27/33] [ENH] Added sample microephys metadata rules --- src/schema/rules/sidecars/microephys.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/schema/rules/sidecars/microephys.yaml b/src/schema/rules/sidecars/microephys.yaml index 173dafbe1f..da5c5f98b3 100644 --- a/src/schema/rules/sidecars/microephys.yaml +++ b/src/schema/rules/sidecars/microephys.yaml @@ -83,3 +83,16 @@ microephysSupplementary: - suffix == ["ecephys", "icephys"] fields: SupplementarySignals: optional + +microephysSample: + selectors: + - datatype == ["ecephys", "icephys"] + - suffix == ["ecephys", "icephys"] + fields: + BodyPart: recommended + BodyPartDetails: recommended + BodyPartDetailsOntology: optional + SampleEnvironment: recommended + SampleEmbedding: optional + SampleThickness: optional + SampleExtractionProtocol: optional From c130ad517baa95536f7a5b86475094eb46025afa Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Wed, 17 Jul 2024 15:49:24 +0200 Subject: [PATCH 28/33] [ENH] Added task microephys metadata rules --- src/schema/rules/sidecars/microephys.yaml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/schema/rules/sidecars/microephys.yaml b/src/schema/rules/sidecars/microephys.yaml index da5c5f98b3..b57a25d0aa 100644 --- a/src/schema/rules/sidecars/microephys.yaml +++ b/src/schema/rules/sidecars/microephys.yaml @@ -96,3 +96,22 @@ microephysSample: SampleEmbedding: optional SampleThickness: optional SampleExtractionProtocol: optional + +microephysTaskInformation: + selectors: + - datatype == ["ecephys", "icephys"] + - suffix == ["ecephys", "icephys"] + fields: + TaskName: + level: recommended + description_addendum: | + A recommended convention is to name resting state task using labels + beginning with `rest`. + TaskDescription: recommended + Instructions: + level: recommended + description_addendum: | + This is especially important in context of resting state recordings and + distinguishing between eyes open and eyes closed paradigms. + CogAtlasID: recommended + CogPOID: recommended From 8c658b85670e323e5b3884175cb9d73a21616739 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Wed, 18 Sep 2024 14:45:24 +0200 Subject: [PATCH 29/33] Fixed example language convention --- src/schema/objects/columns.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/schema/objects/columns.yaml b/src/schema/objects/columns.yaml index 44a8ef645b..c97719519e 100644 --- a/src/schema/objects/columns.yaml +++ b/src/schema/objects/columns.yaml @@ -98,7 +98,7 @@ channel_id: display_name: Channel ID description: | Unique identifier of the channel, only containing letters and numbers. - This id must be reflected in the data files, e.g. as signal annotations in the nix file. + This id must be reflected in the data files, for example, as signal annotations in the nix file. When inserting existing nwb files into a BIDS structure, one can combine the nwb record_id and channel_index to create this unique identifier. type: string @@ -108,8 +108,8 @@ channel_label: display_name: Channel label description: | Human readable identifier. Use this name to specify the - content of signals not generated by electrodes, - e.g. DAQ internal synchronization signals, behavioral signals, behavioral cues, etc. + content of signals not generated by electrodes. For example, + 'DAQ internal synchronization signals', 'behavioral signals', 'behavioral cues', etc. type: string color: name: color @@ -154,13 +154,13 @@ data_entity_id: description: | The corresponding (UUID) identifier of the recording trace in the data file. The same `data_entity_id` can be listed multiple times with different - `onset` & `duration` values to select a subset of a recording trace. + `onset` and `duration` values to select a subset of a recording trace. type: string depth__probes: name: depth display_name: Depth description: | - Physical depth of the probe, e.g. 0.3. + Physical depth of the probe, for example, '0.3'. This dimension should be omitted or set to 0 for two-dimensional (shank-type) probes. This dimension corresponds to the z’axis of the Euler transformation defined by alpha, beta and gamma rotations values. @@ -270,7 +270,7 @@ electrode_count: name: electrode_count display_name: Electrode count description: | - Number of miscellaneous analog electrodes for auxiliary signals (e.g. 2). + Number of miscellaneous analog electrodes for auxiliary signals (for example, '2'). type: number electrode_id: name: electrode_id @@ -372,7 +372,7 @@ height__probes: name: height display_name: Height description: | - Physical height of the probe, e.g. 5. + Physical height of the probe, for example, '5'. This dimension corresponds to the y’axis of the Euler transformation defined by alpha, beta and gamma rotations values below. type: number @@ -460,7 +460,7 @@ manufacturer__probes: name: manufacturer display_name: Manufacturer description: | - Manufacturer of the probes system (e.g. 'openephys', 'alphaomega','blackrock'). + Manufacturer of the probes system (for example, 'openephys', 'alphaomega','blackrock'). type: string mapping: name: mapping @@ -1133,7 +1133,7 @@ width__probes: name: width display_name: Width description: | - Physical width of the probe, e.g. 5. + Physical width of the probe, for example, '5'. This dimension corresponds to the x’axis of the Euler transformation defined by alpha, beta and gamma rotations values. type: number From fe441bccf847f20e3a464524d0c661351594ca88 Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Wed, 18 Sep 2024 14:51:48 +0200 Subject: [PATCH 30/33] Added microephys in schema rule modality --- src/schema/rules/modalities.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/schema/rules/modalities.yaml b/src/schema/rules/modalities.yaml index 2f290ca014..e4bdf56974 100644 --- a/src/schema/rules/modalities.yaml +++ b/src/schema/rules/modalities.yaml @@ -31,3 +31,7 @@ motion: nirs: datatypes: - nirs +microephys: + datatypes: + - ecephys + - icephys From f82553122970351820f777770c35aa4b1ba9735f Mon Sep 17 00:00:00 2001 From: Reema Gupta Date: Wed, 18 Sep 2024 15:15:56 +0200 Subject: [PATCH 31/33] Corrected SampleThickness to SliceThickness --- src/schema/rules/sidecars/microephys.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schema/rules/sidecars/microephys.yaml b/src/schema/rules/sidecars/microephys.yaml index b57a25d0aa..9ba92c2665 100644 --- a/src/schema/rules/sidecars/microephys.yaml +++ b/src/schema/rules/sidecars/microephys.yaml @@ -94,7 +94,7 @@ microephysSample: BodyPartDetailsOntology: optional SampleEnvironment: recommended SampleEmbedding: optional - SampleThickness: optional + SliceThickness: optional SampleExtractionProtocol: optional microephysTaskInformation: From a16c0b03dd4c4a74502e5483d3f755e349b61397 Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Wed, 16 Oct 2024 12:53:57 -0400 Subject: [PATCH 32/33] Add yaml document separator for microephys rules file It is optional, and adding does not solve anything but makes it consistent with the other files in the folder --- src/schema/rules/tabular_data/microephys.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/src/schema/rules/tabular_data/microephys.yaml b/src/schema/rules/tabular_data/microephys.yaml index 162acf0696..d73762fde3 100644 --- a/src/schema/rules/tabular_data/microephys.yaml +++ b/src/schema/rules/tabular_data/microephys.yaml @@ -1,3 +1,4 @@ +--- microephysProbes: selectors: - datatype == ["ecephys", "icephys"] From 9bf1cc99ef8dea4b7cf1d9a57f39c40e0adcf34f Mon Sep 17 00:00:00 2001 From: Yaroslav Halchenko Date: Thu, 17 Oct 2024 20:26:56 -0400 Subject: [PATCH 33/33] BF: should be a dict, not a list of dicts --- src/schema/rules/tabular_data/microephys.yaml | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/src/schema/rules/tabular_data/microephys.yaml b/src/schema/rules/tabular_data/microephys.yaml index d73762fde3..a16655257e 100644 --- a/src/schema/rules/tabular_data/microephys.yaml +++ b/src/schema/rules/tabular_data/microephys.yaml @@ -67,7 +67,7 @@ microephysElectrodes: index_columns: [electrode_id] additional_columns: allowed_if_defined -microephysChannels: +microephysCCChannelsss: selectors: - datatype == ["ecephys", "icephys"] - suffix == "channels" @@ -79,22 +79,22 @@ microephysChannels: - units__channels - sampling_frequency columns: - - channel_id: required - - reference__microephys: required - - type__channels: required - - units__channels: required - - sampling_frequency: optional - - channel_label: optional - - stream_id: optional - - description: optional - - hardware_filters: recommended - - software_filters: recommended - - status: optional - - status_description: optional - - gain: recommended - - time_offset: optional - - time_reference_channel: optional - - ground: optional - - recording_mode: recommended + channel_id: required + reference__microephys: required + type__channels: required + units__channels: required + sampling_frequency: optional + channel_label: optional + stream_id: optional + description: optional + hardware_filters: recommended + software_filters: recommended + status: optional + status_description: optional + gain: recommended + time_offset: optional + time_reference_channel: optional + ground: optional + recording_mode: recommended index_columns: [channel_id] additional_columns: allowed_if_defined