From 3b64a2b372f6a55eb7a1e2da24a2d321045abfd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 18 Mar 2025 11:06:24 +0100 Subject: [PATCH 01/17] Adding dir at dataset level --- src/schema/rules/directories.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/schema/rules/directories.yaml b/src/schema/rules/directories.yaml index db3db194fe..2d1ba9125c 100644 --- a/src/schema/rules/directories.yaml +++ b/src/schema/rules/directories.yaml @@ -22,6 +22,7 @@ raw: - code - derivatives - phenotype + - prov - sourcedata - stimuli - subject @@ -37,6 +38,10 @@ raw: name: phenotype level: optional opaque: false + prov: + name: prov + level: optional + opaque: false sourcedata: name: sourcedata level: optional @@ -70,6 +75,7 @@ derivative: - code - derivatives - phenotype + - prov - sourcedata - stimuli - subject @@ -85,6 +91,10 @@ derivative: name: phenotype level: optional opaque: false + prov: + name: prov + level: optional + opaque: false sourcedata: name: sourcedata level: optional From 45f6af471ce4ad3b54ff211448552d83439f8a69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 18 Mar 2025 11:09:00 +0100 Subject: [PATCH 02/17] Adding dir at any level --- src/schema/rules/directories.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/schema/rules/directories.yaml b/src/schema/rules/directories.yaml index 2d1ba9125c..29e4295482 100644 --- a/src/schema/rules/directories.yaml +++ b/src/schema/rules/directories.yaml @@ -58,16 +58,20 @@ raw: - oneOf: - session - datatype + - prov session: entity: session level: optional opaque: false subdirs: - datatype + - prov datatype: value: datatype level: required opaque: false + subdirs: + - prov derivative: root: @@ -110,13 +114,17 @@ derivative: subdirs: - session - datatype + - prov session: entity: session level: optional opaque: false subdirs: - datatype + - prov datatype: value: datatype level: optional opaque: false + subdirs: + - prov From 4c315ce384f275c2da75e517490dbf92aefea5b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 18 Mar 2025 11:10:08 +0100 Subject: [PATCH 03/17] Adding rule for dir at dataset level --- src/schema/rules/files/common/core.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/schema/rules/files/common/core.yaml b/src/schema/rules/files/common/core.yaml index 9f54db072f..4a5324d52b 100644 --- a/src/schema/rules/files/common/core.yaml +++ b/src/schema/rules/files/common/core.yaml @@ -36,6 +36,9 @@ code: derivatives: level: optional path: derivatives +prov: + level: optional + path: prov sourcedata: level: optional path: sourcedata From b490c91ad7853c3126283d6ca1a91c80559fb4ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 18 Mar 2025 11:18:43 +0100 Subject: [PATCH 04/17] Adding objects: JSONLD extension + top level dir --- src/schema/objects/extensions.yaml | 9 +++++++++ src/schema/objects/files.yaml | 7 +++++++ 2 files changed, 16 insertions(+) diff --git a/src/schema/objects/extensions.yaml b/src/schema/objects/extensions.yaml index 1984661e87..268ecdc173 100644 --- a/src/schema/objects/extensions.yaml +++ b/src/schema/objects/extensions.yaml @@ -130,6 +130,15 @@ json: These sidecar files follow the inheritance principle. There are also a few special cases of JSON files being first-order data files, such as `genetic_info.json`. +jsonld: + value: .jsonld + display_name: JavaScript Object Notation for Linked Data + description: | + A JSON-LD file. + + JSON-LD files are used to describe relations between objects in the dataset. + + For example, the description of provenance uses this file format. kdf: value: .kdf display_name: KRISS KDF diff --git a/src/schema/objects/files.yaml b/src/schema/objects/files.yaml index d2d8761410..cee42d520e 100644 --- a/src/schema/objects/files.yaml +++ b/src/schema/objects/files.yaml @@ -102,6 +102,13 @@ code: (for example the one used to generate the derivatives from the raw data). See the [Code section](SPEC_ROOT/modality-agnostic-files.md#code) for more information. +prov: + display_name: Provenance Records + file_type: directory + description: | + A directory in which to store provenance metadata. + See the [Provenance section](SPEC_ROOT/modality-agnostic-files.md#provenance) + for more information. derivatives: display_name: Derivative data file_type: directory From f2a67a67bf87a786f03c46f7394d2e82c2e78029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 18 Mar 2025 11:21:18 +0100 Subject: [PATCH 05/17] Adding object : prov entity --- src/schema/objects/entities.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/schema/objects/entities.yaml b/src/schema/objects/entities.yaml index a5638bc984..a94f5c8e1d 100644 --- a/src/schema/objects/entities.yaml +++ b/src/schema/objects/entities.yaml @@ -229,6 +229,14 @@ processing: exploited. type: string format: label +prov: + name: prov + display_name: Provenance group + description: | + A grouping of provenance records. + Defining multiple provenance records groups is appropriate when several processings have been performed on data. + type: string + format: label reconstruction: name: rec display_name: Reconstruction From d9bb7b04b9ab39fe93cfdb168b848b2cf3a45c1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 18 Mar 2025 11:57:42 +0100 Subject: [PATCH 06/17] Add suffixes in object.suffixes --- src/schema/objects/suffixes.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/schema/objects/suffixes.yaml b/src/schema/objects/suffixes.yaml index 1e2825ced2..cb4613e33a 100644 --- a/src/schema/objects/suffixes.yaml +++ b/src/schema/objects/suffixes.yaml @@ -502,6 +502,16 @@ VFA: Depending on the provided metadata fields and the sequence type, data may be eligible for DESPOT1, DESPOT2 and their variants ([Deoni et al. 2005](https://doi.org/10.1002/mrm.20314)). +act: + value: act + display_name: Provenance Activity Records + description: | + Provenance Activity records for a group of provenance. Activities represent the transformations that have been applied to the data. +all: + value: all + display_name: Provenance Records + description: | + All types of provenance records (Activity, Entity, Environment, Software) for a group of provenance. angio: value: angio display_name: Angiogram @@ -528,6 +538,11 @@ asllabeling: A deidentified screenshot of the planning of the labeling slab/plane with respect to the imaging slab or slices. This screenshot is based on DICOM macro C.8.13.5.14. +base: + value: base + display_name: Base for Provenance Records + description: | + Base fields for provenance records of a group of provenance. beh: value: beh display_name: Behavioral recording @@ -612,6 +627,16 @@ electrodes: display_name: Electrodes description: | File that gives the location of (i)EEG electrodes. +ent: + value: ent + display_name: Provenance Entity Records + description: | + Provenance Entity records for a group of provenance. Entities are inputs and outputs of Activities. +env: + value: env + display_name: Provenance Environment Records + description: | + Provenance Environment records for a group of provenance. Environments specify the software environment in which the provenance record was obtained. epi: value: epi display_name: EPI @@ -817,6 +842,11 @@ probseg: A probabilistic segmentation. This suffix may only be used in derivative datasets. +soft: + value: soft + display_name: Provenance Software Records + description: | + Provenance Software records for a group of provenance. The Software specifies the software package with which transformations were applied to the data. sbref: value: sbref display_name: Single-band reference image From 64e2c6593d7ed73aaa7fed6c5239e16f764848b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 18 Mar 2025 13:59:39 +0100 Subject: [PATCH 07/17] GeneratedByProv and SidecarGeneratedBy metadata --- src/schema/objects/metadata.yaml | 12 ++++++++++++ src/schema/rules/dataset_metadata.yaml | 2 ++ 2 files changed, 14 insertions(+) diff --git a/src/schema/objects/metadata.yaml b/src/schema/objects/metadata.yaml index 09984125ad..fd8c2a8b10 100644 --- a/src/schema/objects/metadata.yaml +++ b/src/schema/objects/metadata.yaml @@ -1281,6 +1281,12 @@ GeneratedBy: URI: type: string format: uri +GeneratedByProv: + name: GeneratedByProv + display_name: Generated By Prov + description: | + Specify the provenance of a file in the dataset. + type: object GeneticLevel: name: GeneticLevel display_name: Genetic Level @@ -3292,6 +3298,12 @@ ShortChannelCount: The number of short channels. 0 indicates no short channels. type: integer minimum: 0 +SidecarGeneratedBy: + name: SidecarGeneratedBy + display_name: Sidecar Generated By + description: | + Specify the provenance of a sidecar JSON. + type: object SinglesRate: name: SinglesRate display_name: Singles Rate diff --git a/src/schema/rules/dataset_metadata.yaml b/src/schema/rules/dataset_metadata.yaml index d4ecca7da1..504baf6498 100644 --- a/src/schema/rules/dataset_metadata.yaml +++ b/src/schema/rules/dataset_metadata.yaml @@ -17,6 +17,7 @@ dataset_description: ReferencesAndLinks: optional DatasetDOI: optional GeneratedBy: recommended + GeneratedByProv: recommended SourceDatasets: recommended dataset_authors: @@ -38,6 +39,7 @@ derivative_description: - path == "/dataset_description.json" - json.DatasetType == "derivative" fields: + GeneratedByProv: recommended GeneratedBy: required dataset_description_with_genetics: From acc6af0f8c72ab857a965ad7d244506151eb6872 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Wed, 19 Mar 2025 10:28:50 +0100 Subject: [PATCH 08/17] prov entitiy and provenance json/jsonld files --- src/schema/rules/entities.yaml | 1 + src/schema/rules/files/common/tables.yaml | 31 +++++++++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/src/schema/rules/entities.yaml b/src/schema/rules/entities.yaml index 44174d333e..e1d118b754 100644 --- a/src/schema/rules/entities.yaml +++ b/src/schema/rules/entities.yaml @@ -32,3 +32,4 @@ - density - label - description +- prov diff --git a/src/schema/rules/files/common/tables.yaml b/src/schema/rules/files/common/tables.yaml index 21f03f6579..31888b198a 100644 --- a/src/schema/rules/files/common/tables.yaml +++ b/src/schema/rules/files/common/tables.yaml @@ -5,12 +5,14 @@ participants: extensions: - .tsv - .json + samples: level: optional stem: samples extensions: - .tsv - .json + scans: level: optional suffixes: @@ -21,6 +23,7 @@ scans: entities: subject: required session: optional # session is required if session is present in the dataset. + sessions: # This file may only exist if session is present in the dataset. level: optional suffixes: @@ -43,3 +46,31 @@ phenotype: extensions: - .tsv - .json + +# Provenance json files +provenance: + level: optional + suffixes: + - act + - base + - ent + - env + - soft + extensions: + - .json + entities: + subject: optional + session: optional + prov: required + +# Provenance ld files +provenanceld: + level: optional + suffixes: + - all + extensions: + - .jsonld + entities: + subject: optional + session: optional + prov: required From 34fafe1633f4b569bb68e308aaf5f6ce730f8be8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Wed, 19 Mar 2025 10:40:20 +0100 Subject: [PATCH 09/17] provenance json/jsonld files inside new rules file --- .../rules/files/common/modality_agnostic.yaml | 30 +++++++++++++++++++ src/schema/rules/files/common/tables.yaml | 28 ----------------- 2 files changed, 30 insertions(+), 28 deletions(-) create mode 100644 src/schema/rules/files/common/modality_agnostic.yaml diff --git a/src/schema/rules/files/common/modality_agnostic.yaml b/src/schema/rules/files/common/modality_agnostic.yaml new file mode 100644 index 0000000000..2343f4d443 --- /dev/null +++ b/src/schema/rules/files/common/modality_agnostic.yaml @@ -0,0 +1,30 @@ +--- +# Files and directories that can be found at any level of a dataset. + +# Provenance json files +provenance: + level: optional + suffixes: + - act + - base + - ent + - env + - soft + extensions: + - .json + entities: + subject: optional + session: optional + prov: required + +# Provenance ld files +provenanceld: + level: optional + suffixes: + - all + extensions: + - .jsonld + entities: + subject: optional + session: optional + prov: required diff --git a/src/schema/rules/files/common/tables.yaml b/src/schema/rules/files/common/tables.yaml index 31888b198a..7c818acc66 100644 --- a/src/schema/rules/files/common/tables.yaml +++ b/src/schema/rules/files/common/tables.yaml @@ -46,31 +46,3 @@ phenotype: extensions: - .tsv - .json - -# Provenance json files -provenance: - level: optional - suffixes: - - act - - base - - ent - - env - - soft - extensions: - - .json - entities: - subject: optional - session: optional - prov: required - -# Provenance ld files -provenanceld: - level: optional - suffixes: - - all - extensions: - - .jsonld - entities: - subject: optional - session: optional - prov: required From a1fbb8a29ff4d50b0dd3b2b0aaf3442e557843f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Thu, 10 Apr 2025 14:51:29 +0200 Subject: [PATCH 10/17] [SCHEMA] removing provenance subdir level description --- src/schema/objects/metadata.yaml | 24 ++++++++++++++++--- src/schema/rules/directories.yaml | 7 ------ .../rules/files/common/modality_agnostic.yaml | 6 ++--- 3 files changed, 23 insertions(+), 14 deletions(-) diff --git a/src/schema/objects/metadata.yaml b/src/schema/objects/metadata.yaml index fd8c2a8b10..6e290a7f9c 100644 --- a/src/schema/objects/metadata.yaml +++ b/src/schema/objects/metadata.yaml @@ -1281,11 +1281,23 @@ GeneratedBy: URI: type: string format: uri +GeneratedById: + name: GeneratedBy + display_name: Generated By + description: | + Specify the provenance of a file in the dataset, referencing the Id of the Activity (or Activities) responsible for its creation. + anyOf: + - type: string + format: uri + - type: array + items: + type: string + format: uri GeneratedByProv: name: GeneratedByProv display_name: Generated By Prov description: | - Specify the provenance of a file in the dataset. + Specify the provenance of the dataset using BIDS-Prov specifications. type: object GeneticLevel: name: GeneticLevel @@ -3302,8 +3314,14 @@ SidecarGeneratedBy: name: SidecarGeneratedBy display_name: Sidecar Generated By description: | - Specify the provenance of a sidecar JSON. - type: object + Specify the provenance of a sidecar JSON, referencing the Id of the Activity (or Activities) responsible for its creation. + anyOf: + - type: string + format: uri + - type: array + items: + type: string + format: uri SinglesRate: name: SinglesRate display_name: Singles Rate diff --git a/src/schema/rules/directories.yaml b/src/schema/rules/directories.yaml index 29e4295482..06988660ae 100644 --- a/src/schema/rules/directories.yaml +++ b/src/schema/rules/directories.yaml @@ -58,20 +58,17 @@ raw: - oneOf: - session - datatype - - prov session: entity: session level: optional opaque: false subdirs: - datatype - - prov datatype: value: datatype level: required opaque: false subdirs: - - prov derivative: root: @@ -114,17 +111,13 @@ derivative: subdirs: - session - datatype - - prov session: entity: session level: optional opaque: false subdirs: - datatype - - prov datatype: value: datatype level: optional opaque: false - subdirs: - - prov diff --git a/src/schema/rules/files/common/modality_agnostic.yaml b/src/schema/rules/files/common/modality_agnostic.yaml index 2343f4d443..3dbab51678 100644 --- a/src/schema/rules/files/common/modality_agnostic.yaml +++ b/src/schema/rules/files/common/modality_agnostic.yaml @@ -4,6 +4,7 @@ # Provenance json files provenance: level: optional + path: prov/* suffixes: - act - base @@ -13,18 +14,15 @@ provenance: extensions: - .json entities: - subject: optional - session: optional prov: required # Provenance ld files provenanceld: level: optional + path: prov/* suffixes: - all extensions: - .jsonld entities: - subject: optional - session: optional prov: required From 6622d082ee0ec1de4fa2c17b706a8bd462c30d70 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Thu, 10 Apr 2025 15:13:54 +0200 Subject: [PATCH 11/17] Yaml lint --- src/schema/objects/metadata.yaml | 6 ++++-- src/schema/objects/suffixes.yaml | 15 ++++++++++----- src/schema/rules/checks/deprecations.yml | 1 + 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/schema/objects/metadata.yaml b/src/schema/objects/metadata.yaml index 6e290a7f9c..8966fdfc0e 100644 --- a/src/schema/objects/metadata.yaml +++ b/src/schema/objects/metadata.yaml @@ -1285,7 +1285,8 @@ GeneratedById: name: GeneratedBy display_name: Generated By description: | - Specify the provenance of a file in the dataset, referencing the Id of the Activity (or Activities) responsible for its creation. + Specify the provenance of a file in the dataset, referencing the Id + of the Activity (or Activities) responsible for its creation. anyOf: - type: string format: uri @@ -3314,7 +3315,8 @@ SidecarGeneratedBy: name: SidecarGeneratedBy display_name: Sidecar Generated By description: | - Specify the provenance of a sidecar JSON, referencing the Id of the Activity (or Activities) responsible for its creation. + Specify the provenance of a sidecar JSON, + referencing the Id of the Activity (or Activities) responsible for its creation. anyOf: - type: string format: uri diff --git a/src/schema/objects/suffixes.yaml b/src/schema/objects/suffixes.yaml index cb4613e33a..e9971c99b0 100644 --- a/src/schema/objects/suffixes.yaml +++ b/src/schema/objects/suffixes.yaml @@ -506,12 +506,14 @@ act: value: act display_name: Provenance Activity Records description: | - Provenance Activity records for a group of provenance. Activities represent the transformations that have been applied to the data. + Provenance Activity records for a group of provenance. + Activities represent the transformations that have been applied to the data. all: value: all display_name: Provenance Records description: | - All types of provenance records (Activity, Entity, Environment, Software) for a group of provenance. + All types of provenance records (Activity, Entity, Environment, Software) + for a group of provenance. angio: value: angio display_name: Angiogram @@ -631,12 +633,14 @@ ent: value: ent display_name: Provenance Entity Records description: | - Provenance Entity records for a group of provenance. Entities are inputs and outputs of Activities. + Provenance Entity records for a group of provenance. + Entities are inputs and outputs of Activities. env: value: env display_name: Provenance Environment Records description: | - Provenance Environment records for a group of provenance. Environments specify the software environment in which the provenance record was obtained. + Provenance Environment records for a group of provenance. + Environments specify the software environment in which the provenance record was obtained. epi: value: epi display_name: EPI @@ -846,7 +850,8 @@ soft: value: soft display_name: Provenance Software Records description: | - Provenance Software records for a group of provenance. The Software specifies the software package with which transformations were applied to the data. + Provenance Software records for a group of provenance. + The Software specifies the software package with which transformations were applied to the data. sbref: value: sbref display_name: Single-band reference image diff --git a/src/schema/rules/checks/deprecations.yml b/src/schema/rules/checks/deprecations.yml index d7a3c27372..d45be8356a 100644 --- a/src/schema/rules/checks/deprecations.yml +++ b/src/schema/rules/checks/deprecations.yml @@ -1,3 +1,4 @@ +--- AnatomicalLandmarkCoordinateSystemDeprecation: issue: code: ELEKTA_NEUROMAG_DEPRECATED From 99faa5199d1e610bd5c8cb1a822d95830fa156f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Fri, 25 Apr 2025 15:06:38 +0200 Subject: [PATCH 12/17] Into JSON only provenance files --- src/schema/objects/extensions.yaml | 9 --------- src/schema/rules/dataset_metadata.yaml | 2 +- src/schema/rules/files/common/modality_agnostic.yaml | 11 ----------- 3 files changed, 1 insertion(+), 21 deletions(-) diff --git a/src/schema/objects/extensions.yaml b/src/schema/objects/extensions.yaml index 268ecdc173..1984661e87 100644 --- a/src/schema/objects/extensions.yaml +++ b/src/schema/objects/extensions.yaml @@ -130,15 +130,6 @@ json: These sidecar files follow the inheritance principle. There are also a few special cases of JSON files being first-order data files, such as `genetic_info.json`. -jsonld: - value: .jsonld - display_name: JavaScript Object Notation for Linked Data - description: | - A JSON-LD file. - - JSON-LD files are used to describe relations between objects in the dataset. - - For example, the description of provenance uses this file format. kdf: value: .kdf display_name: KRISS KDF diff --git a/src/schema/rules/dataset_metadata.yaml b/src/schema/rules/dataset_metadata.yaml index 504baf6498..d755f4e01f 100644 --- a/src/schema/rules/dataset_metadata.yaml +++ b/src/schema/rules/dataset_metadata.yaml @@ -16,7 +16,7 @@ dataset_description: EthicsApprovals: optional ReferencesAndLinks: optional DatasetDOI: optional - GeneratedBy: recommended + GeneratedBy: deprecated GeneratedByProv: recommended SourceDatasets: recommended diff --git a/src/schema/rules/files/common/modality_agnostic.yaml b/src/schema/rules/files/common/modality_agnostic.yaml index 3dbab51678..9ec04116fd 100644 --- a/src/schema/rules/files/common/modality_agnostic.yaml +++ b/src/schema/rules/files/common/modality_agnostic.yaml @@ -15,14 +15,3 @@ provenance: - .json entities: prov: required - -# Provenance ld files -provenanceld: - level: optional - path: prov/* - suffixes: - - all - extensions: - - .jsonld - entities: - prov: required From fdd60f2a000b68252ad9488900cef5d5b7752b90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 20 May 2025 13:47:47 +0200 Subject: [PATCH 13/17] GeneratedByProv only contain uri(s) --- src/schema/objects/metadata.yaml | 13 ++++++++++--- src/schema/rules/dataset_metadata.yaml | 2 +- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/schema/objects/metadata.yaml b/src/schema/objects/metadata.yaml index 8966fdfc0e..3304ba2b18 100644 --- a/src/schema/objects/metadata.yaml +++ b/src/schema/objects/metadata.yaml @@ -1281,7 +1281,7 @@ GeneratedBy: URI: type: string format: uri -GeneratedById: +GeneratedBy: name: GeneratedBy display_name: Generated By description: | @@ -1298,8 +1298,15 @@ GeneratedByProv: name: GeneratedByProv display_name: Generated By Prov description: | - Specify the provenance of the dataset using BIDS-Prov specifications. - type: object + Specify the provenance of the dataset, referencing the Id + of the Activity (or Activities) responsible for its creation. + anyOf: + - type: string + format: uri + - type: array + items: + type: string + format: uri GeneticLevel: name: GeneticLevel display_name: Genetic Level diff --git a/src/schema/rules/dataset_metadata.yaml b/src/schema/rules/dataset_metadata.yaml index d755f4e01f..1f062703f8 100644 --- a/src/schema/rules/dataset_metadata.yaml +++ b/src/schema/rules/dataset_metadata.yaml @@ -40,7 +40,7 @@ derivative_description: - json.DatasetType == "derivative" fields: GeneratedByProv: recommended - GeneratedBy: required + GeneratedBy: deprecated dataset_description_with_genetics: selectors: From 93c75e1f3ba276e8fea7c5c99db7236045720698 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Wed, 21 May 2025 14:41:30 +0200 Subject: [PATCH 14/17] Change name for GeneratedBy in JSON sidecars --- src/schema/objects/metadata.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schema/objects/metadata.yaml b/src/schema/objects/metadata.yaml index 3304ba2b18..5800789f03 100644 --- a/src/schema/objects/metadata.yaml +++ b/src/schema/objects/metadata.yaml @@ -1281,7 +1281,7 @@ GeneratedBy: URI: type: string format: uri -GeneratedBy: +GeneratedById: name: GeneratedBy display_name: Generated By description: | From 0211151c7d025885f691d63146c422babb6ef85d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Wed, 21 May 2025 15:02:37 +0200 Subject: [PATCH 15/17] Adding Provenance records in the schema: Activities, Entities, Software and Environments --- src/schema/objects/metadata.yaml | 177 +++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) diff --git a/src/schema/objects/metadata.yaml b/src/schema/objects/metadata.yaml index 6f864aa0f2..7b28019ecc 100644 --- a/src/schema/objects/metadata.yaml +++ b/src/schema/objects/metadata.yaml @@ -49,6 +49,70 @@ AcquisitionVoxelSize: type: number exclusiveMinimum: 0 unit: mm +Activities: + name: Activities + display_name: Activities + description: | + Describe Activities provenance records. + type: array + minItems: 1 + items: + type: object + required: [Id, Label, Command] + properties: + Id: + name: Id + description: Identifier for the activity. + type: string + format: uri + Label: + name: Label + description: Name of the tool, script, or function used (e.g. “bet”, "recon-all", "myFunc", "docker"). + type: string + format: label + Command: + name: Command + description: Command(s) used to run the tool, including all parameters. + type: string + AssociatedWith: + name: AssociatedWith + description: | + Identifier(s) of the software package(s) used to compute this activity. + The corresponding Software must be defined with its own Software record). + anyOf: + - type: string + format: uri + - type: array + items: + type: string + format: uri + Used: + name: Used + description: | + Identifier(s) of entity(ies) or environment(s) used by this activity. + The corresponding Entities (resp. Environments) must be defined with their own Entity (resp. Environment) record. + anyOf: + - type: string + format: uri + - type: array + items: + type: string + format: uri + Type: + name: Type + description: A term from a controlled vocabulary that more specifically describes the Activity. + type: string + format: uri + StartedAtTime: + name: StartedAtTime + description: A timestamp tracking when this activity started. + type: string + format: date + EndedAtTime: + name: EndedAtTime + description: A timestamp tracking when this activity ended. + type: string + format: date Anaesthesia: name: Anaesthesia display_name: Anaesthesia @@ -1084,6 +1148,87 @@ EncodingTechnique: For example, `"Cartesian"`, `"EPSI"`, `"Spiral"`, or `"Density-weighted concentric ring trajectory"`. type: string +Entities: + name: Entities + display_name: Entities + description: | + Describe Entities provenance records. + type: array + minItems: 1 + items: + type: object + required: [Id, Label] + recommanded: [Digest] + properties: + Id: + name: Id + description: Identifier for the entity. + type: string + format: bids_uri + Label: + name: Label + description: A name for the entity. + type: string + format: label + Digest: + name: Digest + description: | + For files, this would include checksums of files. + It would take the form {"": "value"}. + type: object + AtLocation: + name: AtLocation + description: | + For input files, this is the relative path to the file on disk. + type: string + GeneratedBy: + name: GeneratedBy + description: | + Identifier(s) of the Activity(ies) which generated this Entity. + The corresponding Activity must be defined with its own Activity record. + type: string + format: uri + Type: + name: Type + description: A term from a controlled vocabulary that more specifically describes the Entity. + type: string + format: uri +Environments: + name: Environments + display_name: Environments + description: | + Describe Environments provenance records. + type: array + minItems: 1 + items: + type: object + required: [Id, Label] + properties: + Id: + name: Id + description: | + Identifier for the environment (this identifier will be used to associated activities with this environment). + type: string + format: uri + Label: + name: Label + description: A name for the environment. + type: string + format: label + EnvVars: + name: EnvVars + description: | + A dictionary defining the environment variables as key-value pairs. + type: object + OperatingSystem: + name: OperatingSystem + description: Name of the operating system. + type: string + Dependencies: + name: Dependencies + description: | + A dictionary defining the software used and their versions as key-value pairs. + type: object EpochLength: name: EpochLength display_name: Epoch Length @@ -3399,6 +3544,38 @@ SliceTiming: type: number minimum: 0 unit: s +Software: + name: Software + display_name: Software + description: | + Describe Software provenance records. + type: array + minItems: 1 + items: + type: object + required: [Id, Label, Version] + properties: + Id: + name: Id + description: | + that will be used to associate activities with this software package (e.g., urn:1264-1233-11231-12312, "urn:bet-o1ef4rt") + type: string + format: uri + Label: + name: Label + description: Name of the software package. + type: string + format: label + Version: + name: Version + description: Version of the software package. + type: string + AltIdentifier: + name: AltIdentifier + description: | + URI of the RRID for this software package (cf. https://rrid.site/). + type: string + format: uri SoftwareFilters: name: SoftwareFilters display_name: Software Filters From a6c694322c380a485e20e26f465480284f7dc474 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Fri, 23 May 2025 15:19:41 +0200 Subject: [PATCH 16/17] Codespell --- src/schema/objects/metadata.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/schema/objects/metadata.yaml b/src/schema/objects/metadata.yaml index 7b28019ecc..bfe5cbe1bd 100644 --- a/src/schema/objects/metadata.yaml +++ b/src/schema/objects/metadata.yaml @@ -1158,7 +1158,7 @@ Entities: items: type: object required: [Id, Label] - recommanded: [Digest] + recommended: [Digest] properties: Id: name: Id @@ -3576,6 +3576,14 @@ Software: URI of the RRID for this software package (cf. https://rrid.site/). type: string format: uri + ActedOnBehalfOf: + name: ActedOnBehalfOf + description: | + Identifier of another software package that was responsible for using this software package in the context of the activities associated to it. + The corresponding software must be defined with its own Software record. + Example: if software A launches software B to perform activity C, then B ActedOnBehalfOf A. + type: string + format: uri SoftwareFilters: name: SoftwareFilters display_name: Software Filters From ded04204ee29f6d4e2b68a90ae839339d43f0da6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Boris=20Cl=C3=A9net?= Date: Tue, 3 Jun 2025 10:36:27 +0200 Subject: [PATCH 17/17] Provenance datatype + removing base suffix --- src/schema/objects/datatypes.yaml | 5 +++++ src/schema/objects/suffixes.yaml | 5 ----- src/schema/rules/directories.yaml | 1 - src/schema/rules/files/common/modality_agnostic.yaml | 4 ++-- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/schema/objects/datatypes.yaml b/src/schema/objects/datatypes.yaml index 333cc4d922..cb16fa1b53 100644 --- a/src/schema/objects/datatypes.yaml +++ b/src/schema/objects/datatypes.yaml @@ -68,6 +68,11 @@ phenotype: Participant level measurement data (for example, responses from multiple questionnaires) split into individual files separate from `participants.tsv`. +prov: + value: prov + display_name: Provenance + description: | + Provenance records describing how files in the dataset were generated. nirs: value: nirs display_name: Near-Infrared Spectroscopy diff --git a/src/schema/objects/suffixes.yaml b/src/schema/objects/suffixes.yaml index e9971c99b0..5c8bdb5993 100644 --- a/src/schema/objects/suffixes.yaml +++ b/src/schema/objects/suffixes.yaml @@ -540,11 +540,6 @@ asllabeling: A deidentified screenshot of the planning of the labeling slab/plane with respect to the imaging slab or slices. This screenshot is based on DICOM macro C.8.13.5.14. -base: - value: base - display_name: Base for Provenance Records - description: | - Base fields for provenance records of a group of provenance. beh: value: beh display_name: Behavioral recording diff --git a/src/schema/rules/directories.yaml b/src/schema/rules/directories.yaml index 52f0f8078c..1faadfa765 100644 --- a/src/schema/rules/directories.yaml +++ b/src/schema/rules/directories.yaml @@ -77,7 +77,6 @@ raw: value: datatype level: required opaque: false - subdirs: derivative: root: diff --git a/src/schema/rules/files/common/modality_agnostic.yaml b/src/schema/rules/files/common/modality_agnostic.yaml index 9ec04116fd..f95295c57d 100644 --- a/src/schema/rules/files/common/modality_agnostic.yaml +++ b/src/schema/rules/files/common/modality_agnostic.yaml @@ -4,10 +4,10 @@ # Provenance json files provenance: level: optional - path: prov/* + datatypes: + - prov suffixes: - act - - base - ent - env - soft