diff --git a/src/schema/objects/datatypes.yaml b/src/schema/objects/datatypes.yaml index 333cc4d922..cb16fa1b53 100644 --- a/src/schema/objects/datatypes.yaml +++ b/src/schema/objects/datatypes.yaml @@ -68,6 +68,11 @@ phenotype: Participant level measurement data (for example, responses from multiple questionnaires) split into individual files separate from `participants.tsv`. +prov: + value: prov + display_name: Provenance + description: | + Provenance records describing how files in the dataset were generated. nirs: value: nirs display_name: Near-Infrared Spectroscopy diff --git a/src/schema/objects/entities.yaml b/src/schema/objects/entities.yaml index 885d54e91b..89c3e0b0db 100644 --- a/src/schema/objects/entities.yaml +++ b/src/schema/objects/entities.yaml @@ -229,6 +229,14 @@ processing: exploited. type: string format: label +prov: + name: prov + display_name: Provenance group + description: | + A grouping of provenance records. + Defining multiple provenance records groups is appropriate when several processings have been performed on data. + type: string + format: label reconstruction: name: rec display_name: Reconstruction diff --git a/src/schema/objects/files.yaml b/src/schema/objects/files.yaml index b16a8c9c1d..3b7553ffa0 100644 --- a/src/schema/objects/files.yaml +++ b/src/schema/objects/files.yaml @@ -102,6 +102,13 @@ code: (for example the one used to generate the derivatives from the raw data). See the [Code section](SPEC_ROOT/modality-agnostic-files/code.md) for more information. +prov: + display_name: Provenance Records + file_type: directory + description: | + A directory in which to store provenance metadata. + See the [Provenance section](SPEC_ROOT/modality-agnostic-files.md#provenance) + for more information. derivatives: display_name: Derivative data file_type: directory diff --git a/src/schema/objects/metadata.yaml b/src/schema/objects/metadata.yaml index 8af01e23a4..bfe5cbe1bd 100644 --- a/src/schema/objects/metadata.yaml +++ b/src/schema/objects/metadata.yaml @@ -49,6 +49,70 @@ AcquisitionVoxelSize: type: number exclusiveMinimum: 0 unit: mm +Activities: + name: Activities + display_name: Activities + description: | + Describe Activities provenance records. + type: array + minItems: 1 + items: + type: object + required: [Id, Label, Command] + properties: + Id: + name: Id + description: Identifier for the activity. + type: string + format: uri + Label: + name: Label + description: Name of the tool, script, or function used (e.g. “bet”, "recon-all", "myFunc", "docker"). + type: string + format: label + Command: + name: Command + description: Command(s) used to run the tool, including all parameters. + type: string + AssociatedWith: + name: AssociatedWith + description: | + Identifier(s) of the software package(s) used to compute this activity. + The corresponding Software must be defined with its own Software record). + anyOf: + - type: string + format: uri + - type: array + items: + type: string + format: uri + Used: + name: Used + description: | + Identifier(s) of entity(ies) or environment(s) used by this activity. + The corresponding Entities (resp. Environments) must be defined with their own Entity (resp. Environment) record. + anyOf: + - type: string + format: uri + - type: array + items: + type: string + format: uri + Type: + name: Type + description: A term from a controlled vocabulary that more specifically describes the Activity. + type: string + format: uri + StartedAtTime: + name: StartedAtTime + description: A timestamp tracking when this activity started. + type: string + format: date + EndedAtTime: + name: EndedAtTime + description: A timestamp tracking when this activity ended. + type: string + format: date Anaesthesia: name: Anaesthesia display_name: Anaesthesia @@ -1084,6 +1148,87 @@ EncodingTechnique: For example, `"Cartesian"`, `"EPSI"`, `"Spiral"`, or `"Density-weighted concentric ring trajectory"`. type: string +Entities: + name: Entities + display_name: Entities + description: | + Describe Entities provenance records. + type: array + minItems: 1 + items: + type: object + required: [Id, Label] + recommended: [Digest] + properties: + Id: + name: Id + description: Identifier for the entity. + type: string + format: bids_uri + Label: + name: Label + description: A name for the entity. + type: string + format: label + Digest: + name: Digest + description: | + For files, this would include checksums of files. + It would take the form {"": "value"}. + type: object + AtLocation: + name: AtLocation + description: | + For input files, this is the relative path to the file on disk. + type: string + GeneratedBy: + name: GeneratedBy + description: | + Identifier(s) of the Activity(ies) which generated this Entity. + The corresponding Activity must be defined with its own Activity record. + type: string + format: uri + Type: + name: Type + description: A term from a controlled vocabulary that more specifically describes the Entity. + type: string + format: uri +Environments: + name: Environments + display_name: Environments + description: | + Describe Environments provenance records. + type: array + minItems: 1 + items: + type: object + required: [Id, Label] + properties: + Id: + name: Id + description: | + Identifier for the environment (this identifier will be used to associated activities with this environment). + type: string + format: uri + Label: + name: Label + description: A name for the environment. + type: string + format: label + EnvVars: + name: EnvVars + description: | + A dictionary defining the environment variables as key-value pairs. + type: object + OperatingSystem: + name: OperatingSystem + description: Name of the operating system. + type: string + Dependencies: + name: Dependencies + description: | + A dictionary defining the software used and their versions as key-value pairs. + type: object EpochLength: name: EpochLength display_name: Epoch Length @@ -1281,6 +1426,32 @@ GeneratedBy: URI: type: string format: uri +GeneratedById: + name: GeneratedBy + display_name: Generated By + description: | + Specify the provenance of a file in the dataset, referencing the Id + of the Activity (or Activities) responsible for its creation. + anyOf: + - type: string + format: uri + - type: array + items: + type: string + format: uri +GeneratedByProv: + name: GeneratedByProv + display_name: Generated By Prov + description: | + Specify the provenance of the dataset, referencing the Id + of the Activity (or Activities) responsible for its creation. + anyOf: + - type: string + format: uri + - type: array + items: + type: string + format: uri GeneticLevel: name: GeneticLevel display_name: Genetic Level @@ -3292,6 +3463,19 @@ ShortChannelCount: The number of short channels. 0 indicates no short channels. type: integer minimum: 0 +SidecarGeneratedBy: + name: SidecarGeneratedBy + display_name: Sidecar Generated By + description: | + Specify the provenance of a sidecar JSON, + referencing the Id of the Activity (or Activities) responsible for its creation. + anyOf: + - type: string + format: uri + - type: array + items: + type: string + format: uri SinglesRate: name: SinglesRate display_name: Singles Rate @@ -3360,6 +3544,46 @@ SliceTiming: type: number minimum: 0 unit: s +Software: + name: Software + display_name: Software + description: | + Describe Software provenance records. + type: array + minItems: 1 + items: + type: object + required: [Id, Label, Version] + properties: + Id: + name: Id + description: | + that will be used to associate activities with this software package (e.g., urn:1264-1233-11231-12312, "urn:bet-o1ef4rt") + type: string + format: uri + Label: + name: Label + description: Name of the software package. + type: string + format: label + Version: + name: Version + description: Version of the software package. + type: string + AltIdentifier: + name: AltIdentifier + description: | + URI of the RRID for this software package (cf. https://rrid.site/). + type: string + format: uri + ActedOnBehalfOf: + name: ActedOnBehalfOf + description: | + Identifier of another software package that was responsible for using this software package in the context of the activities associated to it. + The corresponding software must be defined with its own Software record. + Example: if software A launches software B to perform activity C, then B ActedOnBehalfOf A. + type: string + format: uri SoftwareFilters: name: SoftwareFilters display_name: Software Filters diff --git a/src/schema/objects/suffixes.yaml b/src/schema/objects/suffixes.yaml index 1e2825ced2..5c8bdb5993 100644 --- a/src/schema/objects/suffixes.yaml +++ b/src/schema/objects/suffixes.yaml @@ -502,6 +502,18 @@ VFA: Depending on the provided metadata fields and the sequence type, data may be eligible for DESPOT1, DESPOT2 and their variants ([Deoni et al. 2005](https://doi.org/10.1002/mrm.20314)). +act: + value: act + display_name: Provenance Activity Records + description: | + Provenance Activity records for a group of provenance. + Activities represent the transformations that have been applied to the data. +all: + value: all + display_name: Provenance Records + description: | + All types of provenance records (Activity, Entity, Environment, Software) + for a group of provenance. angio: value: angio display_name: Angiogram @@ -612,6 +624,18 @@ electrodes: display_name: Electrodes description: | File that gives the location of (i)EEG electrodes. +ent: + value: ent + display_name: Provenance Entity Records + description: | + Provenance Entity records for a group of provenance. + Entities are inputs and outputs of Activities. +env: + value: env + display_name: Provenance Environment Records + description: | + Provenance Environment records for a group of provenance. + Environments specify the software environment in which the provenance record was obtained. epi: value: epi display_name: EPI @@ -817,6 +841,12 @@ probseg: A probabilistic segmentation. This suffix may only be used in derivative datasets. +soft: + value: soft + display_name: Provenance Software Records + description: | + Provenance Software records for a group of provenance. + The Software specifies the software package with which transformations were applied to the data. sbref: value: sbref display_name: Single-band reference image diff --git a/src/schema/rules/checks/deprecations.yml b/src/schema/rules/checks/deprecations.yml index d7a3c27372..d45be8356a 100644 --- a/src/schema/rules/checks/deprecations.yml +++ b/src/schema/rules/checks/deprecations.yml @@ -1,3 +1,4 @@ +--- AnatomicalLandmarkCoordinateSystemDeprecation: issue: code: ELEKTA_NEUROMAG_DEPRECATED diff --git a/src/schema/rules/dataset_metadata.yaml b/src/schema/rules/dataset_metadata.yaml index d4ecca7da1..1f062703f8 100644 --- a/src/schema/rules/dataset_metadata.yaml +++ b/src/schema/rules/dataset_metadata.yaml @@ -16,7 +16,8 @@ dataset_description: EthicsApprovals: optional ReferencesAndLinks: optional DatasetDOI: optional - GeneratedBy: recommended + GeneratedBy: deprecated + GeneratedByProv: recommended SourceDatasets: recommended dataset_authors: @@ -38,7 +39,8 @@ derivative_description: - path == "/dataset_description.json" - json.DatasetType == "derivative" fields: - GeneratedBy: required + GeneratedByProv: recommended + GeneratedBy: deprecated dataset_description_with_genetics: selectors: diff --git a/src/schema/rules/directories.yaml b/src/schema/rules/directories.yaml index 3bfea7c26e..1faadfa765 100644 --- a/src/schema/rules/directories.yaml +++ b/src/schema/rules/directories.yaml @@ -23,6 +23,7 @@ raw: - derivatives - logs - phenotype + - prov - sourcedata - stimuli - subject @@ -46,6 +47,10 @@ raw: name: phenotype level: optional opaque: false + prov: + name: prov + level: optional + opaque: false sourcedata: name: sourcedata level: optional @@ -80,6 +85,7 @@ derivative: - derivatives - logs - phenotype + - prov - sourcedata - stimuli - subject @@ -99,6 +105,10 @@ derivative: name: phenotype level: optional opaque: false + prov: + name: prov + level: optional + opaque: false sourcedata: name: sourcedata level: optional diff --git a/src/schema/rules/entities.yaml b/src/schema/rules/entities.yaml index 44174d333e..e1d118b754 100644 --- a/src/schema/rules/entities.yaml +++ b/src/schema/rules/entities.yaml @@ -32,3 +32,4 @@ - density - label - description +- prov diff --git a/src/schema/rules/files/common/core.yaml b/src/schema/rules/files/common/core.yaml index fe8535d0ce..d83b7cd580 100644 --- a/src/schema/rules/files/common/core.yaml +++ b/src/schema/rules/files/common/core.yaml @@ -36,6 +36,9 @@ code: derivatives: level: optional path: derivatives +prov: + level: optional + path: prov docs: level: optional path: docs diff --git a/src/schema/rules/files/common/modality_agnostic.yaml b/src/schema/rules/files/common/modality_agnostic.yaml new file mode 100644 index 0000000000..f95295c57d --- /dev/null +++ b/src/schema/rules/files/common/modality_agnostic.yaml @@ -0,0 +1,17 @@ +--- +# Files and directories that can be found at any level of a dataset. + +# Provenance json files +provenance: + level: optional + datatypes: + - prov + suffixes: + - act + - ent + - env + - soft + extensions: + - .json + entities: + prov: required diff --git a/src/schema/rules/files/common/tables.yaml b/src/schema/rules/files/common/tables.yaml index 21f03f6579..7c818acc66 100644 --- a/src/schema/rules/files/common/tables.yaml +++ b/src/schema/rules/files/common/tables.yaml @@ -5,12 +5,14 @@ participants: extensions: - .tsv - .json + samples: level: optional stem: samples extensions: - .tsv - .json + scans: level: optional suffixes: @@ -21,6 +23,7 @@ scans: entities: subject: required session: optional # session is required if session is present in the dataset. + sessions: # This file may only exist if session is present in the dataset. level: optional suffixes: