diff --git a/mkdocs.yml b/mkdocs.yml index 4b9e3449bd..02634ac3dc 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -10,6 +10,7 @@ nav: - Data summary files: modality-agnostic-files/data-summary-files.md - Phenotypic and assessment data: modality-agnostic-files/phenotypic-and-assessment-data.md - Code: modality-agnostic-files/code.md + - Provenance: modality-agnostic-files/provenance.md - Events: modality-agnostic-files/events.md - Modality specific files: - Magnetic Resonance Imaging: modality-specific-files/magnetic-resonance-imaging-data.md @@ -125,6 +126,9 @@ markdown_extensions: - name: tsvgz class: tsv format: !!python/name:bidsschematools.render.tsv.fence + - name: mermaid + class: mermaid + format: !!python/name:pymdownx.superfences.fence_code_format - admonition - pymdownx.details plugins: diff --git a/package-lock.json b/package-lock.json index 80538ce56c..2c4ac5a71f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6,7 +6,7 @@ "": { "dependencies": { "all-contributors-cli": "^6.26.1", - "prettier": "^3.7.4", + "prettier": "^3.8.0", "remark-cli": "^12.0.1", "remark-gfm": "^4.0.1", "remark-lint-no-trailing-spaces": "^4.0.3", @@ -2609,9 +2609,9 @@ } }, "node_modules/prettier": { - "version": "3.7.4", - "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.7.4.tgz", - "integrity": "sha512-v6UNi1+3hSlVvv8fSaoUbggEM5VErKmmpGA7Pl3HF8V6uKY7rvClBOJlH6yNwQtfTueNkGVpOv/mtWL9L4bgRA==", + "version": "3.8.0", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.8.0.tgz", + "integrity": "sha512-yEPsovQfpxYfgWNhCfECjG5AQaO+K3dp6XERmOepyPDVqcJm+bjyCVO3pmU+nAPe0N5dDvekfGezt/EIiRe1TA==", "license": "MIT", "bin": { "prettier": "bin/prettier.cjs" diff --git a/package.json b/package.json index 110f5c6c50..5501d28f94 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "remark-lint-no-trailing-spaces": "^4.0.3", "remark-preset-lint-markdown-style-guide": "^6.0.1", "remark-preset-lint-recommended": "^7.0.1", - "prettier": "^3.7.4", + "prettier": "^3.8.0", "all-contributors-cli": "^6.26.1" } } diff --git a/src/common-principles.md b/src/common-principles.md index 8d98266129..13cffa0ff4 100644 --- a/src/common-principles.md +++ b/src/common-principles.md @@ -979,10 +979,10 @@ Bare DOIs such as `10.18112/openneuro.ds000001.v1.0.0` are [DEPRECATED][]. ### BIDS URI -To reference files in BIDS datasets, the following URI scheme may be used: +To reference files or directories in BIDS datasets, the following URI scheme may be used: ```plain -bids:[]: +bids:[]:[#] ``` The scheme component `bids` identifies a BIDS URI, @@ -991,6 +991,7 @@ The `dataset-name` component is an identifier for a BIDS dataset, and the `relative-path` component is the location of a resource within that BIDS dataset, relative to the root of that dataset. The `relative-path` MUST NOT start with a forward-slash character (`/`). +The `fragment` MAY be used to identify a resource that is subordinate to the file or directory. Examples: @@ -998,11 +999,17 @@ Examples: bids::sub-01/fmap/sub-01_dir-AP_epi.nii.gz bids:ds000001:sub-02/anat/sub-02_T1w.nii.gz bids:myderivatives:sub-03/func/sub-03_task-rest_space-MNI152_bold.nii.gz +bids:fmriprep:sub-001/anat/sub-001_T1w_preproc.nii.gz#aa56ztg8 +bids::prov#preprocessing-00f3a18f ``` If no dataset name is specified, the URI is relative to the current BIDS dataset. This is made more precise in the next section. +!!! note + A BIDS dataset can be referenced using the BIDS URI of its root directory. + For example: `bids:ds000001:.` + #### Resolution of BIDS URIs In order to resolve a BIDS URI, the dataset name must be mapped to a BIDS dataset. @@ -1054,7 +1061,7 @@ No protocol is currently proposed to automatically resolve all possible BIDS URI BIDS URIs are parsable as standard [URIs][] with scheme `bids` and path `[]:`. -The authority, query and fragment components are unused. +The authority and query components are unused. Future versions of BIDS may specify interpretations for these components, but MUST NOT change the interpretation of a previously valid BIDS URI. For example, a future version may specify an authority that would allow BIDS diff --git a/src/introduction.md b/src/introduction.md index 84e4ceea7c..f62dad0160 100644 --- a/src/introduction.md +++ b/src/introduction.md @@ -196,6 +196,15 @@ For example: - (publication forthcoming) +### Other extensions specific publications + +#### Provenance + +- Boris Clénet, Yaroslav O Halchenko, Satrajit Ghosh, Camille Maumet (2025). + **Extending the Brain Imaging Data Structure specification to provenance metadata**. + Distribits, Oct 2025, Germany. + [https://hal.science/hal-05406141](https://hal.science/hal-05406141) + ### Research Resource Identifier (RRID) BIDS has also a diff --git a/src/metaschema.json b/src/metaschema.json index 7d5b83ebf5..6a3efde619 100644 --- a/src/metaschema.json +++ b/src/metaschema.json @@ -362,6 +362,15 @@ }, "additionalProperties": false }, + "modality_agnostic": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9_]+$": { + "$ref": "#/definitions/suffixRule" + } + }, + "additionalProperties": false + }, "tables": { "type": "object", "patternProperties": { @@ -375,7 +384,7 @@ "additionalProperties": false } }, - "required": ["core", "tables"], + "required": ["core", "modality_agnostic", "tables"], "additionalProperties": false }, "deriv": { diff --git a/src/modality-agnostic-files/dataset-description.md b/src/modality-agnostic-files/dataset-description.md index 769afb0e9f..2087faa7a8 100644 --- a/src/modality-agnostic-files/dataset-description.md +++ b/src/modality-agnostic-files/dataset-description.md @@ -26,16 +26,9 @@ and a guide for using macros can be found at --> {{ MACROS___make_json_table('dataset_metadata.dataset_description') }} -Each object in the `GeneratedBy` array includes the following REQUIRED, RECOMMENDED -and OPTIONAL keys: - - -{{ MACROS___make_subobject_table("metadata.GeneratedBy.items") }} +!!! Note + See the [Provenance of a BIDS dataset](provenance.md#provenance-of-a-bids-dataset) section + for more information on how to describe provenance using the `GeneratedBy` field. Example: diff --git a/src/modality-agnostic-files/provenance.md b/src/modality-agnostic-files/provenance.md new file mode 100644 index 0000000000..eab4998cfb --- /dev/null +++ b/src/modality-agnostic-files/provenance.md @@ -0,0 +1,950 @@ +# Provenance + +Support for provenance was developed as +a [BIDS Extension Proposal](../extensions.md#bids-extension-proposals). +Please see [Citing BIDS](../introduction.md#citing-bids) on how to appropriately credit +this extension when referring to it in the context of the academic literature. + +!!! example "Example datasets" + + Several [example datasets](https://bids.neuroimaging.io/datasets/examples.html#provenance) + have been formatted using this specification and can be used + for practical guidance when curating a new dataset. + +This part of the BIDS specification is aimed at describing the provenance of a BIDS dataset. +This description is retrospective: it describes a set of steps that were executed in order to +establish the dataset and is based on [W3C PROV](https://www.w3.org/TR/2013/REC-prov-o-20130430/) +(see [Provenance graph](#provenance-graph)). + +Provenance information SHOULD be included in a BIDS dataset when possible. +If provenance information is included, +it MUST be described using the conventions detailed hereafter. +Provenance information reflects the provenance of a full dataset +and/or of specific files at any level of the BIDS hierarchy. +Provenance information SHOULD not include human subject identifying data. + +!!! Note + + Throughout this document, the terms `Id` and `Label` are used to provide identification + for JSON objects related to provenance. + `Id` is used to unambiguously [identify those objects](#provenance-identifiers) + that may be referenced elsewhere, + permitting automated tools to [construct and query a graph](#provenance-graph). + `Label` is a human-readable name for that object, which need not be unique, + and should not be confused with the BIDS term + [`label`](../glossary.md#label-common_principles). + +## Provenance of a BIDS file + +Provenance of a BIDS file SHOULD be stored inside its sidecar JSON. + +For that purpose, any sidecar JSON file MAY include the following keys: + + +{{ MACROS___make_sidecar_table('prov.CommonProvenanceFields') }} + +Each object in the `Checksum` array includes the following keys: + + +{{ MACROS___make_subobject_table("metadata.Checksum.items") }} + +!!! example "Example of metadata in a sidecar JSON file" + + ```JSON + { + "GeneratedBy": "bids::prov#conversion-00f3a18f", + "SidecarGeneratedBy": [ + "bids::prov#preparation-conversion-1xkhm1ft", + "bids::prov#conversion-00f3a18f" + ], + "Checksum": [ + { + "ChecksumAlgorithm": "spdx:checksumAlgorithm_sha256", + "ChecksumValue": "66eeafb465559148e0222d4079558a8354eb09b9efabcc47cd5b8af6eed51907" + } + ] + } + ``` + For a complete example see + [Provenance of DICOM to NIfTI conversion with `heudiconv`]( + https://github.com/bclenet/bids-examples/tree/BEP028_heudiconv/provenance_heudiconv). + +## Provenance of a BIDS dataset + +Provenance of a BIDS dataset (raw, derivative, or study) SHOULD be stored +inside its `dataset_description.json` file. +The `dataset_description.json` file of a **BIDS raw dataset** or **BIDS study dataset** MAY +include the `GeneratedBy` key to describe provenance. +The `dataset_description.json` file of a **BIDS derivative dataset** MUST +include the `GeneratedBy` key to describe provenance. + +The `GeneratedBy` field MAY contain either of the following values: + +- Identifier(s) of the activity/activities responsible for the creation of the dataset +(see [Description using identifiers](#description-using-identifiers)). + +- A description of pipelines or processes responsible for the creation of the dataset +(see [Description of pipelines or processes](#description-of-pipelines-or-processes)). + +### Description using identifiers + +This section details how to describe provenance of a dataset using identifiers. +The following field is intended for use in `dataset_description.json` to provide +provenance information that applies to the entire dataset. + + +{{ MACROS___make_metadata_table( + { + "GeneratedBy__Id__Dataset": "RECOMMENDED for BIDS raw datasets and BIDS study datasets,\ + REQUIRED for BIDS derivative datasets" + } +) }} + +!!! example "Example of `GeneratedBy` contents in a `dataset_description.json`" + + ```JSON + { + "GeneratedBy": "bids::prov#preprocessing-xMpFqB5q" + } + ``` + For a complete example see [Provenance of fMRI preprocessing with `fMRIPrep`]( + https://github.com/bclenet/bids-examples/tree/BEP028_fmriprep/provenance_fmriprep). + +### Description of processes or pipelines + +This section details how to describe the provenance of a dataset using an array of objects +representing pipelines or processes that generated the dataset. + +!!! warning + + This description can be equivalently represented using the previous section. + This modeling is kept for backward-compatibility but might be removed + in future BIDS releases (see BIDS 2.0). + + +{{ MACROS___make_metadata_table( + { + "GeneratedBy": "RECOMMENDED for BIDS raw datasets and BIDS study datasets,\ + REQUIRED for BIDS derivative datasets" + } +) }} + +Each object in the `GeneratedBy` array includes the following REQUIRED, RECOMMENDED +and OPTIONAL keys: + + +{{ MACROS___make_metadata_table( + { + "Name__GeneratedBy": "REQUIRED", + "Version__GeneratedBy": "RECOMMENDED", + "Description__GeneratedBy": 'RECOMMENDED if `Name` is `"Manual"`, OPTIONAL otherwise', + "CodeURL": "OPTIONAL", + "Container": "OPTIONAL" + } +) }} + +!!! example "Example of `GeneratedBy` contents in a `dataset_description.json`" + + ```JSON + { + "GeneratedBy": [ + { + "Name": "reproin", + "Version": "0.6.0", + "Container": { + "Type": "docker", + "Tag": "repronim/reproin:0.6.0" + } + } + ] + } + ``` + +## Provenance files + +Any provenance information that can't be stored in either sidecar JSON files +(see [Provenance of BIDS file](#provenance-of-a-bids-file)) or in `dataset_description.json` +(see [Provenance of BIDS dataset](#provenance-of-a-bids-dataset)) MUST be stored in +provenance files under the `/prov/` directory. + + +{{ MACROS___make_filename_template( + "common", + datatypes=["prov"], + suffixes=["act", "io", "soft", "env"]) +}} + +!!! note + + The [`prov entity`](../appendices/entities.md#prov) allows to group related provenance files, + using an arbitrary value for `