diff --git a/src/modality-specific-files/behavioral-experiments.md b/src/modality-specific-files/behavioral-experiments.md index 81af91e65c..e624b38fcc 100644 --- a/src/modality-specific-files/behavioral-experiments.md +++ b/src/modality-specific-files/behavioral-experiments.md @@ -1,4 +1,4 @@ -# Behavioral experiments (with no neural recordings) +# Behavioral recordings !!! example "Example datasets" @@ -15,19 +15,14 @@ and a guide for using macros can be found at --> {{ MACROS___make_filename_template("raw", datatypes=["beh"]) }} -In addition to logs from behavioral experiments -performed alongside imaging data acquisitions, -one MAY also include data from experiments -performed with no neural recordings. -The results of those experiments MAY be stored in the `beh` directory -using the same formats for event timing (`_events.tsv`), -metadata (`_events.json`), -physiological (`_physio.tsv.gz`, `_physio.json`) -and other continuous recordings (`_stim.tsv.gz`, `_stim.json`) -as for tasks performed during MRI, electrophysiological or other neural recordings. -Additionally, events files -that do not include the mandatory `onset` and `duration` columns -MAY be included, +The `beh` directory MAY store behavioral recordings such as audio (`_audio.*`), video (`_video.*`), combined audio-video (`_audiovideo.*`), and still image (`_image.*`) recordings, physiological (`_physio.*`) recordings, and other continuous recordings (`_stim.tsv.gz`, `_stim.json`). +Audio, video, audio-video, and image recordings MAY be of subjects performing tasks, resting-state behavior, or recordings of stimuli being presented to the subject. +Audio/video recordings MAY occur simultaneously with other recordings, such as BOLD or EEG. +Relative timing between files may be determined by consulting the `scans.tsv` file. +If no `scans.tsv` file is present, the alignment is undefined. +The `beh` directory MAY also contain event timing files (`_events.tsv`) and their associated metadata (`_events.json`) for behavioral experiments that do not have corresponding neuroimaging or functional data. + +Additionally, events files that do not include the mandatory `onset` and `duration` columns MAY be included, but MUST be labeled `_beh.tsv` rather than `_events.tsv`. The following OPTIONAL columns are pre-defined for behavioral data files: @@ -76,6 +71,243 @@ A guide for using macros can be found at --> {{ MACROS___make_sidecar_table("beh.BEHInstitutionInformation") }} +## Audio, video, and audio-video recordings and images + +Audio and video recordings of behaving subjects MAY be stored in the `beh` directory +using the `_audio`, `_video`, and `_audiovideo` suffixes. +The `_audio` suffix is for audio-only recordings, `_video` for video-only recordings, +and `_audiovideo` for recordings that contain both audio and video streams. +These recordings are typically used to capture vocalizations, speech, facial expressions, +body movements, or other behavioral aspects during experimental tasks or rest periods. + +Still images captured during behavioral experiments MAY be stored in the `beh` directory +using the `_image` suffix. +These images are typically used for training frames for pose estimation, +snapshots of behavioral setups, or individual frames extracted from video recordings. + +!!! warning "Privacy and personally identifiable information" + + Audio and video recordings and images of human subjects often contain personally identifiable + information (PII) such as faces, voices, and other identifying features. + Data curators MUST take special care to ensure compliance with applicable privacy + regulations (such as HIPAA in the United States, GDPR in the European Union, or other + local data protection laws) when handling these recordings. + + These recordings are generally more suitable for internal use or for sharing + non-human subject data, unless appropriate privacy protections are implemented. + +### File formats + +Audio recordings MUST use one of the following extensions: + +- `.flac` - Free Lossless Audio Codec +- `.mp3` - MPEG Audio Layer III +- `.ogg` - Ogg Vorbis +- `.wav` - Waveform Audio File Format + +Video and audio-video recordings MUST use one of the following extensions: + +- `.mp4` - MPEG-4 Part 14 +- `.mkv` - Matroska video container +- `.avi` - Audio Video Interleave + +Image files MUST use one of the following extensions: + +- `.jpg` - JPEG image +- `.png` - Portable Network Graphics + +### Entities + +Audio and video files MAY use the following entities: + +- `task` - OPTIONAL for audio and video recordings +- `acq` - OPTIONAL, can distinguish different recording setups +- `run` - OPTIONAL, for multiple recordings with identical parameters +- `recording` - OPTIONAL, to differentiate simultaneous recordings from different angles, locations, or devices +- `split` - OPTIONAL, for continuous recordings split into multiple files + +### Examples + + +{{ MACROS___make_filetree_example( + { + "sub-01": { + "beh": { + "sub-01_task-rest_video.mp4": "", + "sub-01_task-rest_video.json": "", + "sub-01_task-interview_audiovideo.mp4": "", + "sub-01_task-interview_audiovideo.json": "", + "sub-01_task-stroop_recording-face_video.mp4": "", + "sub-01_task-stroop_recording-face_video.json": "", + "sub-01_task-stroop_recording-room_video.mp4": "", + "sub-01_task-stroop_recording-room_video.json": "", + "sub-01_task-rest_image.jpg": "", + "sub-01_task-rest_image.json": "", + "sub-01_task-vocalization_audio.wav": "", + "sub-01_task-vocalization_audio.json": "", + }, + }, + } +) }} + +For continuous recordings split into multiple files: + + +{{ MACROS___make_filetree_example( + { + "sub-01": { + "ses-01": { + "beh": { + "sub-01_ses-01_task-freeplay_run-01_split-001_video.mp4": "", + "sub-01_ses-01_task-freeplay_run-01_split-002_video.mp4": "", + "sub-01_ses-01_task-freeplay_run-01_split-003_video.mp4": "", + "sub-01_ses-01_task-freeplay_run-01_video.json": "", + }, + }, + }, + } +) }} + +### Sidecar JSON for audio, video, audio-video recordings, and images + +The following metadata fields are available for audio, video, audio-video recordings, and images: + + +{{ MACROS___make_sidecar_table("beh.AudioVideoImageDevice") }} + +!!! note "Licensing for recordings containing participants" + + Audio, video, and image recordings of participants may have different licensing + restrictions than the main dataset due to privacy considerations. The optional + `License` field can be used to specify different terms for individual recordings + that contain identifiable participant data. If not specified, the recording + inherits the license from `dataset_description.json`. + +The general stream properties of these files (duration, codecs, sample rate, +dimensions, and frame rate) are described by the common media file metadata +(see [Common media file metadata](../appendices/media-files.md)): + +{{ MACROS___make_sidecar_table("media.MediaDuration") }} + +The following fields are available for audio recordings (`_audio`) and audio-video recordings (`_audiovideo`): + +{{ MACROS___make_sidecar_table("media.MediaAudioProperties") }} + +The following fields are available for video recordings (`_video`) and audio-video recordings (`_audiovideo`): + +{{ MACROS___make_sidecar_table(["media.MediaVideoProperties", "media.MediaImageProperties"]) }} + +The following fields are available for image files (`_image`): + +{{ MACROS___make_sidecar_table("media.MediaImageProperties") }} + +### Example audio-video sidecar JSON + +For an audio-video file containing both video and audio streams: + +```JSON +{ + "TaskName": "RestingState", + "Device": "Sony FDR-AX53", + "RecordingDuration": 600.5, + "AudioCodec": "aac", + "AudioSampleRate": 48000, + "AudioChannelCount": 2, + "VideoCodec": "h264", + "VideoFrameRate": 30.0, + "VideoFrameCount": 18015, + "ImageWidth": 1920, + "ImageHeight": 1080 +} +``` + +### Example video sidecar JSON + +For a video-only recording (no audio stream): + +```JSON +{ + "TaskName": "RestingState", + "Device": "Sony FDR-AX53", + "RecordingDuration": 600.5, + "VideoCodec": "h264", + "VideoFrameRate": 30.0, + "VideoFrameCount": 18015, + "ImageWidth": 1920, + "ImageHeight": 1080 +} +``` + +### Example audio sidecar JSON + +For an audio-only recording: + +```JSON +{ + "TaskName": "Vocalization", + "Device": "Zoom H6 Handy Recorder", + "RecordingDuration": 300.2, + "AudioCodec": "flac", + "AudioSampleRate": 44100, + "AudioChannelCount": 2, + "AudioBitDepth": 24 +} +``` + +### Example image sidecar JSON + +For a still image: + +```JSON +{ + "TaskName": "Reaching", + "Device": "GoPro Hero 10", + "ImageWidth": 1920, + "ImageHeight": 1080, + "DevicePosition": "overhead" +} +``` + +### Annotations and events + +Behavioral annotations or event markers for audio and video recordings +SHOULD be stored in accompanying `_events.tsv` files following the standard +[events file format](../modality-agnostic-files/events.md). +These events files use the same filename entities as the audio/video file they describe, +but with the `_events` suffix. + +For example: + + +{{ MACROS___make_filetree_example( + { + "sub-01": { + "beh": { + "sub-01_task-speech_audio.wav": "", + "sub-01_task-speech_audio.json": "", + "sub-01_task-speech_events.tsv": "", + "sub-01_task-speech_events.json": "", + }, + }, + } +) }} + ## Example `_beh.tsv` ```tsv diff --git a/src/schema/objects/metadata.yaml b/src/schema/objects/metadata.yaml index f7af1cbb9b..e035e6b309 100644 --- a/src/schema/objects/metadata.yaml +++ b/src/schema/objects/metadata.yaml @@ -968,6 +968,13 @@ Descriptors: - type: array items: type: string +Device: + name: Device + display_name: Device + description: | + Free-form description of the device used to record the data + (for example, `"iPhone 12"`, `"Canon EOS R5"`). + type: string DeviceSerialNumber: name: DeviceSerialNumber display_name: Device Serial Number @@ -4771,3 +4778,13 @@ iEEGReference: this field should have a general description and the channel specific reference should be defined in the `channels.tsv` file. type: string + +DevicePosition: + name: DevicePosition + display_name: Device Position + description: | + Free-form description of the recording device's placement relative to the subject or scene. + + This applies to any recording device, such as a camera or microphone. + Examples include "front", "profile-left", "ceiling", "room-corner", or "overhead". + type: string diff --git a/src/schema/rules/files/raw/beh.yaml b/src/schema/rules/files/raw/beh.yaml index df6f9dac06..4d96c62b41 100644 --- a/src/schema/rules/files/raw/beh.yaml +++ b/src/schema/rules/files/raw/beh.yaml @@ -9,3 +9,88 @@ noncontinuous: - .json datatypes: - beh + entities: + subject: required + session: optional + task: required + acquisition: optional + run: optional + +# Audio recordings +audio: + suffixes: + - audio + extensions: + - .flac + - .mp3 + - .ogg + - .wav + - .json + datatypes: + - beh + entities: + subject: required + session: optional + task: optional + acquisition: optional + run: optional + recording: optional + split: optional + +# Video recordings +video: + suffixes: + - video + extensions: + - .mp4 + - .mkv + - .avi + - .json + datatypes: + - beh + entities: + subject: required + session: optional + task: optional + acquisition: optional + run: optional + recording: optional + split: optional + +# Combined audio-video recordings +audiovideo: + suffixes: + - audiovideo + extensions: + - .mp4 + - .mkv + - .avi + - .json + datatypes: + - beh + entities: + subject: required + session: optional + task: optional + acquisition: optional + run: optional + recording: optional + split: optional + +# Still images +image: + suffixes: + - image + extensions: + - .jpg + - .png + - .json + datatypes: + - beh + entities: + subject: required + session: optional + task: optional + acquisition: optional + run: optional + recording: optional diff --git a/src/schema/rules/sidecars/beh.yaml b/src/schema/rules/sidecars/beh.yaml index f2d8410914..fa160abd98 100644 --- a/src/schema/rules/sidecars/beh.yaml +++ b/src/schema/rules/sidecars/beh.yaml @@ -25,3 +25,17 @@ BEHInstitutionInformation: InstitutionName: recommended InstitutionAddress: recommended InstitutionalDepartmentName: recommended + +# Behavioral-specific metadata for media files. +# General audio/video/image stream properties (codecs, sample rate, dimensions, +# frame rate, duration) are defined by the common media rules in +# `rules/sidecars/media.yaml` and apply to these suffixes regardless of datatype. +AudioVideoImageDevice: + selectors: + - datatype == "beh" + - intersects([suffix], ["audio", "video", "audiovideo", "image"]) + fields: + Device: optional + DeviceSerialNumber: optional + DevicePosition: optional + License: optional