From 15bf8c26c4349c1827dbc0132a3a022e26f41c8d Mon Sep 17 00:00:00 2001 From: natachaperez Date: Wed, 7 Aug 2019 15:36:56 +0200 Subject: [PATCH 1/7] json context for bids documents only --- bids_context.json | 101 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 bids_context.json diff --git a/bids_context.json b/bids_context.json new file mode 100644 index 0000000..1194dfd --- /dev/null +++ b/bids_context.json @@ -0,0 +1,101 @@ +{ + "@context": { + "@version": 1.1, + "records": { + "@id": "@graph", + "@container": "@type" + }, + "bids": "https://bids.neuroimaging.io#", + "Acquisition Matrix": "https://bids.neuroimaging.io#AcquisitionMatrix", + "Coil Combination Method" : "https://bids.neuroimaging.io#CoilCombinationMethod", + "Echo Time" : "https://bids.neuroimaging.io#EchoTime", + "Field of View Dimensions" : "https://bids.neuroimaging.io#FieldOfViewDimensions", + "Field of View Shape" : "https://bids.neuroimaging.io#FieldOfViewShape", + "Flip Angle" : "https://bids.neuroimaging.io#FlipAngle", + "Gradient Set Type" : "https://bids.neuroimaging.io#GradientSetType", + "Inversion Time" : "https://bids.neuroimaging.io#InversionTime", + "MatrixCoilMode" : "https://bids.neuroimaging.io#MatrixCoilMode", + "Negative Contrast" : "https://bids.neuroimaging.io#NegativeContrast", + "Number Of Slices" : "https://bids.neuroimaging.io#NumberOfSlices", + "Number Shots" : "https://bids.neuroimaging.io#NumberShots", + "Parallel Acquisition" : "https://bids.neuroimaging.io#ParallelAcquisition", + "Parallel Acquisition Technique" : "https://bids.neuroimaging.io#ParallelAcquisitionTechnique", + "Parallel Reduction Factor In-plane" : "https://bids.neuroimaging.io#ParallelReductionFactorIn-plane", + "Phase Encoding Direction" : "https://bids.neuroimaging.io#PhaseEncodingDirection", + "Pixel Bandwidth" : "https://bids.neuroimaging.io#PixelBandwidth", + "Plane Orientation Sequence" : "https://bids.neuroimaging.io#PlaneOrientationSequence", + "Receive Coil Type" : "https://bids.neuroimaging.io#ReceiveCoilType", + "Receive Coil Name" : "https://bids.neuroimaging.io#ReceiveCoilName", + "Scan Options" : "https://bids.neuroimaging.io#ScanOptions", + "Scanning Sequence" : "https://bids.neuroimaging.io#ScanningSequence", + "Sequence Name" : "https://bids.neuroimaging.io#SequenceName", + "Sequence Variant" : "https://bids.neuroimaging.io#SequenceVariant", + "Slice Acquisition Order" : "https://bids.neuroimaging.io#SliceAcquisitionOrder", + "Slice Encoding Direction" : "https://bids.neuroimaging.io#SliceEncodingDirection", + "Slice Thickness" : "https://bids.neuroimaging.io#SliceThickness", + "Slice Timing" : "https://bids.neuroimaging.io#SliceTiming", + "Spacing Between Slices" : "https://bids.neuroimaging.io#SpacingBetweenSlices", + "Total Readout Time" : "https://bids.neuroimaging.io#TotalReadoutTime", + "Manufacturer" : "https://bids.neuroimaging.io#Manufacturer", + "Manufacturers Model Name" : "https://bids.neuroimaging.io#ManufacturersModelName", + "Magnetic Field Strength" : "https://bids.neuroimaging.io#MagneticFieldStrength", + "Device Serial Number" : "https://bids.neuroimaging.io#DeviceSerialNumber", + "Software Versions" : "https://bids.neuroimaging.io#SoftwareVersions", + "MR Transmit Coil Sequence" : "https://bids.neuroimaging.io#MRTransmitCoilSequence", + "Partial Fourier" : "https://bids.neuroimaging.io#PartialFourier", + "Partial Fourier Direction" : "https://bids.neuroimaging.io#PartialFourierDirection", + "Effective Echo Spacing" : "https://bids.neuroimaging.io#EffectiveEchoSpacing", + "Receive Coil Active Elements" : "https://bids.neuroimaging.io#ReceiveCoilActiveElements", + "Non Linear Gradient Correction" : "https://bids.neuroimaging.io#NonLinearGradientCorrection", + "Dwell Time" : "https://bids.neuroimaging.io#DwellTime", + "Multiband Acceleration Factor" : "https://bids.neuroimaging.io#MultibandAccelerationFactor", + "Anatomical Landmark Coordinates" : "https://bids.neuroimaging.io#AnatomicalLandmarkCoordinates", + "Institution Name" : "https://bids.neuroimaging.io#InstitutionName", + "Institution Address" : "https://bids.neuroimaging.io#Institution Address", + "Institutional Department Name" : "https://bids.neuroimaging.io#InstitutionalDepartmentName", + "Author" : "https://bids.neuroimaging.io#Author", + "License" : "https://bids.neuroimaging.io#License", + "Description" : "https://bids.neuroimaging.io#Description", + "Age" : "https://bids.neuroimaging.io#Age", + "Ethnicity" : "https://bids.neuroimaging.io#Ethnicity", + "Family Name" : "https://bids.neuroimaging.io#FamilyName", + "Handedness" : "https://bids.neuroimaging.io#Handedness", + "Sex" : "https://bids.neuroimaging.io#Sex", + "Race" : "https://bids.neuroimaging.io#Race", + "Participant Id" : "https://bids.neuroimaging.io#ParticipantId", + "Sampling Frequency" : "https://bids.neuroimaging.io#SamplingFrequency", + "Trigger" : "https://bids.neuroimaging.io#Trigger", + "Angiography" : "https://bids.neuroimaging.io#Angiography", + "Combined Proton Density/T2" : "https://bids.neuroimaging.io#CombinedProtonDensityT2", + "FLAIR" : "https://bids.neuroimaging.io#FLAIR", + "FLASH" : "https://bids.neuroimaging.io#FLASH", + "Proton Density Map" : "https://bids.neuroimaging.io#ProtonDensityMap", + "T1 Map" : "https://bids.neuroimaging.io#T1Map", + "T1 Rho Map" : "https://bids.neuroimaging.io#T1RhoMap", + "T2 Map" : "https://bids.neuroimaging.io#T2Map", + "Inplane T1" : "https://bids.neuroimaging.io#InplaneT1", + "Inplane T2" : "https://bids.neuroimaging.io#InplaneT2", + "Duration" : "https://bids.neuroimaging.io#Duration", + "Onset" : "https://bids.neuroimaging.io#Onset", + "Sample" : "https://bids.neuroimaging.io#Sample", + "Response Time" : "https://bids.neuroimaging.io#ResponseTime", + "Value" : "https://bids.neuroimaging.io#Value", + "Trial Type" : "https://bids.neuroimaging.io#TrialType", + "Acquisition Duration" : "https://bids.neuroimaging.io#AcquisitionDuration", + "BOLD" : "https://bids.neuroimaging.io#BOLD", + "Delay After Trigger" : "https://bids.neuroimaging.io#DelayAfterTrigger", + "Delay Time" : "https://bids.neuroimaging.io#DelayTime", + "Instructions" : "https://bids.neuroimaging.io#Instructions", + "Number Of Volumes Discarded By Scanner" : "https://bids.neuroimaging.io#NumberOfVolumesDiscardedByScanner", + "Number Of Volumes Discarded By User" : "https://bids.neuroimaging.io#NumberOfVolumesDiscardedByUser", + "Phase" : "https://bids.neuroimaging.io#Phase", + "Repetition Time" : "https://bids.neuroimaging.io#RepetitionTime", + "Task Description" : "https://bids.neuroimaging.io#TaskDescription", + "Task Name" : "https://bids.neuroimaging.io#TaskName", + "Volume Timing" : "https://bids.neuroimaging.io#VolumeTiming", + "CBV" : "https://bids.neuroimaging.io#CBV", + "Cog Atlas ID" : "https://bids.neuroimaging.io#CogAtlasID", + "CogPO ID" : "https://bids.neuroimaging.io#CogPOID", + } +} + From 9f9e3f37afed65e296ddf8f25b102bd853bcda84 Mon Sep 17 00:00:00 2001 From: natachaperez Date: Wed, 7 Aug 2019 15:51:35 +0200 Subject: [PATCH 2/7] json context for bids documents only --- bids_context.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bids_context.json b/bids_context.json index 1194dfd..f19de60 100644 --- a/bids_context.json +++ b/bids_context.json @@ -1,3 +1,9 @@ +""" +WIP: json document to store bids attributes from bids spec +I will add bids attributes which are commons with some +nidm-experiments attributes +""" + { "@context": { "@version": 1.1, From 78ae846c11950566328c77c16b8f50b9d0907b95 Mon Sep 17 00:00:00 2001 From: natachaperez <51317064+natachaperez@users.noreply.github.com> Date: Wed, 7 Aug 2019 16:17:34 +0200 Subject: [PATCH 3/7] Wrap up BIDS/ NIDM-Experiment In this jupyter notebook, you will find documents illustrating: - How to go from a BIDS dataset to a JSON-LD BIDS dataset - How to go from a BIDS dataset to a JSON-LD NIDM-experiment dataset Moreover, you will find a report on how close/different those representations are, what could be done in future work to make them closer and what are the issues (if any) that seem much harder to solve. --- BIDS to JSON-LD.ipynb | 99 ++++++++++++ BIDS to NIDM-E.ipynb | 360 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 459 insertions(+) create mode 100644 BIDS to JSON-LD.ipynb create mode 100644 BIDS to NIDM-E.ipynb diff --git a/BIDS to JSON-LD.ipynb b/BIDS to JSON-LD.ipynb new file mode 100644 index 0000000..6e9e6d8 --- /dev/null +++ b/BIDS to JSON-LD.ipynb @@ -0,0 +1,99 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# BIDS dataset to a JSON-LD BIDS dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This document illustrates an example showing how to go from a bids dataset to a JSON-LD BIDS dataset. More specifically, one wants to : \n", + "- add gently context into existing json files : T1w.json, taskxxx.json\n", + "- create an other JSON file with attached JSON-LD context to store metadata only present at the root of the BIDS directory (~/nidm-specs/nidm/nidm-experiment/terms/nidme.json)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It is a great milestone to foster the complementary of both standards with the mapping from JSON attributes in BIDS to NIDM-experiment URLs. Indeed, NIDM adds to BIDS the capability to track provenance and disambiguate experimental details and data elements. Moreover, the definition of attributes is more complete and relevant in NIDM. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "1st step : to review existing JSON files with the metadata types in BIDS and compare them to the NIDM-Experiment data. \n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "ename": "FileNotFoundError", + "evalue": "[Errno 2] File b'/Downloads/Report_BIDS2NIDME.csv' does not exist: b'/Downloads/Report_BIDS2NIDME.csv'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mpath\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'~'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'/Downloads'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Report_BIDS2NIDME.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6\u001b[0m \u001b[0mprint\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda2/envs/pynidm371/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m 700\u001b[0m skip_blank_lines=skip_blank_lines)\n\u001b[1;32m 701\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 702\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 703\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 704\u001b[0m \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda2/envs/pynidm371/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 427\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 428\u001b[0m \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 429\u001b[0;31m \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 430\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 431\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda2/envs/pynidm371/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m 893\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 894\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 895\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 896\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 897\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda2/envs/pynidm371/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m 1120\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1121\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1122\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1123\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1124\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'python'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda2/envs/pynidm371/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m 1851\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'usecols'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1852\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1853\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1854\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1855\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n", + "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] File b'/Downloads/Report_BIDS2NIDME.csv' does not exist: b'/Downloads/Report_BIDS2NIDME.csv'" + ] + } + ], + "source": [ + "import os \n", + "import pandas as pd \n", + "\n", + "path = os.path.join('/Downloads','Report_BIDS2NIDME.csv')\n", + "df = pd.read_csv(path)\n", + "print (df)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/BIDS to NIDM-E.ipynb b/BIDS to NIDM-E.ipynb new file mode 100644 index 0000000..c89623c --- /dev/null +++ b/BIDS to NIDM-E.ipynb @@ -0,0 +1,360 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# BIDS dataset to a JSON-LD NIDM-Experiment dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "(work under conda environment)\n", + "This document illustrates an example showing how to go from a BIDS dataset to a JSON-LD NIDM-experiment dataset.\n", + "We pick our bids dataset from DataLad where the hierarchy of folders follows the bids standard. For this example, we import the ABIDE DataLad dataset from CMU_a site : http://datasets.datalad.org/?dir=/abide/RawDataBIDS/CMU_a " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "conda install -c conda-forge git-annex\n", + "pip install datalad \n", + "datalad install ///abide/RawDataBIDS/CMU_a" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We use the BIDSMRI2NIDM.py file from PyNIDM/nidm/experiment/tools.\n", + "This program will convert a BIDS MRI dataset to a NIDM-Experiment RDF document. It will parse phenotype information and simply store variables/values and link to the associated json data dictionary file.\n", + "Argument used: \n", + "- -d [root directory of BIDS dataset]\n", + "- -jsonld [If flag set, output is json-ld not TURTLE]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "python ~/PyNIDM/nidm/experiment/tools/BIDSMRI2NIDM.py -d ~/bids_dataset/CMU_a/" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Datalad hosted abide dataset doesn’t have participants.tsv files and doesn’t have a dataset_description.json file so it is really not even valid BIDS. The participants.tsv file is optional but we must have a datasaet_description.json file. \n", + "One has to create some basic dataset_description.json files for each abide and adhd200 site so one could run bidsmri2nidm on it and add it in the github repo.\n", + "For instance see https://bids-specification.readthedocs.io/en/stable/03-modality-agnostic-files.html to put the required fields in the dataset_description.json file. \n", + "Below our dataset_description.json file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "{\n", + "\t\"Name\": \"ABIDE dataset CMU_a Site\",\n", + "\t\"BIDSVersion\": \"1.0.1\",\n", + "\t\"License\": \"CC BY-SA 4.0\"\n", + "\t\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "One must also parse other json files available at the root of the bids directory: T1w.json and taskxxx.json files. \n", + "if key from T1w.json file or taskxxx.json file is mapped to term in BIDS_Constants.py then add to NIDM object. \n", + "See function bidsmri2project(directory, args):\n", + "- for dataset_description.json file " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def bidsmri2project(directory, args):\n", + " #Parse dataset_description.json file in BIDS directory\n", + " if (os.path.isdir(os.path.join(directory))):\n", + " try:\n", + " with open(os.path.join(directory,'dataset_description.json')) as data_file:\n", + " dataset = json.load(data_file)\n", + " except OSError:\n", + " logging.critical(\"Cannot find dataset_description.json file which is required in the BIDS spec\")\n", + " exit(\"-1\")\n", + " else:\n", + " logging.critical(\"Error: BIDS directory %s does not exist!\" %os.path.join(directory))\n", + " exit(\"-1\")\n", + "\n", + " #create project / nidm-exp doc\n", + " project = Project()\n", + "\n", + " #add various attributes if they exist in BIDS dataset\n", + " for key in dataset:\n", + " #if key from dataset_description file is mapped to term in BIDS_Constants.py then add to NIDM object\n", + " if key in BIDS_Constants.dataset_description:\n", + " if type(dataset[key]) is list:\n", + " project.add_attributes({BIDS_Constants.dataset_description[key]:\"\".join(dataset[key])})\n", + " else:\n", + " project.add_attributes({BIDS_Constants.dataset_description[key]:dataset[key]})\n", + " #add absolute location of BIDS directory on disk for later finding of files which are stored relatively in NIDM document\n", + " project.add_attributes({Constants.PROV['Location']:directory})\n", + "\n", + " #get BIDS layout\n", + " bids_layout = BIDSLayout(directory) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- For T1w.json file " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + " if file_tpl.entities['datatype']=='anat':\n", + " #do something with anatomicals\n", + " acq_obj = MRObject(acq)\n", + " #add image contrast type\n", + " if file_tpl.entities['suffix'] in BIDS_Constants.scans:\n", + " acq_obj.add_attributes({Constants.NIDM_IMAGE_CONTRAST_TYPE:BIDS_Constants.scans[file_tpl.entities['suffix']]})\n", + " else:\n", + " logging.info(\"WARNING: No matching image contrast type found in BIDS_Constants.py for %s\" % file_tpl.entities['suffix'])\n", + "\n", + " #add image usage type\n", + " if file_tpl.entities['datatype'] in BIDS_Constants.scans:\n", + " acq_obj.add_attributes({Constants.NIDM_IMAGE_USAGE_TYPE:BIDS_Constants.scans[file_tpl.entities['datatype']]})\n", + " else:\n", + " logging.info(\"WARNING: No matching image usage type found in BIDS_Constants.py for %s\" % file_tpl.entities['datatype'])\n", + " #add file link\n", + " #make relative link to\n", + " acq_obj.add_attributes({Constants.NIDM_FILENAME:getRelPathToBIDS(join(file_tpl.dirname,file_tpl.filename), directory)})\n", + "\n", + " #add sha512 sum\n", + " if isfile(join(directory,file_tpl.dirname,file_tpl.filename)):\n", + " acq_obj.add_attributes({Constants.CRYPTO_SHA512:getsha512(join(directory,file_tpl.dirname,file_tpl.filename))})\n", + " else:\n", + " logging.info(\"WARNINGL file %s doesn't exist! No SHA512 sum stored in NIDM files...\" %join(directory,file_tpl.dirname,file_tpl.filename))\n", + " #get associated JSON file if exists\n", + " #There is T1w.json file with information \n", + " json_data = (bids_layout.get(suffix=file_tpl.entities['suffix'],subject=subject_id))[0].metadata\n", + " if len(json_data.info)>0:\n", + " for key in json_data.info.items():\n", + " if key in BIDS_Constants.json_keys:\n", + " if type(json_data.info[key]) is list:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key.replace(\" \", \"_\")]:''.join(str(e) for e in json_data.info[key])})\n", + " else:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key.replace(\" \", \"_\")]:json_data.info[key]})\n", + " \n", + " #Parse T1w.json file in BIDS directory to add the attributes contained inside\n", + " if (os.path.isdir(os.path.join(directory))):\n", + " try:\n", + " with open(os.path.join(directory,'T1w.json')) as data_file:\n", + " dataset = json.load(data_file)\n", + " except OSError:\n", + " logging.critical(\"Cannot find T1w.json file which is required in the BIDS spec\")\n", + " exit(\"-1\")\n", + " else:\n", + " logging.critical(\"Error: BIDS directory %s does not exist!\" %os.path.join(directory))\n", + " exit(\"-1\")\n", + "\n", + " #add various attributes if they exist in BIDS dataset\n", + " for key in dataset:\n", + " #if key from T1w.json file is mapped to term in BIDS_Constants.py then add to NIDM object\n", + " if key in BIDS_Constants.json_keys:\n", + " if type(dataset[key]) is list:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key]:\"\".join(dataset[key])})\n", + " else:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key]:dataset[key]}) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- for taskxxx.json file " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + " \n", + " elif file_tpl.entities['datatype'] == 'func':\n", + " #do something with functionals\n", + " acq_obj = MRObject(acq)\n", + " #add image contrast type\n", + " if file_tpl.entities['suffix'] in BIDS_Constants.scans:\n", + " acq_obj.add_attributes({Constants.NIDM_IMAGE_CONTRAST_TYPE:BIDS_Constants.scans[file_tpl.entities['suffix']]})\n", + " else:\n", + " logging.info(\"WARNING: No matching image contrast type found in BIDS_Constants.py for %s\" % file_tpl.entities['suffix'])\n", + "\n", + " #add image usage type\n", + " if file_tpl.entities['datatype'] in BIDS_Constants.scans:\n", + " acq_obj.add_attributes({Constants.NIDM_IMAGE_USAGE_TYPE:BIDS_Constants.scans[file_tpl.entities['datatype']]})\n", + " else:\n", + " logging.info(\"WARNING: No matching image usage type found in BIDS_Constants.py for %s\" % file_tpl.entities['datatype'])\n", + " #make relative link to\n", + " acq_obj.add_attributes({Constants.NIDM_FILENAME:getRelPathToBIDS(join(file_tpl.dirname,file_tpl.filename), directory)})\n", + " #add sha512 sum\n", + " if isfile(join(directory,file_tpl.dirname,file_tpl.filename)):\n", + " acq_obj.add_attributes({Constants.CRYPTO_SHA512:getsha512(join(directory,file_tpl.dirname,file_tpl.filename))})\n", + " else:\n", + " logging.info(\"WARNINGL file %s doesn't exist! No SHA512 sum stored in NIDM files...\" %join(directory,file_tpl.dirname,file_tpl.filename))\n", + "\n", + " if 'run' in file_tpl.entities:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[\"run\"]:file_tpl.entities['run']})\n", + "\n", + " #get associated JSON file if exists\n", + " json_data = (bids_layout.get(suffix=file_tpl.entities['suffix'],subject=subject_id))[0].metadata\n", + "\n", + " if len(json_data.info)>0:\n", + " for key in json_data.info.items():\n", + " if key in BIDS_Constants.json_keys:\n", + " if type(json_data.info[key]) is list:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key.replace(\" \", \"_\")]:''.join(str(e) for e in json_data.info[key])})\n", + " else:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key.replace(\" \", \"_\")]:json_data.info[key]})\n", + " #get associated events TSV file\n", + " if 'run' in file_tpl.entities:\n", + " events_file = bids_layout.get(subject=subject_id, extensions=['.tsv'],modality=file_tpl.entities['datatype'],task=file_tpl.entities['task'],run=file_tpl.entities['run'])\n", + " else:\n", + " events_file = bids_layout.get(subject=subject_id, extensions=['.tsv'],modality=file_tpl.entities['datatype'],task=file_tpl.entities['task'])\n", + " #if there is an events file then this is task-based so create an acquisition object for the task file and link\n", + " if events_file:\n", + " #for now create acquisition object and link it to the associated scan\n", + " events_obj = AcquisitionObject(acq)\n", + " #add prov type, task name as prov:label, and link to filename of events file\n", + "\n", + " events_obj.add_attributes({PROV_TYPE:Constants.NIDM_MRI_BOLD_EVENTS,BIDS_Constants.json_keys[\"TaskName\"]: json_data[\"TaskName\"], Constants.NIDM_FILENAME:getRelPathToBIDS(events_file[0].filename, directory)})\n", + " #link it to appropriate MR acquisition entity\n", + " events_obj.wasAttributedTo(acq_obj)\n", + " \n", + " #Parse task-rest_bold.json file in BIDS directory to add the attributes contained inside\n", + " if (os.path.isdir(os.path.join(directory))):\n", + " try:\n", + " with open(os.path.join(directory,'task-rest_bold.json')) as data_file:\n", + " dataset = json.load(data_file)\n", + " except OSError:\n", + " logging.critical(\"Cannot find task-rest_bold.json file which is required in the BIDS spec\")\n", + " exit(\"-1\")\n", + " else:\n", + " logging.critical(\"Error: BIDS directory %s does not exist!\" %os.path.join(directory))\n", + " exit(\"-1\")\n", + "\n", + " #add various attributes if they exist in BIDS dataset\n", + " for key in dataset:\n", + " #if key from task-rest_bold.json file is mapped to term in BIDS_Constants.py then add to NIDM object\n", + " if key in BIDS_Constants.json_keys:\n", + " if type(dataset[key]) is list:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key]:\",\".join(map(str,dataset[key]))})\n", + " else:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key]:dataset[key]}) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The idea behind is to create acquisition objects for each scan for each subject using the information available in the bids directory. \n", + "Below, an extract of the nidme.json document created with the acquisition informations for one subject: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + " {\n", + " \"@id\": \"nidm:8edfb514-b761-11e9-8f81-e1efbef5a165\",\n", + " \"@type\": [\n", + " \"Entity\",\n", + " \"AcquisitionObject\"\n", + " ],\n", + " \"crypto:sha512\": \"840affc90aeda40601a48f8d5cd3421fb73e0cbecc3111c4a3194aef74a716b732e7649b8b36d55f0b5559a88531ac10acd00bf27a95d52b59da78a88b7eeb8f\",\n", + " \"dicom:AcquisitionMatrix\": \"256x256\",\n", + " \"dicom:EchoTime\": 0.00248,\n", + " \"dicom:FlipAngle\": {\n", + " \"@type\": \"xsd:int\",\n", + " \"@value\": \"8\"\n", + " },\n", + " \"dicom:InversionTime\": 1.1,\n", + " \"dicom:MagneticFieldStrength\": {\n", + " \"@type\": \"xsd:int\",\n", + " \"@value\": \"3\"\n", + " },\n", + " \"dicom:PixelBandwidth\": 170.0,\n", + " \"dicom:RepetitionTime\": 1.87,\n", + " \"dicom:ScanningSequence\": \"MPRAGE\",\n", + " \"nidm:PhaseEncodingDirection\": \"j-\",\n", + " \"hadAcquisitionModality\": {\n", + " \"@id\": \"nidm:MagneticResonanceImaging\"\n", + " },\n", + " \"hadImageContrastType\": {\n", + " \"@id\": \"nidm:T1Weighted\"\n", + " },\n", + " \"HadImageUsageType\": {\n", + " \"@id\": \"nidm:Anatomical\"\n", + " },\n", + " \"filename\": \"/udd/nperez/sub-0050656/anat/sub-0050656_T1w.nii.gz\",\n", + " \"prov:wasGeneratedBy\": {\n", + " \"@id\": \"nidm:8edfacae-b761-11e9-8f81-e1efbef5a165\"\n", + " }\n", + " }," + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, some limitations appear: \n", + "- we only extract the data that map to BIDS_Constants.py and not all bids attributes are referenced in this document \n", + "- for bval and bvec files, what to do with those?" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 78be24c5bfd6f9c42804715cbfbfd311eedc4dba Mon Sep 17 00:00:00 2001 From: natachaperez <51317064+natachaperez@users.noreply.github.com> Date: Thu, 8 Aug 2019 09:38:43 +0200 Subject: [PATCH 4/7] Wrap up BIDS NIDM-Experiment These documents illustrate an example showing: - How to go from a BIDS dataset to a JSON-LD BIDS dataset - How to go from a BIDS dataset to a JSON-LD NIDM-experiment dataset We will further discuss how close/different those representations are, what could be done in future work to make them closer and what are the issues (if any) that seem much harder to solve. From 8f1ab0ceb117de6e7f422ac6b7d64275e7fd6b32 Mon Sep 17 00:00:00 2001 From: natachaperez <51317064+natachaperez@users.noreply.github.com> Date: Thu, 8 Aug 2019 09:40:07 +0200 Subject: [PATCH 5/7] Delete BIDS to JSON-LD.ipynb --- BIDS to JSON-LD.ipynb | 99 ------------------------------------------- 1 file changed, 99 deletions(-) delete mode 100644 BIDS to JSON-LD.ipynb diff --git a/BIDS to JSON-LD.ipynb b/BIDS to JSON-LD.ipynb deleted file mode 100644 index 6e9e6d8..0000000 --- a/BIDS to JSON-LD.ipynb +++ /dev/null @@ -1,99 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# BIDS dataset to a JSON-LD BIDS dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "This document illustrates an example showing how to go from a bids dataset to a JSON-LD BIDS dataset. More specifically, one wants to : \n", - "- add gently context into existing json files : T1w.json, taskxxx.json\n", - "- create an other JSON file with attached JSON-LD context to store metadata only present at the root of the BIDS directory (~/nidm-specs/nidm/nidm-experiment/terms/nidme.json)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "It is a great milestone to foster the complementary of both standards with the mapping from JSON attributes in BIDS to NIDM-experiment URLs. Indeed, NIDM adds to BIDS the capability to track provenance and disambiguate experimental details and data elements. Moreover, the definition of attributes is more complete and relevant in NIDM. " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "1st step : to review existing JSON files with the metadata types in BIDS and compare them to the NIDM-Experiment data. \n" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "scrolled": true - }, - "outputs": [ - { - "ename": "FileNotFoundError", - "evalue": "[Errno 2] File b'/Downloads/Report_BIDS2NIDME.csv' does not exist: b'/Downloads/Report_BIDS2NIDME.csv'", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mpath\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'~'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'/Downloads'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Report_BIDS2NIDME.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6\u001b[0m \u001b[0mprint\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda2/envs/pynidm371/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m 700\u001b[0m skip_blank_lines=skip_blank_lines)\n\u001b[1;32m 701\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 702\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 703\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 704\u001b[0m \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda2/envs/pynidm371/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 427\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 428\u001b[0m \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 429\u001b[0;31m \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 430\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 431\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda2/envs/pynidm371/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m 893\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 894\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 895\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 896\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 897\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda2/envs/pynidm371/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m 1120\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1121\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1122\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1123\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1124\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'python'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m~/miniconda2/envs/pynidm371/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m 1851\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'usecols'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1852\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1853\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1854\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1855\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n", - "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n", - "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] File b'/Downloads/Report_BIDS2NIDME.csv' does not exist: b'/Downloads/Report_BIDS2NIDME.csv'" - ] - } - ], - "source": [ - "import os \n", - "import pandas as pd \n", - "\n", - "path = os.path.join('/Downloads','Report_BIDS2NIDME.csv')\n", - "df = pd.read_csv(path)\n", - "print (df)\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 042ded46b829879ecfcf0377e4db4dbc829405e5 Mon Sep 17 00:00:00 2001 From: natachaperez <51317064+natachaperez@users.noreply.github.com> Date: Thu, 8 Aug 2019 09:40:16 +0200 Subject: [PATCH 6/7] Delete BIDS to NIDM-E.ipynb --- BIDS to NIDM-E.ipynb | 360 ------------------------------------------- 1 file changed, 360 deletions(-) delete mode 100644 BIDS to NIDM-E.ipynb diff --git a/BIDS to NIDM-E.ipynb b/BIDS to NIDM-E.ipynb deleted file mode 100644 index c89623c..0000000 --- a/BIDS to NIDM-E.ipynb +++ /dev/null @@ -1,360 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# BIDS dataset to a JSON-LD NIDM-Experiment dataset" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "(work under conda environment)\n", - "This document illustrates an example showing how to go from a BIDS dataset to a JSON-LD NIDM-experiment dataset.\n", - "We pick our bids dataset from DataLad where the hierarchy of folders follows the bids standard. For this example, we import the ABIDE DataLad dataset from CMU_a site : http://datasets.datalad.org/?dir=/abide/RawDataBIDS/CMU_a " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "conda install -c conda-forge git-annex\n", - "pip install datalad \n", - "datalad install ///abide/RawDataBIDS/CMU_a" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "We use the BIDSMRI2NIDM.py file from PyNIDM/nidm/experiment/tools.\n", - "This program will convert a BIDS MRI dataset to a NIDM-Experiment RDF document. It will parse phenotype information and simply store variables/values and link to the associated json data dictionary file.\n", - "Argument used: \n", - "- -d [root directory of BIDS dataset]\n", - "- -jsonld [If flag set, output is json-ld not TURTLE]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "python ~/PyNIDM/nidm/experiment/tools/BIDSMRI2NIDM.py -d ~/bids_dataset/CMU_a/" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The Datalad hosted abide dataset doesn’t have participants.tsv files and doesn’t have a dataset_description.json file so it is really not even valid BIDS. The participants.tsv file is optional but we must have a datasaet_description.json file. \n", - "One has to create some basic dataset_description.json files for each abide and adhd200 site so one could run bidsmri2nidm on it and add it in the github repo.\n", - "For instance see https://bids-specification.readthedocs.io/en/stable/03-modality-agnostic-files.html to put the required fields in the dataset_description.json file. \n", - "Below our dataset_description.json file:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "{\n", - "\t\"Name\": \"ABIDE dataset CMU_a Site\",\n", - "\t\"BIDSVersion\": \"1.0.1\",\n", - "\t\"License\": \"CC BY-SA 4.0\"\n", - "\t\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "One must also parse other json files available at the root of the bids directory: T1w.json and taskxxx.json files. \n", - "if key from T1w.json file or taskxxx.json file is mapped to term in BIDS_Constants.py then add to NIDM object. \n", - "See function bidsmri2project(directory, args):\n", - "- for dataset_description.json file " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def bidsmri2project(directory, args):\n", - " #Parse dataset_description.json file in BIDS directory\n", - " if (os.path.isdir(os.path.join(directory))):\n", - " try:\n", - " with open(os.path.join(directory,'dataset_description.json')) as data_file:\n", - " dataset = json.load(data_file)\n", - " except OSError:\n", - " logging.critical(\"Cannot find dataset_description.json file which is required in the BIDS spec\")\n", - " exit(\"-1\")\n", - " else:\n", - " logging.critical(\"Error: BIDS directory %s does not exist!\" %os.path.join(directory))\n", - " exit(\"-1\")\n", - "\n", - " #create project / nidm-exp doc\n", - " project = Project()\n", - "\n", - " #add various attributes if they exist in BIDS dataset\n", - " for key in dataset:\n", - " #if key from dataset_description file is mapped to term in BIDS_Constants.py then add to NIDM object\n", - " if key in BIDS_Constants.dataset_description:\n", - " if type(dataset[key]) is list:\n", - " project.add_attributes({BIDS_Constants.dataset_description[key]:\"\".join(dataset[key])})\n", - " else:\n", - " project.add_attributes({BIDS_Constants.dataset_description[key]:dataset[key]})\n", - " #add absolute location of BIDS directory on disk for later finding of files which are stored relatively in NIDM document\n", - " project.add_attributes({Constants.PROV['Location']:directory})\n", - "\n", - " #get BIDS layout\n", - " bids_layout = BIDSLayout(directory) " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- For T1w.json file " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - " if file_tpl.entities['datatype']=='anat':\n", - " #do something with anatomicals\n", - " acq_obj = MRObject(acq)\n", - " #add image contrast type\n", - " if file_tpl.entities['suffix'] in BIDS_Constants.scans:\n", - " acq_obj.add_attributes({Constants.NIDM_IMAGE_CONTRAST_TYPE:BIDS_Constants.scans[file_tpl.entities['suffix']]})\n", - " else:\n", - " logging.info(\"WARNING: No matching image contrast type found in BIDS_Constants.py for %s\" % file_tpl.entities['suffix'])\n", - "\n", - " #add image usage type\n", - " if file_tpl.entities['datatype'] in BIDS_Constants.scans:\n", - " acq_obj.add_attributes({Constants.NIDM_IMAGE_USAGE_TYPE:BIDS_Constants.scans[file_tpl.entities['datatype']]})\n", - " else:\n", - " logging.info(\"WARNING: No matching image usage type found in BIDS_Constants.py for %s\" % file_tpl.entities['datatype'])\n", - " #add file link\n", - " #make relative link to\n", - " acq_obj.add_attributes({Constants.NIDM_FILENAME:getRelPathToBIDS(join(file_tpl.dirname,file_tpl.filename), directory)})\n", - "\n", - " #add sha512 sum\n", - " if isfile(join(directory,file_tpl.dirname,file_tpl.filename)):\n", - " acq_obj.add_attributes({Constants.CRYPTO_SHA512:getsha512(join(directory,file_tpl.dirname,file_tpl.filename))})\n", - " else:\n", - " logging.info(\"WARNINGL file %s doesn't exist! No SHA512 sum stored in NIDM files...\" %join(directory,file_tpl.dirname,file_tpl.filename))\n", - " #get associated JSON file if exists\n", - " #There is T1w.json file with information \n", - " json_data = (bids_layout.get(suffix=file_tpl.entities['suffix'],subject=subject_id))[0].metadata\n", - " if len(json_data.info)>0:\n", - " for key in json_data.info.items():\n", - " if key in BIDS_Constants.json_keys:\n", - " if type(json_data.info[key]) is list:\n", - " acq_obj.add_attributes({BIDS_Constants.json_keys[key.replace(\" \", \"_\")]:''.join(str(e) for e in json_data.info[key])})\n", - " else:\n", - " acq_obj.add_attributes({BIDS_Constants.json_keys[key.replace(\" \", \"_\")]:json_data.info[key]})\n", - " \n", - " #Parse T1w.json file in BIDS directory to add the attributes contained inside\n", - " if (os.path.isdir(os.path.join(directory))):\n", - " try:\n", - " with open(os.path.join(directory,'T1w.json')) as data_file:\n", - " dataset = json.load(data_file)\n", - " except OSError:\n", - " logging.critical(\"Cannot find T1w.json file which is required in the BIDS spec\")\n", - " exit(\"-1\")\n", - " else:\n", - " logging.critical(\"Error: BIDS directory %s does not exist!\" %os.path.join(directory))\n", - " exit(\"-1\")\n", - "\n", - " #add various attributes if they exist in BIDS dataset\n", - " for key in dataset:\n", - " #if key from T1w.json file is mapped to term in BIDS_Constants.py then add to NIDM object\n", - " if key in BIDS_Constants.json_keys:\n", - " if type(dataset[key]) is list:\n", - " acq_obj.add_attributes({BIDS_Constants.json_keys[key]:\"\".join(dataset[key])})\n", - " else:\n", - " acq_obj.add_attributes({BIDS_Constants.json_keys[key]:dataset[key]}) " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "- for taskxxx.json file " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - " \n", - " elif file_tpl.entities['datatype'] == 'func':\n", - " #do something with functionals\n", - " acq_obj = MRObject(acq)\n", - " #add image contrast type\n", - " if file_tpl.entities['suffix'] in BIDS_Constants.scans:\n", - " acq_obj.add_attributes({Constants.NIDM_IMAGE_CONTRAST_TYPE:BIDS_Constants.scans[file_tpl.entities['suffix']]})\n", - " else:\n", - " logging.info(\"WARNING: No matching image contrast type found in BIDS_Constants.py for %s\" % file_tpl.entities['suffix'])\n", - "\n", - " #add image usage type\n", - " if file_tpl.entities['datatype'] in BIDS_Constants.scans:\n", - " acq_obj.add_attributes({Constants.NIDM_IMAGE_USAGE_TYPE:BIDS_Constants.scans[file_tpl.entities['datatype']]})\n", - " else:\n", - " logging.info(\"WARNING: No matching image usage type found in BIDS_Constants.py for %s\" % file_tpl.entities['datatype'])\n", - " #make relative link to\n", - " acq_obj.add_attributes({Constants.NIDM_FILENAME:getRelPathToBIDS(join(file_tpl.dirname,file_tpl.filename), directory)})\n", - " #add sha512 sum\n", - " if isfile(join(directory,file_tpl.dirname,file_tpl.filename)):\n", - " acq_obj.add_attributes({Constants.CRYPTO_SHA512:getsha512(join(directory,file_tpl.dirname,file_tpl.filename))})\n", - " else:\n", - " logging.info(\"WARNINGL file %s doesn't exist! No SHA512 sum stored in NIDM files...\" %join(directory,file_tpl.dirname,file_tpl.filename))\n", - "\n", - " if 'run' in file_tpl.entities:\n", - " acq_obj.add_attributes({BIDS_Constants.json_keys[\"run\"]:file_tpl.entities['run']})\n", - "\n", - " #get associated JSON file if exists\n", - " json_data = (bids_layout.get(suffix=file_tpl.entities['suffix'],subject=subject_id))[0].metadata\n", - "\n", - " if len(json_data.info)>0:\n", - " for key in json_data.info.items():\n", - " if key in BIDS_Constants.json_keys:\n", - " if type(json_data.info[key]) is list:\n", - " acq_obj.add_attributes({BIDS_Constants.json_keys[key.replace(\" \", \"_\")]:''.join(str(e) for e in json_data.info[key])})\n", - " else:\n", - " acq_obj.add_attributes({BIDS_Constants.json_keys[key.replace(\" \", \"_\")]:json_data.info[key]})\n", - " #get associated events TSV file\n", - " if 'run' in file_tpl.entities:\n", - " events_file = bids_layout.get(subject=subject_id, extensions=['.tsv'],modality=file_tpl.entities['datatype'],task=file_tpl.entities['task'],run=file_tpl.entities['run'])\n", - " else:\n", - " events_file = bids_layout.get(subject=subject_id, extensions=['.tsv'],modality=file_tpl.entities['datatype'],task=file_tpl.entities['task'])\n", - " #if there is an events file then this is task-based so create an acquisition object for the task file and link\n", - " if events_file:\n", - " #for now create acquisition object and link it to the associated scan\n", - " events_obj = AcquisitionObject(acq)\n", - " #add prov type, task name as prov:label, and link to filename of events file\n", - "\n", - " events_obj.add_attributes({PROV_TYPE:Constants.NIDM_MRI_BOLD_EVENTS,BIDS_Constants.json_keys[\"TaskName\"]: json_data[\"TaskName\"], Constants.NIDM_FILENAME:getRelPathToBIDS(events_file[0].filename, directory)})\n", - " #link it to appropriate MR acquisition entity\n", - " events_obj.wasAttributedTo(acq_obj)\n", - " \n", - " #Parse task-rest_bold.json file in BIDS directory to add the attributes contained inside\n", - " if (os.path.isdir(os.path.join(directory))):\n", - " try:\n", - " with open(os.path.join(directory,'task-rest_bold.json')) as data_file:\n", - " dataset = json.load(data_file)\n", - " except OSError:\n", - " logging.critical(\"Cannot find task-rest_bold.json file which is required in the BIDS spec\")\n", - " exit(\"-1\")\n", - " else:\n", - " logging.critical(\"Error: BIDS directory %s does not exist!\" %os.path.join(directory))\n", - " exit(\"-1\")\n", - "\n", - " #add various attributes if they exist in BIDS dataset\n", - " for key in dataset:\n", - " #if key from task-rest_bold.json file is mapped to term in BIDS_Constants.py then add to NIDM object\n", - " if key in BIDS_Constants.json_keys:\n", - " if type(dataset[key]) is list:\n", - " acq_obj.add_attributes({BIDS_Constants.json_keys[key]:\",\".join(map(str,dataset[key]))})\n", - " else:\n", - " acq_obj.add_attributes({BIDS_Constants.json_keys[key]:dataset[key]}) " - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The idea behind is to create acquisition objects for each scan for each subject using the information available in the bids directory. \n", - "Below, an extract of the nidme.json document created with the acquisition informations for one subject: " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - " {\n", - " \"@id\": \"nidm:8edfb514-b761-11e9-8f81-e1efbef5a165\",\n", - " \"@type\": [\n", - " \"Entity\",\n", - " \"AcquisitionObject\"\n", - " ],\n", - " \"crypto:sha512\": \"840affc90aeda40601a48f8d5cd3421fb73e0cbecc3111c4a3194aef74a716b732e7649b8b36d55f0b5559a88531ac10acd00bf27a95d52b59da78a88b7eeb8f\",\n", - " \"dicom:AcquisitionMatrix\": \"256x256\",\n", - " \"dicom:EchoTime\": 0.00248,\n", - " \"dicom:FlipAngle\": {\n", - " \"@type\": \"xsd:int\",\n", - " \"@value\": \"8\"\n", - " },\n", - " \"dicom:InversionTime\": 1.1,\n", - " \"dicom:MagneticFieldStrength\": {\n", - " \"@type\": \"xsd:int\",\n", - " \"@value\": \"3\"\n", - " },\n", - " \"dicom:PixelBandwidth\": 170.0,\n", - " \"dicom:RepetitionTime\": 1.87,\n", - " \"dicom:ScanningSequence\": \"MPRAGE\",\n", - " \"nidm:PhaseEncodingDirection\": \"j-\",\n", - " \"hadAcquisitionModality\": {\n", - " \"@id\": \"nidm:MagneticResonanceImaging\"\n", - " },\n", - " \"hadImageContrastType\": {\n", - " \"@id\": \"nidm:T1Weighted\"\n", - " },\n", - " \"HadImageUsageType\": {\n", - " \"@id\": \"nidm:Anatomical\"\n", - " },\n", - " \"filename\": \"/udd/nperez/sub-0050656/anat/sub-0050656_T1w.nii.gz\",\n", - " \"prov:wasGeneratedBy\": {\n", - " \"@id\": \"nidm:8edfacae-b761-11e9-8f81-e1efbef5a165\"\n", - " }\n", - " }," - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "However, some limitations appear: \n", - "- we only extract the data that map to BIDS_Constants.py and not all bids attributes are referenced in this document \n", - "- for bval and bvec files, what to do with those?" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 8cfc09a40962214e5d93ec382364c8cacab662f9 Mon Sep 17 00:00:00 2001 From: natachaperez Date: Thu, 8 Aug 2019 10:19:33 +0200 Subject: [PATCH 7/7] jupyter notebook --- BIDS2JSONLD.ipynb | 99 +++++++++++++ BIDS2NIDME.ipynb | 360 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 459 insertions(+) create mode 100644 BIDS2JSONLD.ipynb create mode 100644 BIDS2NIDME.ipynb diff --git a/BIDS2JSONLD.ipynb b/BIDS2JSONLD.ipynb new file mode 100644 index 0000000..6e9e6d8 --- /dev/null +++ b/BIDS2JSONLD.ipynb @@ -0,0 +1,99 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# BIDS dataset to a JSON-LD BIDS dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "This document illustrates an example showing how to go from a bids dataset to a JSON-LD BIDS dataset. More specifically, one wants to : \n", + "- add gently context into existing json files : T1w.json, taskxxx.json\n", + "- create an other JSON file with attached JSON-LD context to store metadata only present at the root of the BIDS directory (~/nidm-specs/nidm/nidm-experiment/terms/nidme.json)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "It is a great milestone to foster the complementary of both standards with the mapping from JSON attributes in BIDS to NIDM-experiment URLs. Indeed, NIDM adds to BIDS the capability to track provenance and disambiguate experimental details and data elements. Moreover, the definition of attributes is more complete and relevant in NIDM. " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "1st step : to review existing JSON files with the metadata types in BIDS and compare them to the NIDM-Experiment data. \n" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "ename": "FileNotFoundError", + "evalue": "[Errno 2] File b'/Downloads/Report_BIDS2NIDME.csv' does not exist: b'/Downloads/Report_BIDS2NIDME.csv'", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mFileNotFoundError\u001b[0m Traceback (most recent call last)", + "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 4\u001b[0m \u001b[0mpath\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'~'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'/Downloads'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'Report_BIDS2NIDME.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 6\u001b[0m \u001b[0mprint\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda2/envs/pynidm371/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, tupleize_cols, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m 700\u001b[0m skip_blank_lines=skip_blank_lines)\n\u001b[1;32m 701\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 702\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 703\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 704\u001b[0m \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda2/envs/pynidm371/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m 427\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 428\u001b[0m \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 429\u001b[0;31m \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 430\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 431\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda2/envs/pynidm371/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m 893\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'has_index_names'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 894\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 895\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 896\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 897\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda2/envs/pynidm371/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m 1120\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'c'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1121\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'c'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1122\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1123\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1124\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'python'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32m~/miniconda2/envs/pynidm371/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m 1851\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'usecols'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1852\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1853\u001b[0;31m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 1854\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 1855\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", + "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n", + "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n", + "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] File b'/Downloads/Report_BIDS2NIDME.csv' does not exist: b'/Downloads/Report_BIDS2NIDME.csv'" + ] + } + ], + "source": [ + "import os \n", + "import pandas as pd \n", + "\n", + "path = os.path.join('/Downloads','Report_BIDS2NIDME.csv')\n", + "df = pd.read_csv(path)\n", + "print (df)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/BIDS2NIDME.ipynb b/BIDS2NIDME.ipynb new file mode 100644 index 0000000..c89623c --- /dev/null +++ b/BIDS2NIDME.ipynb @@ -0,0 +1,360 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# BIDS dataset to a JSON-LD NIDM-Experiment dataset" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "(work under conda environment)\n", + "This document illustrates an example showing how to go from a BIDS dataset to a JSON-LD NIDM-experiment dataset.\n", + "We pick our bids dataset from DataLad where the hierarchy of folders follows the bids standard. For this example, we import the ABIDE DataLad dataset from CMU_a site : http://datasets.datalad.org/?dir=/abide/RawDataBIDS/CMU_a " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "conda install -c conda-forge git-annex\n", + "pip install datalad \n", + "datalad install ///abide/RawDataBIDS/CMU_a" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We use the BIDSMRI2NIDM.py file from PyNIDM/nidm/experiment/tools.\n", + "This program will convert a BIDS MRI dataset to a NIDM-Experiment RDF document. It will parse phenotype information and simply store variables/values and link to the associated json data dictionary file.\n", + "Argument used: \n", + "- -d [root directory of BIDS dataset]\n", + "- -jsonld [If flag set, output is json-ld not TURTLE]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "python ~/PyNIDM/nidm/experiment/tools/BIDSMRI2NIDM.py -d ~/bids_dataset/CMU_a/" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The Datalad hosted abide dataset doesn’t have participants.tsv files and doesn’t have a dataset_description.json file so it is really not even valid BIDS. The participants.tsv file is optional but we must have a datasaet_description.json file. \n", + "One has to create some basic dataset_description.json files for each abide and adhd200 site so one could run bidsmri2nidm on it and add it in the github repo.\n", + "For instance see https://bids-specification.readthedocs.io/en/stable/03-modality-agnostic-files.html to put the required fields in the dataset_description.json file. \n", + "Below our dataset_description.json file:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "{\n", + "\t\"Name\": \"ABIDE dataset CMU_a Site\",\n", + "\t\"BIDSVersion\": \"1.0.1\",\n", + "\t\"License\": \"CC BY-SA 4.0\"\n", + "\t\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "One must also parse other json files available at the root of the bids directory: T1w.json and taskxxx.json files. \n", + "if key from T1w.json file or taskxxx.json file is mapped to term in BIDS_Constants.py then add to NIDM object. \n", + "See function bidsmri2project(directory, args):\n", + "- for dataset_description.json file " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def bidsmri2project(directory, args):\n", + " #Parse dataset_description.json file in BIDS directory\n", + " if (os.path.isdir(os.path.join(directory))):\n", + " try:\n", + " with open(os.path.join(directory,'dataset_description.json')) as data_file:\n", + " dataset = json.load(data_file)\n", + " except OSError:\n", + " logging.critical(\"Cannot find dataset_description.json file which is required in the BIDS spec\")\n", + " exit(\"-1\")\n", + " else:\n", + " logging.critical(\"Error: BIDS directory %s does not exist!\" %os.path.join(directory))\n", + " exit(\"-1\")\n", + "\n", + " #create project / nidm-exp doc\n", + " project = Project()\n", + "\n", + " #add various attributes if they exist in BIDS dataset\n", + " for key in dataset:\n", + " #if key from dataset_description file is mapped to term in BIDS_Constants.py then add to NIDM object\n", + " if key in BIDS_Constants.dataset_description:\n", + " if type(dataset[key]) is list:\n", + " project.add_attributes({BIDS_Constants.dataset_description[key]:\"\".join(dataset[key])})\n", + " else:\n", + " project.add_attributes({BIDS_Constants.dataset_description[key]:dataset[key]})\n", + " #add absolute location of BIDS directory on disk for later finding of files which are stored relatively in NIDM document\n", + " project.add_attributes({Constants.PROV['Location']:directory})\n", + "\n", + " #get BIDS layout\n", + " bids_layout = BIDSLayout(directory) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- For T1w.json file " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + " if file_tpl.entities['datatype']=='anat':\n", + " #do something with anatomicals\n", + " acq_obj = MRObject(acq)\n", + " #add image contrast type\n", + " if file_tpl.entities['suffix'] in BIDS_Constants.scans:\n", + " acq_obj.add_attributes({Constants.NIDM_IMAGE_CONTRAST_TYPE:BIDS_Constants.scans[file_tpl.entities['suffix']]})\n", + " else:\n", + " logging.info(\"WARNING: No matching image contrast type found in BIDS_Constants.py for %s\" % file_tpl.entities['suffix'])\n", + "\n", + " #add image usage type\n", + " if file_tpl.entities['datatype'] in BIDS_Constants.scans:\n", + " acq_obj.add_attributes({Constants.NIDM_IMAGE_USAGE_TYPE:BIDS_Constants.scans[file_tpl.entities['datatype']]})\n", + " else:\n", + " logging.info(\"WARNING: No matching image usage type found in BIDS_Constants.py for %s\" % file_tpl.entities['datatype'])\n", + " #add file link\n", + " #make relative link to\n", + " acq_obj.add_attributes({Constants.NIDM_FILENAME:getRelPathToBIDS(join(file_tpl.dirname,file_tpl.filename), directory)})\n", + "\n", + " #add sha512 sum\n", + " if isfile(join(directory,file_tpl.dirname,file_tpl.filename)):\n", + " acq_obj.add_attributes({Constants.CRYPTO_SHA512:getsha512(join(directory,file_tpl.dirname,file_tpl.filename))})\n", + " else:\n", + " logging.info(\"WARNINGL file %s doesn't exist! No SHA512 sum stored in NIDM files...\" %join(directory,file_tpl.dirname,file_tpl.filename))\n", + " #get associated JSON file if exists\n", + " #There is T1w.json file with information \n", + " json_data = (bids_layout.get(suffix=file_tpl.entities['suffix'],subject=subject_id))[0].metadata\n", + " if len(json_data.info)>0:\n", + " for key in json_data.info.items():\n", + " if key in BIDS_Constants.json_keys:\n", + " if type(json_data.info[key]) is list:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key.replace(\" \", \"_\")]:''.join(str(e) for e in json_data.info[key])})\n", + " else:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key.replace(\" \", \"_\")]:json_data.info[key]})\n", + " \n", + " #Parse T1w.json file in BIDS directory to add the attributes contained inside\n", + " if (os.path.isdir(os.path.join(directory))):\n", + " try:\n", + " with open(os.path.join(directory,'T1w.json')) as data_file:\n", + " dataset = json.load(data_file)\n", + " except OSError:\n", + " logging.critical(\"Cannot find T1w.json file which is required in the BIDS spec\")\n", + " exit(\"-1\")\n", + " else:\n", + " logging.critical(\"Error: BIDS directory %s does not exist!\" %os.path.join(directory))\n", + " exit(\"-1\")\n", + "\n", + " #add various attributes if they exist in BIDS dataset\n", + " for key in dataset:\n", + " #if key from T1w.json file is mapped to term in BIDS_Constants.py then add to NIDM object\n", + " if key in BIDS_Constants.json_keys:\n", + " if type(dataset[key]) is list:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key]:\"\".join(dataset[key])})\n", + " else:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key]:dataset[key]}) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "- for taskxxx.json file " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + " \n", + " elif file_tpl.entities['datatype'] == 'func':\n", + " #do something with functionals\n", + " acq_obj = MRObject(acq)\n", + " #add image contrast type\n", + " if file_tpl.entities['suffix'] in BIDS_Constants.scans:\n", + " acq_obj.add_attributes({Constants.NIDM_IMAGE_CONTRAST_TYPE:BIDS_Constants.scans[file_tpl.entities['suffix']]})\n", + " else:\n", + " logging.info(\"WARNING: No matching image contrast type found in BIDS_Constants.py for %s\" % file_tpl.entities['suffix'])\n", + "\n", + " #add image usage type\n", + " if file_tpl.entities['datatype'] in BIDS_Constants.scans:\n", + " acq_obj.add_attributes({Constants.NIDM_IMAGE_USAGE_TYPE:BIDS_Constants.scans[file_tpl.entities['datatype']]})\n", + " else:\n", + " logging.info(\"WARNING: No matching image usage type found in BIDS_Constants.py for %s\" % file_tpl.entities['datatype'])\n", + " #make relative link to\n", + " acq_obj.add_attributes({Constants.NIDM_FILENAME:getRelPathToBIDS(join(file_tpl.dirname,file_tpl.filename), directory)})\n", + " #add sha512 sum\n", + " if isfile(join(directory,file_tpl.dirname,file_tpl.filename)):\n", + " acq_obj.add_attributes({Constants.CRYPTO_SHA512:getsha512(join(directory,file_tpl.dirname,file_tpl.filename))})\n", + " else:\n", + " logging.info(\"WARNINGL file %s doesn't exist! No SHA512 sum stored in NIDM files...\" %join(directory,file_tpl.dirname,file_tpl.filename))\n", + "\n", + " if 'run' in file_tpl.entities:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[\"run\"]:file_tpl.entities['run']})\n", + "\n", + " #get associated JSON file if exists\n", + " json_data = (bids_layout.get(suffix=file_tpl.entities['suffix'],subject=subject_id))[0].metadata\n", + "\n", + " if len(json_data.info)>0:\n", + " for key in json_data.info.items():\n", + " if key in BIDS_Constants.json_keys:\n", + " if type(json_data.info[key]) is list:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key.replace(\" \", \"_\")]:''.join(str(e) for e in json_data.info[key])})\n", + " else:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key.replace(\" \", \"_\")]:json_data.info[key]})\n", + " #get associated events TSV file\n", + " if 'run' in file_tpl.entities:\n", + " events_file = bids_layout.get(subject=subject_id, extensions=['.tsv'],modality=file_tpl.entities['datatype'],task=file_tpl.entities['task'],run=file_tpl.entities['run'])\n", + " else:\n", + " events_file = bids_layout.get(subject=subject_id, extensions=['.tsv'],modality=file_tpl.entities['datatype'],task=file_tpl.entities['task'])\n", + " #if there is an events file then this is task-based so create an acquisition object for the task file and link\n", + " if events_file:\n", + " #for now create acquisition object and link it to the associated scan\n", + " events_obj = AcquisitionObject(acq)\n", + " #add prov type, task name as prov:label, and link to filename of events file\n", + "\n", + " events_obj.add_attributes({PROV_TYPE:Constants.NIDM_MRI_BOLD_EVENTS,BIDS_Constants.json_keys[\"TaskName\"]: json_data[\"TaskName\"], Constants.NIDM_FILENAME:getRelPathToBIDS(events_file[0].filename, directory)})\n", + " #link it to appropriate MR acquisition entity\n", + " events_obj.wasAttributedTo(acq_obj)\n", + " \n", + " #Parse task-rest_bold.json file in BIDS directory to add the attributes contained inside\n", + " if (os.path.isdir(os.path.join(directory))):\n", + " try:\n", + " with open(os.path.join(directory,'task-rest_bold.json')) as data_file:\n", + " dataset = json.load(data_file)\n", + " except OSError:\n", + " logging.critical(\"Cannot find task-rest_bold.json file which is required in the BIDS spec\")\n", + " exit(\"-1\")\n", + " else:\n", + " logging.critical(\"Error: BIDS directory %s does not exist!\" %os.path.join(directory))\n", + " exit(\"-1\")\n", + "\n", + " #add various attributes if they exist in BIDS dataset\n", + " for key in dataset:\n", + " #if key from task-rest_bold.json file is mapped to term in BIDS_Constants.py then add to NIDM object\n", + " if key in BIDS_Constants.json_keys:\n", + " if type(dataset[key]) is list:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key]:\",\".join(map(str,dataset[key]))})\n", + " else:\n", + " acq_obj.add_attributes({BIDS_Constants.json_keys[key]:dataset[key]}) " + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The idea behind is to create acquisition objects for each scan for each subject using the information available in the bids directory. \n", + "Below, an extract of the nidme.json document created with the acquisition informations for one subject: " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + " {\n", + " \"@id\": \"nidm:8edfb514-b761-11e9-8f81-e1efbef5a165\",\n", + " \"@type\": [\n", + " \"Entity\",\n", + " \"AcquisitionObject\"\n", + " ],\n", + " \"crypto:sha512\": \"840affc90aeda40601a48f8d5cd3421fb73e0cbecc3111c4a3194aef74a716b732e7649b8b36d55f0b5559a88531ac10acd00bf27a95d52b59da78a88b7eeb8f\",\n", + " \"dicom:AcquisitionMatrix\": \"256x256\",\n", + " \"dicom:EchoTime\": 0.00248,\n", + " \"dicom:FlipAngle\": {\n", + " \"@type\": \"xsd:int\",\n", + " \"@value\": \"8\"\n", + " },\n", + " \"dicom:InversionTime\": 1.1,\n", + " \"dicom:MagneticFieldStrength\": {\n", + " \"@type\": \"xsd:int\",\n", + " \"@value\": \"3\"\n", + " },\n", + " \"dicom:PixelBandwidth\": 170.0,\n", + " \"dicom:RepetitionTime\": 1.87,\n", + " \"dicom:ScanningSequence\": \"MPRAGE\",\n", + " \"nidm:PhaseEncodingDirection\": \"j-\",\n", + " \"hadAcquisitionModality\": {\n", + " \"@id\": \"nidm:MagneticResonanceImaging\"\n", + " },\n", + " \"hadImageContrastType\": {\n", + " \"@id\": \"nidm:T1Weighted\"\n", + " },\n", + " \"HadImageUsageType\": {\n", + " \"@id\": \"nidm:Anatomical\"\n", + " },\n", + " \"filename\": \"/udd/nperez/sub-0050656/anat/sub-0050656_T1w.nii.gz\",\n", + " \"prov:wasGeneratedBy\": {\n", + " \"@id\": \"nidm:8edfacae-b761-11e9-8f81-e1efbef5a165\"\n", + " }\n", + " }," + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "However, some limitations appear: \n", + "- we only extract the data that map to BIDS_Constants.py and not all bids attributes are referenced in this document \n", + "- for bval and bvec files, what to do with those?" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.1" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}