-
-
Notifications
You must be signed in to change notification settings - Fork 20
Pydeface module #373
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Pydeface module #373
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|
| @@ -0,0 +1,89 @@ | ||||||||||
| process PYDEFACE { | ||||||||||
|
|
||||||||||
| tag "${meta.subject}_${meta.session}_${nifti.name}" | ||||||||||
|
|
||||||||||
| container { | ||||||||||
| workflow.containerEngine in ['singularity', 'apptainer'] && !(task.ext.singularity_pull_docker_container ?: false) | ||||||||||
| ? 'docker://poldracklab/pydeface:latest' | ||||||||||
| : 'docker.io/poldracklab/pydeface:latest' | ||||||||||
| } | ||||||||||
|
|
||||||||||
| containerOptions { | ||||||||||
| workflow.containerEngine == 'docker' ? '--entrypoint ""' : '' | ||||||||||
| } | ||||||||||
|
|
||||||||||
| cpus { (params.pydeface_cpus ?: 4) as Integer } | ||||||||||
| memory { params.pydeface_mem ?: '8 GB' } | ||||||||||
| time { params.pydeface_time ?: '2h' } | ||||||||||
|
Comment on lines
+15
to
+17
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. For resources, we use nf-core labels. They used to have a better definition of the kind of resources attached to each, we need to document that better on our side. For this case I'd use the
Suggested change
|
||||||||||
|
|
||||||||||
| input: | ||||||||||
| tuple val(meta), path(nifti) | ||||||||||
|
|
||||||||||
| output: | ||||||||||
| tuple val(meta), path("sub-${meta.subject}/ses-${meta.session}/anat/*_defaced.nii.gz"), emit: defaced | ||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is the bids-like output structure here needed for the |
||||||||||
| path("logs/*.log"), emit: logs | ||||||||||
| path "versions.yml", emit: versions, topic: "versions" | ||||||||||
|
|
||||||||||
| script: | ||||||||||
| """ | ||||||||||
| set -euo pipefail | ||||||||||
|
|
||||||||||
| export OMP_NUM_THREADS="${task.cpus}" | ||||||||||
| export OMP_THREADS="${task.cpus}" | ||||||||||
|
|
||||||||||
| sub="sub-${meta.subject}" | ||||||||||
| ses="ses-${meta.session}" | ||||||||||
|
Comment on lines
+34
to
+35
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. To be general, don't expect to have access to the subject or session here, but you'll always have access to the |
||||||||||
|
|
||||||||||
| mkdir -p "\$sub/\$ses/anat" logs | ||||||||||
|
|
||||||||||
| base="\$(basename "${nifti}" .nii.gz)" | ||||||||||
| out_file="\$sub/\$ses/anat/\${base}_defaced.nii.gz" | ||||||||||
|
|
||||||||||
| log_prefix="sub-${meta.subject}_ses-${meta.session}_\${base}" | ||||||||||
| out_log="logs/\${log_prefix}_out.log" | ||||||||||
| err_log="logs/\${log_prefix}_err.log" | ||||||||||
|
|
||||||||||
| { | ||||||||||
| echo "=== PYDEFACE ===" | ||||||||||
| echo "INPUT: ${nifti}" | ||||||||||
| echo "OUTPUT: \$out_file" | ||||||||||
| echo "CPUS: ${task.cpus}" | ||||||||||
| echo "OMP_THREADS: \$OMP_THREADS" | ||||||||||
| echo "DATE: \$(date -Is)" | ||||||||||
| echo "===============" | ||||||||||
| echo | ||||||||||
| } | tee -a "\$out_log" | ||||||||||
|
|
||||||||||
| if [[ "\$base" == *"_defaced" ]]; then | ||||||||||
| echo "[SKIP] Input already looks defaced: \$base" | tee -a "\$out_log" | ||||||||||
| cp "${nifti}" "\$out_file" | ||||||||||
| else | ||||||||||
|
Comment on lines
+57
to
+60
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This main condition should not be required. The including subworkflow or pipeline will have the job of filtering out those data points |
||||||||||
| if [[ -f /shell-hook.sh ]]; then | ||||||||||
| /bin/bash /shell-hook.sh pydeface "${nifti}" --outfile "\$out_file" \\ | ||||||||||
| 1> >(tee -a "\$out_log") \\ | ||||||||||
| 2> >(tee >(grep -i -e "warning" -e "error" >> "\$err_log") >&2) | ||||||||||
| elif command -v pydeface >/dev/null 2>&1; then | ||||||||||
| pydeface "${nifti}" --outfile "\$out_file" \\ | ||||||||||
| 1> >(tee -a "\$out_log") \\ | ||||||||||
| 2> >(tee >(grep -i -e "warning" -e "error" >> "\$err_log") >&2) | ||||||||||
| elif command -v python3 >/dev/null 2>&1; then | ||||||||||
| python3 -m pydeface "${nifti}" --outfile "\$out_file" \\ | ||||||||||
| 1> >(tee -a "\$out_log") \\ | ||||||||||
| 2> >(tee >(grep -i -e "warning" -e "error" >> "\$err_log") >&2) | ||||||||||
| else | ||||||||||
| echo "ERROR: Neither /shell-hook.sh, pydeface nor python3 available in container" >&2 | ||||||||||
| exit 127 | ||||||||||
| fi | ||||||||||
| fi | ||||||||||
|
|
||||||||||
| : >> "\$err_log" | ||||||||||
|
|
||||||||||
| PYDEFACE_VERSION="\$(pydeface --version 2>&1 | grep -oE '[0-9]+[.][0-9]+([.][0-9]+)?' | head -n 1 || true)" | ||||||||||
| [ -z "\$PYDEFACE_VERSION" ] && PYDEFACE_VERSION="unknown" | ||||||||||
|
|
||||||||||
| cat <<-END_VERSIONS > versions.yml | ||||||||||
| "${task.process}": | ||||||||||
| pydeface: "\$PYDEFACE_VERSION" | ||||||||||
| END_VERSIONS | ||||||||||
| """ | ||||||||||
| } | ||||||||||
| Original file line number | Diff line number | Diff line change | ||
|---|---|---|---|---|
| @@ -0,0 +1,65 @@ | ||||
| name: "pydeface" | ||||
| description: Remove facial structures from MRI NIfTI images using PyDeface. | ||||
| keywords: | ||||
| - neuroimaging | ||||
| - MRI | ||||
| - NIfTI | ||||
| - anonymization | ||||
| - defacing | ||||
| authors: | ||||
| - "@loboehme1" | ||||
| maintainers: | ||||
| - "@loboehme1" | ||||
| input: | ||||
| - - meta: | ||||
| type: map | ||||
| description: | | ||||
| Groovy Map containing sample information. | ||||
| Expected keys for this module currently include `subject` and `session`, | ||||
| for example `[ subject: '01', session: '001' ]`. | ||||
| - nifti: | ||||
| type: file | ||||
| description: | | ||||
| Input anatomical MRI image in NIfTI format. | ||||
| pattern: "*.{nii,nii.gz}" | ||||
| output: | ||||
| defaced: | ||||
| - - meta: | ||||
| type: map | ||||
| description: | | ||||
| Groovy Map containing sample information. | ||||
| Expected keys for this module currently include `subject` and `session`, | ||||
| for example `[ subject: '01', session: '001' ]`. | ||||
| - sub-${meta.subject}/ses-${meta.session}/anat/*_defaced.nii.gz: {} | ||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This block should contain a short description of the image content and descriptors. Here's a (recent module](
|
||||
| versions_pydeface: | ||||
| - - "${task.process}": | ||||
| type: string | ||||
| description: The name of the process. | ||||
| - "pydeface": | ||||
| type: string | ||||
| description: The name of the tool. | ||||
| - "pydeface_version": | ||||
| type: string | ||||
| description: The version of PyDeface used by the process. | ||||
|
Comment on lines
+34
to
+43
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Refer to the module linked above to replace the versions block. You can use the exact same one, copy-pasted ! |
||||
| tools: | ||||
| - "pydeface": | ||||
| description: "A tool to remove facial structure from MRI images." | ||||
| homepage: "https://www.poldracklab.org" | ||||
| documentation: "https://github.com/poldracklab/pydeface" | ||||
| tool_dev_url: "https://github.com/poldracklab/pydeface" | ||||
| doi: "10.5281/zenodo.6856482" | ||||
| licence: | ||||
| - "MIT" | ||||
| identifier: "" | ||||
| topics: | ||||
| versions: | ||||
| - - "${task.process}": | ||||
| type: string | ||||
| description: The name of the process. | ||||
| - "pydeface": | ||||
| type: string | ||||
| description: The name of the tool. | ||||
| '"versions"': | ||||
| - versions.yml: | ||||
| type: string | ||||
| description: The name of the process. | ||||
|
Comment on lines
+54
to
+65
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Is this default ? I have to get a look at newest updates to the nf-core guidelines, we should update documentation on our end to align with the latest additions 😅 |
||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,115 @@ | ||
| nextflow_process { | ||
|
|
||
| name "Test Process PYDEFACE" | ||
| script "../main.nf" | ||
| process "PYDEFACE" | ||
| config "./nextflow.config" | ||
|
|
||
| tag "modules" | ||
| tag "modules_nfneuro" | ||
| tag "pydeface" | ||
| tag "anonymization" | ||
| tag "anonymization/pydeface" | ||
|
|
||
| tag "subworkflows" | ||
| tag "subworkflows/load_test_data" | ||
|
|
||
|
|
||
| test("single T1w nifti creates defaces image and logs") { | ||
| setup { | ||
| run("LOAD_TEST_DATA", alias: "LOAD_DATA") { | ||
| script "../../../../../subworkflows/nf-neuro/load_test_data/main.nf" | ||
| process { | ||
| """ | ||
| input[0] = channel.from( ["T1w.zip"] ) | ||
| input[1] = "test.load-test-data" | ||
| """ | ||
| } | ||
| } | ||
| } | ||
|
|
||
|
|
||
| when { | ||
| process { | ||
| """ | ||
| ch_t1w = LOAD_DATA.out.test_data_directory | ||
| .filter{ it.simpleName == "T1w" } | ||
| .map{ test_data_directory -> [ | ||
| [ subject: '01', session: '001' ], | ||
| file("\${test_data_directory}/T1w.nii.gz") | ||
| ] } | ||
|
|
||
| input[0] = ch_t1w | ||
| """ | ||
|
|
||
| } | ||
| } | ||
|
|
||
| then { | ||
|
|
||
| // check if test case succeeded | ||
| assert process.success | ||
|
|
||
| // exactly one file per input nifti for normal output and publish output | ||
| assert process.out.defaced.size() == 1 | ||
|
|
||
| // logs should exist | ||
| assert process.out.logs.size() >= 1 | ||
|
|
||
| // helper functions | ||
| def asList = { x -> | ||
| x instanceof Collection ? x as List : [x] | ||
| } | ||
|
|
||
| def flatten = { xs -> | ||
| xs.collectMany { it instanceof Collection ? it as List : [it] } | ||
| } | ||
|
|
||
| def basename = { x -> | ||
| new File(x.toString()).name | ||
| } | ||
|
|
||
| def exists = { x -> | ||
| new File(x.toString()).exists() | ||
| } | ||
|
|
||
| def fileSize = { x -> | ||
| new File(x.toString()).length() | ||
| } | ||
|
|
||
| // get all module outputs | ||
| def meta = process.out.defaced[0][0] | ||
| def defacedFiles = asList(process.out.defaced[0][1]) | ||
| def logFiles = flatten(process.out.logs) | ||
|
|
||
| // check meta data | ||
| assert meta.subject == '01' | ||
| assert meta.session == '001' | ||
|
|
||
| // defaced output check | ||
| assert defacedFiles.size() == 1 | ||
| assert basename(defacedFiles[0]) == 'T1w_defaced.nii.gz' | ||
| assert exists(defacedFiles[0]) | ||
| assert fileSize(defacedFiles[0]) > 0 | ||
|
|
||
| // logs check | ||
| assert logFiles.collect { basename(it) }.sort() == [ | ||
| 'sub-01_ses-001_T1w_err.log', | ||
| 'sub-01_ses-001_T1w_out.log' | ||
| ] | ||
|
|
||
| assert logFiles.every { exists(it) } | ||
|
|
||
| assert snapshot( | ||
| [ | ||
| meta: [ | ||
| subject: meta.subject, | ||
| session: meta.session | ||
| ], | ||
| defaced: defacedFiles.collect { basename(it) }.sort(), | ||
| logs: logFiles.collect { basename(it) }.sort() | ||
| ] | ||
| ).match("pydeface_outputs") | ||
| } | ||
| } | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| { | ||
| "pydeface_outputs": { | ||
| "content": [ | ||
| { | ||
| "meta": { | ||
| "subject": "01", | ||
| "session": "001" | ||
| }, | ||
| "defaced": [ | ||
| "T1w_defaced.nii.gz" | ||
| ], | ||
| "logs": [ | ||
| "sub-01_ses-001_T1w_err.log", | ||
| "sub-01_ses-001_T1w_out.log" | ||
| ] | ||
| } | ||
| ], | ||
| "meta": { | ||
| "nf-test": "0.9.3", | ||
| "nextflow": "26.04.4" | ||
| }, | ||
| "timestamp": "2026-06-23T15:55:55.012676904" | ||
| } | ||
| } |
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. You can remove most of the content of this file, defaults like |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,27 @@ | ||
| params.pydeface_cpus = 1 | ||
| params.pydeface_mem = '3 GB' | ||
| params.pydeface_time = '1h' | ||
|
|
||
| process { | ||
| cpus = 1 | ||
| } | ||
|
|
||
| trace { | ||
| enabled = false | ||
| overwrite = true | ||
| } | ||
|
|
||
| timeline { | ||
| enabled = false | ||
| overwrite = true | ||
| } | ||
|
|
||
| report { | ||
| enabled = false | ||
| overwrite = true | ||
| } | ||
|
|
||
| dag { | ||
| enabled = false | ||
| overwrite = true | ||
| } |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Here, use the docker container definition directly. Support for
apptainerand other environments is in the team's backlog and will be deployed over the whole library in the future.