From ec67041270806e9828f240f52b7b8a8569b350d9 Mon Sep 17 00:00:00 2001 From: basile Date: Thu, 12 Sep 2019 12:59:44 -0400 Subject: [PATCH 01/13] first quick attempt to add bids filters to further specify input --- fmriprep/cli/run.py | 7 +++++++ fmriprep/workflows/base.py | 10 ++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/fmriprep/cli/run.py b/fmriprep/cli/run.py index 71d5b482a..82f2eb6ef 100755 --- a/fmriprep/cli/run.py +++ b/fmriprep/cli/run.py @@ -13,6 +13,7 @@ import gc import uuid import warnings +import json from argparse import ArgumentParser from argparse import ArgumentDefaultsHelpFormatter from multiprocessing import cpu_count @@ -81,6 +82,8 @@ def get_parser(): # Re-enable when option is actually implemented # g_bids.add_argument('-r', '--run-id', action='store', default='single_run', # help='select a specific run to be processed') + g_bids.add_argument('bids_filters', action='store', type=Path, + help='the path to a JSON file describing custom BIDS input filter') g_bids.add_argument('-t', '--task-id', action='store', help='select a specific task to be processed') g_bids.add_argument('--echo-idx', action='store', type=int, @@ -509,6 +512,7 @@ def build_workflow(opts, retval): from niworkflows.reports import generate_reports from ..__about__ import __version__ from ..workflows.base import init_fmriprep_wf + from nipype.interfaces.base import Undefined build_log = nlogging.getLogger('nipype.workflow') @@ -522,6 +526,8 @@ def build_workflow(opts, retval): bids_dir = opts.bids_dir.resolve() output_dir = opts.output_dir.resolve() work_dir = opts.work_dir.resolve() + bids_filters_file = opts.bids_filters.resolve() + bids_filters = json.load(bids_filters_file) if bids_filters_file else Undefined retval['return_code'] = 1 retval['workflow'] = None @@ -672,6 +678,7 @@ def build_workflow(opts, retval): use_bbr=opts.use_bbr, use_syn=opts.use_syn_sdc, work_dir=str(work_dir), + bids_filters=bids_filters, ) retval['return_code'] = 0 diff --git a/fmriprep/workflows/base.py b/fmriprep/workflows/base.py index be73a60a6..c8bf520fb 100755 --- a/fmriprep/workflows/base.py +++ b/fmriprep/workflows/base.py @@ -69,6 +69,7 @@ def init_fmriprep_wf( use_bbr, use_syn, work_dir, + bids_filters, ): """ This workflow organizes the execution of FMRIPREP, with a sub-workflow for @@ -123,6 +124,7 @@ def init_fmriprep_wf( use_bbr=True, use_syn=True, work_dir='.', + bids_filters=None, ) @@ -257,6 +259,7 @@ def init_fmriprep_wf( use_aroma=use_aroma, use_bbr=use_bbr, use_syn=use_syn, + bids_filters=bids_filters, ) single_subject_wf.config['execution']['crashdump_dir'] = ( @@ -308,6 +311,7 @@ def init_single_subject_wf( use_aroma, use_bbr, use_syn, + bids_filters, ): """ This workflow organizes the preprocessing pipeline for a single subject. @@ -363,6 +367,7 @@ def init_single_subject_wf( use_aroma=False, use_bbr=True, use_syn=True, + bids_filters=None, ) @@ -449,7 +454,8 @@ def init_single_subject_wf( use_syn : bool **Experimental**: Enable ANTs SyN-based susceptibility distortion correction (SDC). If fieldmaps are present and enabled, this is not run, by default. - + bids_filters : dict + For BIDSDataGrabber output_query Inputs @@ -517,7 +523,7 @@ def init_single_subject_wf( inputnode = pe.Node(niu.IdentityInterface(fields=['subjects_dir']), name='inputnode') - bidssrc = pe.Node(BIDSDataGrabber(subject_data=subject_data, anat_only=anat_only), + bidssrc = pe.Node(BIDSDataGrabber(subject_data=subject_data, anat_only=anat_only, output_query=bids_filters), name='bidssrc') bids_info = pe.Node(BIDSInfo( From 6ed1e217a2d2d324f685ad8a6ab4ae0e54751eb1 Mon Sep 17 00:00:00 2001 From: basile Date: Thu, 12 Sep 2019 14:24:04 -0400 Subject: [PATCH 02/13] fix json loading --- fmriprep/cli/run.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fmriprep/cli/run.py b/fmriprep/cli/run.py index 82f2eb6ef..757ed9455 100755 --- a/fmriprep/cli/run.py +++ b/fmriprep/cli/run.py @@ -82,7 +82,7 @@ def get_parser(): # Re-enable when option is actually implemented # g_bids.add_argument('-r', '--run-id', action='store', default='single_run', # help='select a specific run to be processed') - g_bids.add_argument('bids_filters', action='store', type=Path, + g_bids.add_argument('--bids_filters', action='store', type=Path, help='the path to a JSON file describing custom BIDS input filter') g_bids.add_argument('-t', '--task-id', action='store', help='select a specific task to be processed') @@ -527,7 +527,7 @@ def build_workflow(opts, retval): output_dir = opts.output_dir.resolve() work_dir = opts.work_dir.resolve() bids_filters_file = opts.bids_filters.resolve() - bids_filters = json.load(bids_filters_file) if bids_filters_file else Undefined + bids_filters = json.load(open(bids_filters_file)) if bids_filters_file else Undefined retval['return_code'] = 1 retval['workflow'] = None From 98f498f39e934b9e23d3e40d572b26f420158841 Mon Sep 17 00:00:00 2001 From: basile Date: Thu, 12 Sep 2019 15:17:12 -0400 Subject: [PATCH 03/13] make the changes to use niworkflows BIDSDataGrabber --- fmriprep/cli/run.py | 3 +-- fmriprep/workflows/base.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/fmriprep/cli/run.py b/fmriprep/cli/run.py index 757ed9455..2d33f2486 100755 --- a/fmriprep/cli/run.py +++ b/fmriprep/cli/run.py @@ -512,7 +512,6 @@ def build_workflow(opts, retval): from niworkflows.reports import generate_reports from ..__about__ import __version__ from ..workflows.base import init_fmriprep_wf - from nipype.interfaces.base import Undefined build_log = nlogging.getLogger('nipype.workflow') @@ -527,7 +526,7 @@ def build_workflow(opts, retval): output_dir = opts.output_dir.resolve() work_dir = opts.work_dir.resolve() bids_filters_file = opts.bids_filters.resolve() - bids_filters = json.load(open(bids_filters_file)) if bids_filters_file else Undefined + bids_filters = json.load(open(bids_filters_file)) if bids_filters_file else None retval['return_code'] = 1 retval['workflow'] = None diff --git a/fmriprep/workflows/base.py b/fmriprep/workflows/base.py index c8bf520fb..a8bab29df 100755 --- a/fmriprep/workflows/base.py +++ b/fmriprep/workflows/base.py @@ -471,7 +471,7 @@ def init_single_subject_wf( 'bold': ['/completely/made/up/path/sub-01_task-nback_bold.nii.gz'] } else: - subject_data = collect_data(layout, subject_id, task_id, echo_idx)[0] + subject_data = collect_data(layout, subject_id, task_id, echo_idx, bids_filters=bids_filters)[0] # Make sure we always go through these two checks if not anat_only and subject_data['bold'] == []: @@ -523,7 +523,7 @@ def init_single_subject_wf( inputnode = pe.Node(niu.IdentityInterface(fields=['subjects_dir']), name='inputnode') - bidssrc = pe.Node(BIDSDataGrabber(subject_data=subject_data, anat_only=anat_only, output_query=bids_filters), + bidssrc = pe.Node(BIDSDataGrabber(subject_data=subject_data, anat_only=anat_only), name='bidssrc') bids_info = pe.Node(BIDSInfo( From a8ddc78bdd234dfd53d274c9d22f8b1d93e9dd2a Mon Sep 17 00:00:00 2001 From: Basile Date: Mon, 30 Sep 2019 09:56:09 -0400 Subject: [PATCH 04/13] Update fmriprep/cli/run.py Co-Authored-By: Oscar Esteban --- fmriprep/cli/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fmriprep/cli/run.py b/fmriprep/cli/run.py index 2d33f2486..3b367df0e 100755 --- a/fmriprep/cli/run.py +++ b/fmriprep/cli/run.py @@ -526,7 +526,8 @@ def build_workflow(opts, retval): output_dir = opts.output_dir.resolve() work_dir = opts.work_dir.resolve() bids_filters_file = opts.bids_filters.resolve() - bids_filters = json.load(open(bids_filters_file)) if bids_filters_file else None + bids_filters = json.loads(opts.bids_filters.read_text()) \ + if opts.bids_filters else None retval['return_code'] = 1 retval['workflow'] = None From 66584035a30aada0b01e97c63b40592a34284616 Mon Sep 17 00:00:00 2001 From: Basile Date: Mon, 30 Sep 2019 09:56:16 -0400 Subject: [PATCH 05/13] Update fmriprep/cli/run.py Co-Authored-By: Oscar Esteban --- fmriprep/cli/run.py | 1 - 1 file changed, 1 deletion(-) diff --git a/fmriprep/cli/run.py b/fmriprep/cli/run.py index 3b367df0e..14b51ef51 100755 --- a/fmriprep/cli/run.py +++ b/fmriprep/cli/run.py @@ -525,7 +525,6 @@ def build_workflow(opts, retval): bids_dir = opts.bids_dir.resolve() output_dir = opts.output_dir.resolve() work_dir = opts.work_dir.resolve() - bids_filters_file = opts.bids_filters.resolve() bids_filters = json.loads(opts.bids_filters.read_text()) \ if opts.bids_filters else None From 67afdccc51a5a4b7d1d9e84ab74754033b753947 Mon Sep 17 00:00:00 2001 From: Basile Date: Mon, 30 Sep 2019 10:03:57 -0400 Subject: [PATCH 06/13] hyphen-separated option Co-Authored-By: Chris Markiewicz --- fmriprep/cli/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fmriprep/cli/run.py b/fmriprep/cli/run.py index 14b51ef51..f4b905561 100755 --- a/fmriprep/cli/run.py +++ b/fmriprep/cli/run.py @@ -82,7 +82,7 @@ def get_parser(): # Re-enable when option is actually implemented # g_bids.add_argument('-r', '--run-id', action='store', default='single_run', # help='select a specific run to be processed') - g_bids.add_argument('--bids_filters', action='store', type=Path, + g_bids.add_argument('--bids-filters', action='store', type=Path, help='the path to a JSON file describing custom BIDS input filter') g_bids.add_argument('-t', '--task-id', action='store', help='select a specific task to be processed') From d6228790560c8241e8ff02f30796af6689c7852d Mon Sep 17 00:00:00 2001 From: basile Date: Mon, 30 Sep 2019 13:53:33 -0400 Subject: [PATCH 07/13] fix formatting and tests --- docs/workflows.rst | 1 + fmriprep/workflows/base.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/workflows.rst b/docs/workflows.rst index aa145cb86..b12e76ad9 100644 --- a/docs/workflows.rst +++ b/docs/workflows.rst @@ -58,6 +58,7 @@ is presented below: use_aroma=False, use_bbr=True, use_syn=True, + bids_filters=None, ) T1w/T2w preprocessing diff --git a/fmriprep/workflows/base.py b/fmriprep/workflows/base.py index a8bab29df..f993864d8 100755 --- a/fmriprep/workflows/base.py +++ b/fmriprep/workflows/base.py @@ -471,7 +471,8 @@ def init_single_subject_wf( 'bold': ['/completely/made/up/path/sub-01_task-nback_bold.nii.gz'] } else: - subject_data = collect_data(layout, subject_id, task_id, echo_idx, bids_filters=bids_filters)[0] + subject_data = collect_data(layout, subject_id, task_id, echo_idx, + bids_filters=bids_filters)[0] # Make sure we always go through these two checks if not anat_only and subject_data['bold'] == []: From 42792ea1da9a28067d1f7a20fabc0ae32a0d8471 Mon Sep 17 00:00:00 2001 From: basile Date: Wed, 27 Nov 2019 17:25:12 -0500 Subject: [PATCH 08/13] pep8+merge --- fmriprep/workflows/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fmriprep/workflows/base.py b/fmriprep/workflows/base.py index 5b74d57d3..c267571b9 100755 --- a/fmriprep/workflows/base.py +++ b/fmriprep/workflows/base.py @@ -472,7 +472,7 @@ def init_single_subject_wf( } else: subject_data = collect_data(layout, subject_id, task_id, echo_idx, - bids_filters=bids_filters)[0] + bids_filters=bids_filters)[0] # Make sure we always go through these two checks if not anat_only and subject_data['bold'] == []: From f589334fd9b0c5f9f78ea036472611684d09b439 Mon Sep 17 00:00:00 2001 From: basile Date: Tue, 10 Dec 2019 14:31:53 -0500 Subject: [PATCH 09/13] docs, fixes --- fmriprep/cli/run.py | 7 +++++-- fmriprep/workflows/base.py | 6 +++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/fmriprep/cli/run.py b/fmriprep/cli/run.py index e7c758e95..ef8c599da 100755 --- a/fmriprep/cli/run.py +++ b/fmriprep/cli/run.py @@ -83,8 +83,11 @@ def get_parser(): # Re-enable when option is actually implemented # g_bids.add_argument('-r', '--run-id', action='store', default='single_run', # help='select a specific run to be processed') - g_bids.add_argument('--bids-filters', action='store', type=Path, - help='the path to a JSON file describing custom BIDS input filter') + g_bids.add_argument( + '--bids-filters', action='store', type=Path, + help='the path to a JSON file describing custom BIDS input filter' + ' using pybids {:{:,...},...} ' + '(https://github.com/bids-standard/pybids/blob/master/bids/layout/config/bids.json)') g_bids.add_argument('-t', '--task-id', action='store', help='select a specific task to be processed') g_bids.add_argument('--echo-idx', action='store', type=int, diff --git a/fmriprep/workflows/base.py b/fmriprep/workflows/base.py index c267571b9..155db3b19 100755 --- a/fmriprep/workflows/base.py +++ b/fmriprep/workflows/base.py @@ -208,6 +208,9 @@ def init_fmriprep_wf( If fieldmaps are present and enabled, this is not run, by default. work_dir : str Directory in which to store workflow execution state and temporary files + bids_filters : dict + Provides finer specification of the pipeline input files using pybids entities filters. + A dict with the following structure {:{:,...},...} """ fmriprep_wf = Workflow(name='fmriprep_wf') @@ -455,7 +458,8 @@ def init_single_subject_wf( **Experimental**: Enable ANTs SyN-based susceptibility distortion correction (SDC). If fieldmaps are present and enabled, this is not run, by default. bids_filters : dict - For BIDSDataGrabber output_query + Provides finer specification of the pipeline input files using pybids entities filters. + A dict with the following structure {:{:,...},...} Inputs From b12ffb86aa6264f9e93dbb2c5a336f1bf1f96cd7 Mon Sep 17 00:00:00 2001 From: basile Date: Wed, 11 Dec 2019 15:59:20 -0500 Subject: [PATCH 10/13] fix testing --- fmriprep/workflows/base.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fmriprep/workflows/base.py b/fmriprep/workflows/base.py index d65306179..cd09bd020 100755 --- a/fmriprep/workflows/base.py +++ b/fmriprep/workflows/base.py @@ -129,6 +129,7 @@ def init_fmriprep_wf( use_bbr=True, use_syn=True, work_dir='.', + bids_filters=None, ) @@ -377,6 +378,7 @@ def init_single_subject_wf( use_aroma=False, use_bbr=True, use_syn=True, + bids_filters=None, ) Parameters From 7d0a9f1b240bed2e7b17c3fe037d01afd70934c9 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Tue, 11 Feb 2020 16:37:45 -0500 Subject: [PATCH 11/13] PIN: niworkflows master, smriprep feature branch --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index bfbeecc42..46c1f939d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,14 +24,14 @@ install_requires = nibabel >= 3.0.1 nipype >=1.3.1 nitime - niworkflows ~= 1.1.6 + niworkflows @ git+https://github.com/poldracklab/niworkflows.git@edeea81352f3f2b276f11e42e7829e1c5a03770f numpy pandas psutil >=5.4 pybids ~= 0.9.4 pyyaml sdcflows ~=1.1.0 - smriprep ~= 0.5.1 + smriprep @ git+https://github.com/bpinsard/smriprep.git@ef552bbad2418b558a2f639be502bcf98ec1cf3b tedana >=0.0.5 templateflow ~= 0.4.2rc1 test_requires = From f2de5c60e12fee1c280b6c2abc1c36937ebd44f2 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Thu, 13 Feb 2020 21:08:54 -0500 Subject: [PATCH 12/13] ENH: Change CLI option name --- fmriprep/cli/run.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/fmriprep/cli/run.py b/fmriprep/cli/run.py index 03bcf3b24..3fba59145 100755 --- a/fmriprep/cli/run.py +++ b/fmriprep/cli/run.py @@ -78,9 +78,9 @@ def get_parser(): # g_bids.add_argument('-r', '--run-id', action='store', default='single_run', # help='select a specific run to be processed') g_bids.add_argument( - '--bids-filters', action='store', type=Path, - help='the path to a JSON file describing custom BIDS input filter' - ' using pybids {:{:,...},...} ' + '--bids-filter-file', action='store', type=Path, metavar='PATH', + help='a JSON file describing custom BIDS input filter using pybids ' + '{:{:,...},...} ' '(https://github.com/bids-standard/pybids/blob/master/bids/layout/config/bids.json)') g_bids.add_argument('-t', '--task-id', action='store', help='select a specific task to be processed') @@ -505,8 +505,7 @@ def build_workflow(opts, retval): bids_dir = opts.bids_dir.resolve() output_dir = opts.output_dir.resolve() work_dir = opts.work_dir.resolve() - bids_filters = json.loads(opts.bids_filters.read_text()) \ - if opts.bids_filters else None + bids_filters = json.loads(opts.bids_filter_file.read_text()) if opts.bids_filter_file else None if opts.clean_workdir: from niworkflows.utils.misc import clean_directory From ec73b2dec2b39c0d3c8abbe11388871c803d3d42 Mon Sep 17 00:00:00 2001 From: Chris Markiewicz Date: Fri, 14 Feb 2020 08:56:24 -0500 Subject: [PATCH 13/13] PIN: smriprep pre-release --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 46c1f939d..e9dd0d5e5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,7 +31,7 @@ install_requires = pybids ~= 0.9.4 pyyaml sdcflows ~=1.1.0 - smriprep @ git+https://github.com/bpinsard/smriprep.git@ef552bbad2418b558a2f639be502bcf98ec1cf3b + smriprep @ git+https://github.com/poldracklab/smriprep.git@07a7ba3fce8a9953923bcd0fa6ffbffac401208d tedana >=0.0.5 templateflow ~= 0.4.2rc1 test_requires =