From 6dbf9e2d0c340b23d75018c9d3b6b665212f80c1 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Wed, 3 Oct 2018 23:05:45 -0400 Subject: [PATCH 01/38] Simplify raw_to_bids --- doc/api.rst | 13 +++- examples/convert_eeg_to_bids.py | 27 +++---- examples/convert_group_studies.py | 16 +++-- examples/convert_mne_sample.py | 14 ++-- mne_bids/__init__.py | 2 +- mne_bids/io.py | 4 +- mne_bids/mne_bids.py | 116 ++++++++++-------------------- mne_bids/utils.py | 14 ++++ 8 files changed, 96 insertions(+), 110 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index af0613bd0..4021e0e2c 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -13,7 +13,17 @@ MNE BIDS (:py:mod:`mne_bids.mne_bids`): .. autosummary:: :toctree: generated/ - raw_to_bids + write_raw_bids + make_bids_filename + +IO (:py:mod:`mne_bids.io`): + +.. current_module:: mne_bids.io + +.. autosummary:: + :toctree: generated/ + + read_raw Utils (:py:mod:`mne_bids.utils`): @@ -24,7 +34,6 @@ Utils (:py:mod:`mne_bids.utils`): print_dir_tree make_bids_folders - make_bids_filename make_dataset_description copyfile_brainvision diff --git a/examples/convert_eeg_to_bids.py b/examples/convert_eeg_to_bids.py index d5b126334..f5e592f69 100644 --- a/examples/convert_eeg_to_bids.py +++ b/examples/convert_eeg_to_bids.py @@ -26,7 +26,7 @@ from mne.datasets import eegbci from mne.io import read_raw_edf -from mne_bids import raw_to_bids +from mne_bids import write_raw_bids, make_bids_filename from mne_bids.utils import print_dir_tree ############################################################################### @@ -104,7 +104,7 @@ # * output_path # # ... as you can see in the docstring: -print(raw_to_bids.__doc__) +print(write_raw_bids.__doc__) ############################################################################### # We loaded 'S001R02.edf', which corresponds to subject 1 in the second task. @@ -126,9 +126,10 @@ trial_type = {'rest': 0, 'imagine left fist': 1, 'imagine right fist': 2} # Now convert our data to be in a new BIDS dataset. -raw_to_bids(subject_id=subject_id, task=task, raw_file=raw_file, - output_path=output_path, kind=kind, event_id=trial_type, - overwrite=False) +bids_fname = make_bids_filename(subject=subject_id, task=task, + suffix='eeg.edf') +write_raw_bids(raw_file, bids_fname, output_path, kind=kind, + event_id=trial_type) ############################################################################### # What does our fresh BIDS directory look like? @@ -159,17 +160,11 @@ 'S{:03}'.format(subj_idx), 'S{:03}R{:02}.edf'.format(subj_idx, task_idx)) raw = read_raw_edf(edf_path, preload=True) - - # `kind` and `trial_type` were already defined above - raw_to_bids(subject_id='{:03}'.format(subj_idx), - task=task_names[task_idx], - run=run_mapping[task_idx], - raw_file=raw, - output_path=output_path, - kind=kind, - event_id=trial_type, - overwrite=True - ) + bids_fname = make_bids_filename( + subject='{:03}'.format(subj_idx), task=task_names[task_idx], + run=run_mapping[task_idx], suffix='eeg.edf') + write_raw_bids(raw, bids_fname, output_path, kind=kind, + event_id=trial_type, overwrite=True) ############################################################################### # Step 3: Check and compare with standard diff --git a/examples/convert_group_studies.py b/examples/convert_group_studies.py index 1ec5840f2..a43440c8a 100644 --- a/examples/convert_group_studies.py +++ b/examples/convert_group_studies.py @@ -24,7 +24,8 @@ # Let us import ``mne_bids`` import os.path as op -from mne_bids import raw_to_bids +from mne_bids import write_raw_bids, make_bids_filename +from mne_bids.io import read_raw from mne_bids.datasets import fetch_faces_data from mne_bids.utils import print_dir_tree @@ -63,13 +64,16 @@ for subject_id in subject_ids: subject = 'sub%03d' % subject_id for run in runs: - raw_file = op.join(data_path, repo, subject, 'MEG', - 'run_%02d_raw.fif' % run) + fname = op.join(data_path, repo, subject, 'MEG', + 'run_%02d_raw.fif' % run) + + raw = read_raw(fname) + bids_fname = make_bids_filename( + subject_id='%02d' % subject_id, session='01', run=run, + task='VisualFaces', suffix='meg.fif') # Make it BIDS compatible - raw_to_bids(subject_id='%02d' % subject_id, session_id='01', run=run, - task='VisualFaces', raw_file=raw_file, event_id=event_id, - output_path=output_path, overwrite=True) + write_raw_bids(raw, bids_fname, output_path, event_id=event_id) ############################################################################### # Now let's see the structure of the BIDS folder we created. diff --git a/examples/convert_mne_sample.py b/examples/convert_mne_sample.py index 1f47ca605..ca8f945da 100644 --- a/examples/convert_mne_sample.py +++ b/examples/convert_mne_sample.py @@ -18,7 +18,8 @@ import os.path as op from mne.datasets import sample -from mne_bids import raw_to_bids +from mne_bids import write_raw_bids, make_bids_filename +from mne_bids.io import read_raw from mne_bids.utils import print_dir_tree ############################################################################### @@ -28,17 +29,18 @@ event_id = {'Auditory/Left': 1, 'Auditory/Right': 2, 'Visual/Left': 3, 'Visual/Right': 4, 'Smiley': 5, 'Button': 32} -raw_file = op.join(data_path, 'MEG', 'sample', 'sample_audvis_raw.fif') +fname = op.join(data_path, 'MEG', 'sample', 'sample_audvis_raw.fif') events_data = op.join(data_path, 'MEG', 'sample', 'sample_audvis_raw-eve.fif') output_path = op.join(data_path, '..', 'MNE-sample-data-bids') ############################################################################### # Finally, we specify the raw_file and events_data -raw_to_bids(subject_id='01', run='01', session_id='01', task='audiovisual', - raw_file=raw_file, events_data=events_data, - output_path=output_path, event_id=event_id, - overwrite=False) +raw = read_raw(fname) +bids_fname = make_bids_filename(subject='01', run='01', session='01', + task='audiovisual', suffix='meg.fif') +write_raw_bids(raw, bids_fname, output_path, events_data=events_data, + event_id=event_id, overwrite=True) ############################################################################### # Now let's see the structure of the BIDS folder we created. diff --git a/mne_bids/__init__.py b/mne_bids/__init__.py index c4899432f..8638b010f 100644 --- a/mne_bids/__init__.py +++ b/mne_bids/__init__.py @@ -3,6 +3,6 @@ __version__ = '0.1.dev0' -from .mne_bids import raw_to_bids # noqa +from .mne_bids import write_raw_bids # noqa from .utils import make_bids_folders, make_bids_filename # noqa from .config import BIDS_VERSION # noqa diff --git a/mne_bids/io.py b/mne_bids/io.py index 6fec70df7..00449ce34 100644 --- a/mne_bids/io.py +++ b/mne_bids/io.py @@ -32,8 +32,8 @@ def _parse_ext(raw_fname, verbose=False): return fname, ext -def _read_raw(raw_fname, electrode=None, hsp=None, hpi=None, config=None, - montage=None, verbose=None): +def read_raw(raw_fname, electrode=None, hsp=None, hpi=None, config=None, + montage=None, verbose=None): """Read a raw file into MNE, making inferences based on extension.""" fname, ext = _parse_ext(raw_fname) diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 761fc5eb5..1fdc16351 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -31,8 +31,8 @@ make_dataset_description, _write_json, _write_tsv, _read_events, _mkdir_p, age_on_date, copyfile_brainvision, copyfile_eeglab, - _infer_eeg_placement_scheme) -from .io import (_parse_ext, _read_raw, ALLOWED_EXTENSIONS) + _infer_eeg_placement_scheme, _parse_bids_filename) +from .io import (_parse_ext, ALLOWED_EXTENSIONS) ALLOWED_KINDS = ['meg', 'eeg', 'ieeg'] @@ -528,30 +528,20 @@ def _sidecar_json(raw, task, manufacturer, fname, kind, eeg_ref=None, return fname -def raw_to_bids(subject_id, task, raw_file, output_path, session_id=None, - acquisition=None, run=None, kind='meg', events_data=None, - event_id=None, hpi=None, electrode=None, hsp=None, - eeg_ref=None, eeg_gnd=None, config=None, - overwrite=False, verbose=True): +def write_raw_bids(raw, bids_fname, output_path, kind='meg', events_data=None, + event_id=None, eeg_ref=None, eeg_gnd=None, overwrite=False, + verbose=True): """Walk over a folder of files and create BIDS compatible folder. Parameters ---------- - subject_id : str - The subject name in BIDS compatible format ('01', '02', etc.) - task : str - Name of the task the data is based on. - raw_file : str | instance of mne.Raw + raw : instance of mne.Raw The raw data. If a string, it is assumed to be the path to the raw data file. Otherwise it must be an instance of mne.Raw - output_path : str - The path of the BIDS compatible folder - session_id : str | None - The session name in BIDS compatible format. - acquisition : str | None - Acquisition parameter for the dataset. - run : int | None - The run number for this dataset. + bids_fname : str + The file path of the BIDS compatible raw file. In the case of multifile + systems (e.g., vhdr, .ds etc.), this is the path to a folder containing + all the files. kind : str, one of ('meg', 'eeg', 'ieeg') The kind of data being converted. Defaults to "meg". events_data : str | array | None @@ -560,24 +550,11 @@ def raw_to_bids(subject_id, task, raw_file, output_path, session_id=None, inferred from the stim channel using `mne.find_events`. event_id : dict | None The event id dict used to create a 'trial_type' column in events.tsv - hpi : None | str - Marker points representing the location of the marker coils with - respect to the MEG Sensors, or path to a marker file. - electrode : None | str - Digitizer points representing the location of the fiducials and the - marker coils with respect to the digitized head shape, or path to a - file containing these points. - hsp : None | str | array, shape = (n_points, 3) - Digitizer head shape points, or path to head shape file. If more than - 10`000 points are in the head shape, they are automatically decimated. eeg_ref : str Description of the type of reference used and (when applicable) of location of the reference electrode. Defaults to None. eeg_gnd : str Description of the location of the ground electrode. Defaults to None. - config : str | None - A path to the configuration file to use if the data is from a BTi - system. overwrite : bool Whether to overwrite existing files or data in files. Defaults to False. @@ -598,26 +575,22 @@ def raw_to_bids(subject_id, task, raw_file, output_path, session_id=None, of the participant correctly. """ - if isinstance(raw_file, string_types): - # We must read in the raw data - raw = _read_raw(raw_file, electrode=electrode, hsp=hsp, hpi=hpi, - config=config, verbose=verbose) - _, ext = _parse_ext(raw_file, verbose=verbose) - raw_fname = raw_file - elif isinstance(raw_file, BaseRaw): - # We got a raw mne object, get back the filename if possible - # Assume that if no filename attr exists, it's a fif file. - raw = raw_file.copy() - if hasattr(raw, 'filenames'): - _, ext = _parse_ext(raw.filenames[0], verbose=verbose) - raw_fname = raw.filenames[0] - else: - # FIXME: How to get the filename if no filenames attribute? - raw_fname = 'unknown_file_name' - ext = '.fif' - else: - raise ValueError('raw_file must be an instance of str or BaseRaw, ' - 'got %s' % type(raw_file)) + if not isinstance(raw, BaseRaw): + raise ValueError('raw_file must be an instance of BaseRaw, ' + 'got %s' % type(raw)) + + # We got a raw mne object, get back the filename if possible + # Assume that if no filename attr exists, it's a fif file. + if not hasattr(raw, 'filenames'): + raise ValueError('raw.filenames is missing.') + + _, ext = _parse_ext(raw.filenames[0], verbose=verbose) + raw_fname = raw.filenames[0] + + params = _parse_bids_filename(bids_fname) + subject_id, session_id = params['sub'], params['ses'] + acquisition, task, run = params['acq'], params['task'], params['run'] + fname = os.path.join(output_path, bids_fname) data_path = make_bids_folders(subject=subject_id, session=session_id, kind=kind, root=output_path, @@ -641,18 +614,6 @@ def raw_to_bids(subject_id, task, raw_file, output_path, session_id=None, data_meta_fname = make_bids_filename( subject=subject_id, session=session_id, task=task, run=run, acquisition=acquisition, suffix='%s.json' % kind, prefix=data_path) - if ext in ['.fif', '.ds', '.vhdr', '.edf', '.bdf', '.set', '.cnt']: - raw_file_bids = make_bids_filename( - subject=subject_id, session=session_id, task=task, run=run, - acquisition=acquisition, suffix='%s%s' % (kind, ext)) - else: - raw_folder = make_bids_filename( - subject=subject_id, session=session_id, task=task, run=run, - acquisition=acquisition, suffix='%s' % kind) - raw_file_bids = make_bids_filename( - subject=subject_id, session=session_id, task=task, run=run, - acquisition=acquisition, suffix='%s%s' % (kind, ext), - prefix=raw_folder) events_tsv_fname = make_bids_filename( subject=subject_id, session=session_id, task=task, acquisition=acquisition, run=run, suffix='events.tsv', @@ -671,7 +632,7 @@ def raw_to_bids(subject_id, task, raw_file, output_path, session_id=None, # save all meta data _participants_tsv(raw, subject_id, "n/a", participants_fname, overwrite, verbose) - _scans_tsv(raw, os.path.join(kind, raw_file_bids), scans_fname, + _scans_tsv(raw, os.path.join(kind, bids_fname), scans_fname, overwrite, verbose) # TODO: Implement coordystem.json and electrodes.tsv for EEG and iEEG @@ -691,14 +652,14 @@ def raw_to_bids(subject_id, task, raw_file, output_path, session_id=None, # set the raw file name to now be the absolute path to ensure the files # are placed in the right location - raw_file_bids = os.path.join(data_path, raw_file_bids) - if os.path.exists(raw_file_bids) and not overwrite: + bids_fname = os.path.join(data_path, bids_fname) + if os.path.exists(bids_fname) and not overwrite: raise OSError(errno.EEXIST, '"%s" already exists. Please set ' - 'overwrite to True.' % raw_file_bids) - _mkdir_p(os.path.dirname(raw_file_bids)) + 'overwrite to True.' % bids_fname) + _mkdir_p(os.path.dirname(bids_fname)) if verbose: - print('Writing data files to %s' % raw_file_bids) + print('Writing data files to %s' % fname) if ext not in ALLOWED_EXTENSIONS: raise ValueError('ext must be in %s, got %s' @@ -711,7 +672,7 @@ def raw_to_bids(subject_id, task, raw_file, output_path, session_id=None, # TODO Update MNE requirement to version 0.17 when it's released if check_version('mne', '0.17.dev'): split_naming = 'bids' - raw.save(raw_file_bids, split_naming=split_naming, + raw.save(bids_fname, split_naming=split_naming, overwrite=True) else: raise NotImplementedError( @@ -722,18 +683,19 @@ def raw_to_bids(subject_id, task, raw_file, output_path, session_id=None, else: # TODO insert arg `split_naming=split_naming` # when MNE releases 0.17 - raw.save(raw_file_bids, overwrite=True) + raw.save(bids_fname, overwrite=True) + # CTF data is saved in a directory elif ext == '.ds': - sh.copytree(raw_fname, raw_file_bids) + sh.copytree(raw_fname, bids_fname) # BrainVision is multifile, copy over all of them and fix pointers elif ext == '.vhdr': - copyfile_brainvision(raw_fname, raw_file_bids) + copyfile_brainvision(raw_fname, bids_fname) # EEGLAB .set might be accompanied by a .fdt - find out and copy it too elif ext == '.set': - copyfile_eeglab(raw_fname, raw_file_bids) + copyfile_eeglab(raw_fname, bids_fname) else: - sh.copyfile(raw_fname, raw_file_bids) + sh.copyfile(raw_fname, bids_fname) # KIT data requires the marker file to be copied over too if hpi is not None: if isinstance(hpi, list): diff --git a/mne_bids/utils.py b/mne_bids/utils.py index d5896f457..e2ec9a166 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -62,6 +62,20 @@ def _mkdir_p(path, overwrite=False, verbose=False): raise +def _parse_bids_filename(fname): + """Get dict from BIDS fname.""" + params = dict(sub=None, ses=None, task=None, acq=None, + run=None, proc=None, recording=None, space=None) + entities = fname.split('_') + for entity in entities[:-1]: + key, value = entity.split('-') + if key not in params: + raise KeyError('Unexpected entity found in filename %s' % entity) + params[key] = value + params['suffix'] = entities[-1] + return params + + def make_bids_filename(subject=None, session=None, task=None, acquisition=None, run=None, processing=None, recording=None, space=None, suffix=None, prefix=None): From 9f1afb27a2a7d1fb5d8235c402a7f33593f2af25 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Sun, 14 Oct 2018 17:46:20 -0400 Subject: [PATCH 02/38] TST start modifying tests --- mne_bids/io.py | 2 +- mne_bids/mne_bids.py | 15 ++++- mne_bids/tests/test_mne_bids.py | 98 ++++++++++++++++----------------- 3 files changed, 61 insertions(+), 54 deletions(-) diff --git a/mne_bids/io.py b/mne_bids/io.py index 00449ce34..70f07c61c 100644 --- a/mne_bids/io.py +++ b/mne_bids/io.py @@ -13,7 +13,7 @@ allowed_extensions_eeg = ['.vhdr', # BrainVision, accompanied by .vmrk, .eeg '.edf', # European Data Format '.bdf', # Biosemi - '.set', # EEGLAB, potentially accompanied by .fdt + '.set', '.fdt', # EEGLAB '.cnt', # Neuroscan ] diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 1fdc16351..7176dc539 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -10,6 +10,8 @@ import os import errno +import os.path as op + import shutil as sh import pandas as pd from collections import defaultdict, OrderedDict @@ -584,13 +586,16 @@ def write_raw_bids(raw, bids_fname, output_path, kind='meg', events_data=None, if not hasattr(raw, 'filenames'): raise ValueError('raw.filenames is missing.') - _, ext = _parse_ext(raw.filenames[0], verbose=verbose) raw_fname = raw.filenames[0] + if '.ds' in op.dirname(raw.filenames[0]): + raw_fname = op.dirname(raw.filenames[0]) + if '.eeg' in raw.filenames[0]: + raw_fname = raw_fname[:-4] + '.vhdr' + _, ext = _parse_ext(raw_fname, verbose=verbose) params = _parse_bids_filename(bids_fname) subject_id, session_id = params['sub'], params['ses'] acquisition, task, run = params['acq'], params['task'], params['run'] - fname = os.path.join(output_path, bids_fname) data_path = make_bids_folders(subject=subject_id, session=session_id, kind=kind, root=output_path, @@ -621,6 +626,10 @@ def write_raw_bids(raw, bids_fname, output_path, kind='meg', events_data=None, channels_fname = make_bids_filename( subject=subject_id, session=session_id, task=task, run=run, acquisition=acquisition, suffix='channels.tsv', prefix=data_path) + if ext not in ['.fif', '.ds', '.vhdr', '.edf', '.bdf', '.set', '.fdt', + '.cnt']: + bids_raw_folder = bids_fname.split('.')[0] + bids_fname = op.join(bids_raw_folder, bids_fname) # Read in Raw object and extract metadata from Raw object if needed orient = ORIENTATION.get(ext, 'n/a') @@ -659,7 +668,7 @@ def write_raw_bids(raw, bids_fname, output_path, kind='meg', events_data=None, _mkdir_p(os.path.dirname(bids_fname)) if verbose: - print('Writing data files to %s' % fname) + print('Writing data files to %s' % bids_fname) if ext not in ALLOWED_EXTENSIONS: raise ValueError('ext must be in %s, got %s' diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index a0bee0f87..8cad30535 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -20,7 +20,8 @@ from mne.utils import _TempDir, run_subprocess from mne.io.constants import FIFF -from mne_bids import raw_to_bids, make_bids_filename, make_bids_folders +from mne_bids import make_bids_filename, make_bids_folders, write_raw_bids +from mne_bids.io import read_raw base_path = op.join(op.dirname(mne.__file__), 'io') subject_id = '01' @@ -31,6 +32,10 @@ run2 = '02' task = 'testing' +partial_bids_fname = make_bids_filename( + subject=subject_id, session=session_id, run=run, acquisition=acq, + task=task) + # for windows, shell = True is needed # to call npm, bids-validator etc. # see: https://stackoverflow.com/questions/ @@ -54,10 +59,10 @@ def test_fif(): events_fname = op.join(data_path, 'MEG', 'sample', 'sample_audvis_trunc_raw-eve.fif') - raw_to_bids(subject_id=subject_id, session_id=session_id, run=run, - acquisition=acq, task=task, raw_file=raw_fname, - events_data=events_fname, output_path=output_path, - event_id=event_id, overwrite=False) + raw = mne.io.read_raw_fif(raw_fname) + bids_fname = partial_bids_fname + '_meg.fif' + write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, + event_id=event_id, overwrite=False) # give the raw object some fake participant data raw = mne.io.read_raw_fif(raw_fname) @@ -67,25 +72,21 @@ def test_fif(): data_path2 = _TempDir() raw_fname2 = op.join(data_path2, 'sample_audvis_raw.fif') raw.save(raw_fname2) - raw_to_bids(subject_id=subject_id2, run=run, task=task, acquisition=acq, - session_id=session_id, raw_file=raw_fname2, - events_data=events_fname, output_path=output_path, - event_id=event_id, overwrite=False) + + bids_fname = bids_fname.replace(subject_id, subject_id2) + write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, + event_id=event_id, overwrite=False) # check that the overwrite parameters work correctly for the participant # data # change the gender but don't force overwrite. raw.info['subject_info'] = {'his_id': subject_id2, 'birthday': (1994, 1, 26), 'sex': 2} with pytest.raises(OSError, match="already exists"): - raw_to_bids(subject_id=subject_id2, run=run, task=task, - acquisition=acq, session_id=session_id, raw_file=raw, - events_data=events_fname, output_path=output_path, - event_id=event_id, overwrite=False) + write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, + event_id=event_id, overwrite=False) # now force the overwrite - raw_to_bids(subject_id=subject_id2, run=run, task=task, - acquisition=acq, session_id=session_id, raw_file=raw, - events_data=events_fname, output_path=output_path, - event_id=event_id, overwrite=True) + write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, + event_id=event_id, overwrite=True) cmd = ['bids-validator', output_path] run_subprocess(cmd, shell=shell) @@ -103,11 +104,11 @@ def test_kit(): headshape_fname = op.join(data_path, 'test_hsp.txt') event_id = dict(cond=1) - raw_to_bids(subject_id=subject_id, session_id=session_id, run=run, - task=task, acquisition=acq, raw_file=raw_fname, - events_data=events_fname, event_id=event_id, hpi=hpi_fname, - electrode=electrode_fname, hsp=headshape_fname, - output_path=output_path, overwrite=False) + bids_fname = partial_bids_fname + '_meg.sqd' + raw = read_raw(raw_fname, hpi=hpi_fname, electrode=electrode_fname, + hsp=headshape_fname) + write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, + event_id=event_id, overwrite=False) cmd = ['bids-validator', output_path] run_subprocess(cmd, shell=shell) assert op.exists(op.join(output_path, 'participants.tsv')) @@ -146,9 +147,9 @@ def test_ctf(): data_path = op.join(testing.data_path(download=False), 'CTF') raw_fname = op.join(data_path, 'testdata_ctf.ds') - raw_to_bids(subject_id=subject_id, session_id=session_id, run=run, - task=task, acquisition=acq, raw_file=raw_fname, - output_path=output_path, overwrite=False) + bids_fname = partial_bids_fname + '_meg.ds' + raw = mne.io.read_raw_ctf(raw_fname) + write_raw_bids(raw, bids_fname, output_path=output_path) cmd = ['bids-validator', output_path] run_subprocess(cmd, shell=shell) @@ -169,10 +170,9 @@ def test_bti(): config_fname = op.join(data_path, 'test_config_linux') headshape_fname = op.join(data_path, 'test_hs_linux') - raw_to_bids(subject_id=subject_id, session_id=session_id, run=run, - task=task, acquisition=acq, raw_file=raw_fname, - config=config_fname, hsp=headshape_fname, - output_path=output_path, verbose=True, overwrite=False) + bids_fname = partial_bids_fname + '_meg.pdf' + raw = read_raw(raw_fname, config=config_fname, hsp=headshape_fname) + write_raw_bids(raw, bids_fname, output_path, verbose=True) assert op.exists(op.join(output_path, 'participants.tsv')) @@ -188,9 +188,9 @@ def test_vhdr(): data_path = op.join(base_path, 'brainvision', 'tests', 'data') raw_fname = op.join(data_path, 'test.vhdr') - raw_to_bids(subject_id=subject_id, session_id=session_id, run=run, - task=task, acquisition=acq, raw_file=raw_fname, - output_path=output_path, overwrite=False, kind='eeg') + bids_fname = partial_bids_fname + '_eeg.vhdr' + raw = read_raw(raw_fname) + write_raw_bids(raw, bids_fname, output_path, overwrite=False, kind='eeg') cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) @@ -215,12 +215,10 @@ def test_edf(): raw.rename_channels({raw.info['ch_names'][1]: 'EMG'}) raw.set_channel_types({'EMG': 'emg'}) - raw_to_bids(subject_id=subject_id, session_id=session_id, run=run, - task=task, acquisition=acq, raw_file=raw, - output_path=output_path, overwrite=False, kind='eeg') - raw_to_bids(subject_id=subject_id, session_id=session_id, run=run2, - task=task, acquisition=acq, raw_file=raw, - output_path=output_path, overwrite=True, kind='eeg') + bids_fname = partial_bids_fname + '_eeg.edf' + write_raw_bids(raw, bids_fname, output_path, overwrite=False, kind='eeg') + bids_fname.replace('run-01', 'run-%02d' % run2) + write_raw_bids(raw, bids_fname, output_path, overwrite=True, kind='eeg') cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) @@ -247,9 +245,10 @@ def test_bdf(): data_path = op.join(base_path, 'edf', 'tests', 'data') raw_fname = op.join(data_path, 'test.bdf') - raw_to_bids(subject_id=subject_id, session_id=session_id, run=run, - task=task, acquisition=acq, raw_file=raw_fname, - output_path=output_path, overwrite=False, kind='eeg') + bids_fname = partial_bids_fname + '_eeg.bdf' + raw = read_raw(raw_fname) + write_raw_bids(raw, bids_fname, output_path, overwrite=False, + kind='eeg') cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) @@ -262,9 +261,9 @@ def test_set(): data_path = op.join(testing.data_path(), 'EEGLAB') raw_fname = op.join(data_path, 'test_raw_onefile.set') - raw_to_bids(subject_id=subject_id, session_id=session_id, run=run, - task=task, acquisition=acq, raw_file=raw_fname, - output_path=output_path, overwrite=False, kind='eeg') + bids_fname = partial_bids_fname + '_eeg.set' + raw = read_raw(raw_fname) + write_raw_bids(raw, bids_fname, output_path, overwrite=False, kind='eeg') cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) @@ -279,9 +278,8 @@ def test_set(): data_path = op.join(testing.data_path(), 'EEGLAB') raw_fname = op.join(data_path, 'test_raw.set') - raw_to_bids(subject_id=subject_id, session_id=session_id, run=run, - task=task, acquisition=acq, raw_file=raw_fname, - output_path=output_path, overwrite=False, kind='eeg') + raw = read_raw(raw_fname) + write_raw_bids(raw, bids_fname, output_path, overwrite=False, kind='eeg') cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) @@ -293,9 +291,9 @@ def test_cnt(): data_path = op.join(testing.data_path(), 'CNT') raw_fname = op.join(data_path, 'scan41_short.cnt') - raw_to_bids(subject_id=subject_id, session_id=session_id, run=run, - task=task, acquisition=acq, raw_file=raw_fname, - output_path=output_path, overwrite=False, kind='eeg') + bids_fname = partial_bids_fname + '_eeg.cnt' + raw = read_raw(raw_fname) + write_raw_bids(raw, bids_fname, output_path, kind='eeg') cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) From e56fbc502954f67c4f36343179699ee0bfe87c79 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Mon, 15 Oct 2018 14:24:38 -0400 Subject: [PATCH 03/38] TST commands + io --- mne_bids/commands/mne_bids_raw_to_bids.py | 28 ++++++++++++++--------- mne_bids/mne_bids.py | 5 ++-- mne_bids/tests/test_commands.py | 2 +- mne_bids/tests/test_io.py | 4 ++-- 4 files changed, 22 insertions(+), 17 deletions(-) diff --git a/mne_bids/commands/mne_bids_raw_to_bids.py b/mne_bids/commands/mne_bids_raw_to_bids.py index 6dab927c6..337c9e6e8 100644 --- a/mne_bids/commands/mne_bids_raw_to_bids.py +++ b/mne_bids/commands/mne_bids_raw_to_bids.py @@ -4,11 +4,12 @@ """Command line interface for mne_bids. example usage: $ mne_bids raw_to_bids --subject_id sub01 --task rest ---raw_file data.edf --output_path new_path +--raw_fname data.edf --output_path new_path """ - -from mne_bids import raw_to_bids +import os +from mne_bids import write_raw_bids, make_bids_filename +from mne_bids.io import read_raw def run(): @@ -24,7 +25,7 @@ def run(): parser.add_option('--task', dest='task', help='Name of the task the data is based on.', metavar='t') - parser.add_option('--raw_file', dest='raw_file', + parser.add_option('--raw_fname', dest='raw_fname', help='The path to the raw MEG file.', metavar='r') parser.add_option('--output_path', dest='output_path', help='The path of the BIDS compatible folder.', @@ -35,6 +36,9 @@ def run(): parser.add_option('--run', dest='run', help='The run number for this dataset.', metavar='run') + parser.add_option('--acq', dest='acq', + help='The acquisition parameter.', + metavar='acq') parser.add_option('--kind', dest='kind', help='The kind of data being converted.', metavar='k') @@ -58,13 +62,15 @@ def run(): opt, args = parser.parse_args() - raw_to_bids(subject_id=opt.subject_id, task=opt.task, - raw_file=opt.raw_file, output_path=opt.output_path, - session_id=opt.session_id, run=opt.run, kind=opt.kind, - events_data=opt.events_data, event_id=opt.event_id, - hpi=opt.hpi, electrode=opt.electrode, hsp=opt.hsp, - config=opt.config, overwrite=opt.overwrite, - verbose=True) + suffix = '%s%s' % (opt.kind, os.path.splitext(opt.raw_fname)[1]) + bids_fname = make_bids_filename( + subject=opt.subject_id, session=opt.session_id, run=opt.run, + acquisition=opt.acq, task=opt.task, suffix=suffix) + raw = read_raw(opt.raw_fname, hpi=opt.hpi, electrode=opt.electrode, + hsp=opt.hsp, config=opt.config) + write_raw_bids(raw, bids_fname, opt.output_path, event_id=opt.event_id, + kind=opt.kind, events_data=opt.events_data, + overwrite=opt.overwrite, verbose=True) is_main = (__name__ == '__main__') diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 7176dc539..1b16e6cd0 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -581,10 +581,9 @@ def write_raw_bids(raw, bids_fname, output_path, kind='meg', events_data=None, raise ValueError('raw_file must be an instance of BaseRaw, ' 'got %s' % type(raw)) - # We got a raw mne object, get back the filename if possible - # Assume that if no filename attr exists, it's a fif file. if not hasattr(raw, 'filenames'): - raise ValueError('raw.filenames is missing.') + raise ValueError('raw.filenames is missing. Please set raw.filenames' + 'as a list with the full path of original raw file.') raw_fname = raw.filenames[0] if '.ds' in op.dirname(raw.filenames[0]): diff --git a/mne_bids/tests/test_commands.py b/mne_bids/tests/test_commands.py index a33e43210..2f9cafa0f 100644 --- a/mne_bids/tests/test_commands.py +++ b/mne_bids/tests/test_commands.py @@ -28,7 +28,7 @@ def test_raw_to_bids(): raw_fname = op.join(data_path, 'MEG', 'sample', 'sample_audvis_trunc_raw.fif') # check_usage(mne_bids_raw_to_bids) - with ArgvSetter(('--subject_id', subject_id, '--task', task, '--raw_file', + with ArgvSetter(('--subject_id', subject_id, '--task', task, '--raw_fname', raw_fname, '--output_path', output_path)): mne_bids_raw_to_bids.run() diff --git a/mne_bids/tests/test_io.py b/mne_bids/tests/test_io.py index a907e16ca..3f4b14f61 100644 --- a/mne_bids/tests/test_io.py +++ b/mne_bids/tests/test_io.py @@ -4,7 +4,7 @@ # License: BSD (3-clause) import pytest -from mne_bids.io import _parse_ext, _read_raw +from mne_bids.io import _parse_ext, read_raw def test_parse_ext(): @@ -26,4 +26,4 @@ def test_read_raw(): # Use a file ending that does not exist f = 'file.bogus' with pytest.raises(ValueError): - _read_raw(f) + read_raw(f) From 2329d6f47d0b7312e848919a9d971965a054eda5 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Mon, 15 Oct 2018 14:32:40 -0400 Subject: [PATCH 04/38] DOC update example --- examples/convert_group_studies.py | 22 +++++++++++----------- mne_bids/mne_bids.py | 4 ++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/examples/convert_group_studies.py b/examples/convert_group_studies.py index a43440c8a..0f65804c1 100644 --- a/examples/convert_group_studies.py +++ b/examples/convert_group_studies.py @@ -24,8 +24,9 @@ # Let us import ``mne_bids`` import os.path as op + +import mne from mne_bids import write_raw_bids, make_bids_filename -from mne_bids.io import read_raw from mne_bids.datasets import fetch_faces_data from mne_bids.utils import print_dir_tree @@ -64,16 +65,15 @@ for subject_id in subject_ids: subject = 'sub%03d' % subject_id for run in runs: - fname = op.join(data_path, repo, subject, 'MEG', - 'run_%02d_raw.fif' % run) - - raw = read_raw(fname) - bids_fname = make_bids_filename( - subject_id='%02d' % subject_id, session='01', run=run, - task='VisualFaces', suffix='meg.fif') - - # Make it BIDS compatible - write_raw_bids(raw, bids_fname, output_path, event_id=event_id) + raw_fname = op.join(data_path, repo, subject, 'MEG', + 'run_%02d_raw.fif' % run) + raw = mne.io.read_raw_fif(raw_fname) + + bids_fname = make_bids_filename(subject='%02d' % subject_id, run=run, + session='01', task='VisualFaces', + suffix='meg.fif') + write_raw_bids(raw, bids_fname, output_path, event_id=event_id, + overwrite=True) ############################################################################### # Now let's see the structure of the BIDS folder we created. diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 1b16e6cd0..4ddf9fe70 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -544,6 +544,10 @@ def write_raw_bids(raw, bids_fname, output_path, kind='meg', events_data=None, The file path of the BIDS compatible raw file. In the case of multifile systems (e.g., vhdr, .ds etc.), this is the path to a folder containing all the files. + output_path : str + The path of the root of the BIDS compatible folder. The session and + subject specific folders will be populated automatically by parsing + bids_fname. kind : str, one of ('meg', 'eeg', 'ieeg') The kind of data being converted. Defaults to "meg". events_data : str | array | None From e12fc4346c96375c80a530136a1141a3866ea2f5 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Mon, 15 Oct 2018 15:23:06 -0400 Subject: [PATCH 05/38] TST more fixes --- mne_bids/mne_bids.py | 5 ++++- mne_bids/tests/test_mne_bids.py | 3 ++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 4ddf9fe70..5dc65f06b 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -22,7 +22,6 @@ from mne.io.pick import channel_type from mne.io import BaseRaw from mne.channels.channels import _unit2human -from mne.externals.six import string_types from mne.utils import check_version from datetime import datetime @@ -596,6 +595,10 @@ def write_raw_bids(raw, bids_fname, output_path, kind='meg', events_data=None, raw_fname = raw_fname[:-4] + '.vhdr' _, ext = _parse_ext(raw_fname, verbose=verbose) + if ext in ('.set', '.fdt') and not check_version('mne', '0.17.dev'): + raise ValueError('MNE version > 0.17dev0 is required for converting' + ' set files. Please update your installation of mne') + params = _parse_bids_filename(bids_fname) subject_id, session_id = params['sub'], params['ses'] acquisition, task, run = params['acq'], params['task'], params['run'] diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index 8cad30535..b8027897e 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -17,7 +17,7 @@ import pandas as pd import mne from mne.datasets import testing -from mne.utils import _TempDir, run_subprocess +from mne.utils import _TempDir, run_subprocess, requires_version from mne.io.constants import FIFF from mne_bids import make_bids_filename, make_bids_folders, write_raw_bids @@ -254,6 +254,7 @@ def test_bdf(): run_subprocess(cmd, shell=shell) +@requires_version('mne', '0.17dev0') def test_set(): """Test raw_to_bids conversion for EEGLAB data.""" # standalone .set file From 3893742d501b688c11357725b9ba2d1ecb16792a Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Mon, 15 Oct 2018 16:38:21 -0400 Subject: [PATCH 06/38] ENH: don't need kind in write_raw_bids --- examples/convert_eeg_to_bids.py | 10 +++------- mne_bids/commands/mne_bids_raw_to_bids.py | 4 ++-- mne_bids/mne_bids.py | 8 +++----- mne_bids/tests/test_mne_bids.py | 15 +++++++-------- mne_bids/utils.py | 8 +++++++- 5 files changed, 22 insertions(+), 23 deletions(-) diff --git a/examples/convert_eeg_to_bids.py b/examples/convert_eeg_to_bids.py index f5e592f69..cc75bf3e8 100644 --- a/examples/convert_eeg_to_bids.py +++ b/examples/convert_eeg_to_bids.py @@ -117,9 +117,6 @@ ############################################################################### # Now we just need to specify a few more EEG details to get something sensible: -# First, tell `MNE-BIDS` that it is working with EEG data: -kind = 'eeg' - # Brief description of the event markers present in the data. This will become # the `trial_type` column in our BIDS `events.tsv`. We know about the event # meaning from the documentation on PhysioBank @@ -128,8 +125,7 @@ # Now convert our data to be in a new BIDS dataset. bids_fname = make_bids_filename(subject=subject_id, task=task, suffix='eeg.edf') -write_raw_bids(raw_file, bids_fname, output_path, kind=kind, - event_id=trial_type) +write_raw_bids(raw_file, bids_fname, output_path, event_id=trial_type) ############################################################################### # What does our fresh BIDS directory look like? @@ -163,8 +159,8 @@ bids_fname = make_bids_filename( subject='{:03}'.format(subj_idx), task=task_names[task_idx], run=run_mapping[task_idx], suffix='eeg.edf') - write_raw_bids(raw, bids_fname, output_path, kind=kind, - event_id=trial_type, overwrite=True) + write_raw_bids(raw, bids_fname, output_path, event_id=trial_type, + overwrite=True) ############################################################################### # Step 3: Check and compare with standard diff --git a/mne_bids/commands/mne_bids_raw_to_bids.py b/mne_bids/commands/mne_bids_raw_to_bids.py index 337c9e6e8..76844304b 100644 --- a/mne_bids/commands/mne_bids_raw_to_bids.py +++ b/mne_bids/commands/mne_bids_raw_to_bids.py @@ -69,8 +69,8 @@ def run(): raw = read_raw(opt.raw_fname, hpi=opt.hpi, electrode=opt.electrode, hsp=opt.hsp, config=opt.config) write_raw_bids(raw, bids_fname, opt.output_path, event_id=opt.event_id, - kind=opt.kind, events_data=opt.events_data, - overwrite=opt.overwrite, verbose=True) + events_data=opt.events_data, overwrite=opt.overwrite, + verbose=True) is_main = (__name__ == '__main__') diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 5dc65f06b..91802b2e0 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -529,7 +529,7 @@ def _sidecar_json(raw, task, manufacturer, fname, kind, eeg_ref=None, return fname -def write_raw_bids(raw, bids_fname, output_path, kind='meg', events_data=None, +def write_raw_bids(raw, bids_fname, output_path, events_data=None, event_id=None, eeg_ref=None, eeg_gnd=None, overwrite=False, verbose=True): """Walk over a folder of files and create BIDS compatible folder. @@ -547,8 +547,6 @@ def write_raw_bids(raw, bids_fname, output_path, kind='meg', events_data=None, The path of the root of the BIDS compatible folder. The session and subject specific folders will be populated automatically by parsing bids_fname. - kind : str, one of ('meg', 'eeg', 'ieeg') - The kind of data being converted. Defaults to "meg". events_data : str | array | None The events file. If a string, a path to the events file. If an array, the MNE events array (shape n_events, 3). If None, events will be @@ -599,8 +597,8 @@ def write_raw_bids(raw, bids_fname, output_path, kind='meg', events_data=None, raise ValueError('MNE version > 0.17dev0 is required for converting' ' set files. Please update your installation of mne') - params = _parse_bids_filename(bids_fname) - subject_id, session_id = params['sub'], params['ses'] + params = _parse_bids_filename(bids_fname, verbose) + subject_id, session_id, kind = params['sub'], params['ses'], params['kind'] acquisition, task, run = params['acq'], params['task'], params['run'] data_path = make_bids_folders(subject=subject_id, session=session_id, diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index b8027897e..d80d81504 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -190,7 +190,7 @@ def test_vhdr(): bids_fname = partial_bids_fname + '_eeg.vhdr' raw = read_raw(raw_fname) - write_raw_bids(raw, bids_fname, output_path, overwrite=False, kind='eeg') + write_raw_bids(raw, bids_fname, output_path, overwrite=False) cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) @@ -216,9 +216,9 @@ def test_edf(): raw.set_channel_types({'EMG': 'emg'}) bids_fname = partial_bids_fname + '_eeg.edf' - write_raw_bids(raw, bids_fname, output_path, overwrite=False, kind='eeg') + write_raw_bids(raw, bids_fname, output_path, overwrite=False) bids_fname.replace('run-01', 'run-%02d' % run2) - write_raw_bids(raw, bids_fname, output_path, overwrite=True, kind='eeg') + write_raw_bids(raw, bids_fname, output_path, overwrite=True) cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) @@ -247,8 +247,7 @@ def test_bdf(): bids_fname = partial_bids_fname + '_eeg.bdf' raw = read_raw(raw_fname) - write_raw_bids(raw, bids_fname, output_path, overwrite=False, - kind='eeg') + write_raw_bids(raw, bids_fname, output_path, overwrite=False) cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) @@ -264,7 +263,7 @@ def test_set(): bids_fname = partial_bids_fname + '_eeg.set' raw = read_raw(raw_fname) - write_raw_bids(raw, bids_fname, output_path, overwrite=False, kind='eeg') + write_raw_bids(raw, bids_fname, output_path, overwrite=False) cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) @@ -280,7 +279,7 @@ def test_set(): raw_fname = op.join(data_path, 'test_raw.set') raw = read_raw(raw_fname) - write_raw_bids(raw, bids_fname, output_path, overwrite=False, kind='eeg') + write_raw_bids(raw, bids_fname, output_path, overwrite=False) cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) @@ -294,7 +293,7 @@ def test_cnt(): bids_fname = partial_bids_fname + '_eeg.cnt' raw = read_raw(raw_fname) - write_raw_bids(raw, bids_fname, output_path, kind='eeg') + write_raw_bids(raw, bids_fname, output_path) cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) diff --git a/mne_bids/utils.py b/mne_bids/utils.py index e2ec9a166..08a79e932 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -62,7 +62,7 @@ def _mkdir_p(path, overwrite=False, verbose=False): raise -def _parse_bids_filename(fname): +def _parse_bids_filename(fname, verbose): """Get dict from BIDS fname.""" params = dict(sub=None, ses=None, task=None, acq=None, run=None, proc=None, recording=None, space=None) @@ -73,6 +73,12 @@ def _parse_bids_filename(fname): raise KeyError('Unexpected entity found in filename %s' % entity) params[key] = value params['suffix'] = entities[-1] + kind, ext = _parse_ext(entities[-1], verbose=verbose) + params['kind'], params['ext'] = kind, ext + if kind not in ('eeg', 'meg', 'ieeg'): + raise ValueError('Please check the filename of your BIDS file.' + 'It should have _eeg, _meg, or _ieeg as suffix.' + 'Got %s' % kind) return params From 8c6ec21a1eb237dede19c1b337094b7f10b980f8 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Mon, 15 Oct 2018 16:40:33 -0400 Subject: [PATCH 07/38] MAINT change variable names --- mne_bids/mne_bids.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 91802b2e0..e9a1a82a3 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -620,10 +620,10 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, coordsystem_fname = make_bids_filename( subject=subject_id, session=session_id, acquisition=acquisition, suffix='coordsystem.json', prefix=data_path) - data_meta_fname = make_bids_filename( + sidecar_fname = make_bids_filename( subject=subject_id, session=session_id, task=task, run=run, acquisition=acquisition, suffix='%s.json' % kind, prefix=data_path) - events_tsv_fname = make_bids_filename( + events_fname = make_bids_filename( subject=subject_id, session=session_id, task=task, acquisition=acquisition, run=run, suffix='events.tsv', prefix=data_path) @@ -655,11 +655,10 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, events = _read_events(events_data, raw) if len(events) > 0: - _events_tsv(events, raw, events_tsv_fname, event_id, overwrite, - verbose) + _events_tsv(events, raw, events_fname, event_id, overwrite, verbose) make_dataset_description(output_path, name=" ", verbose=verbose) - _sidecar_json(raw, task, manufacturer, data_meta_fname, kind, eeg_ref, + _sidecar_json(raw, task, manufacturer, sidecar_fname, kind, eeg_ref, eeg_gnd, overwrite, verbose) _channels_tsv(raw, channels_fname, overwrite, verbose) From bb8b95559e97c20d562b0cce7dbe9a4f79932731 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Tue, 16 Oct 2018 17:43:37 -0400 Subject: [PATCH 08/38] Take care of hpi --- examples/convert_eeg_to_bids.py | 5 ++-- mne_bids/io.py | 5 ++++ mne_bids/mne_bids.py | 45 ++++++++++----------------------- mne_bids/tests/test_mne_bids.py | 26 ++++++++----------- 4 files changed, 33 insertions(+), 48 deletions(-) diff --git a/examples/convert_eeg_to_bids.py b/examples/convert_eeg_to_bids.py index cc75bf3e8..330f7294f 100644 --- a/examples/convert_eeg_to_bids.py +++ b/examples/convert_eeg_to_bids.py @@ -174,8 +174,9 @@ # initial `dataset_description` on top! # # Now it's time to manually check the BIDS directory and the meta files to add -# all the information that MNE-BIDS could not infer. These places are marked -# with "n/a". +# all the information that MNE-BIDS could not infer. For instance, you must +# describe EEGReference and EEGGround yourself. It's easy to find these by +# searching for "n/a" in the sidecar files. # # Remember that there is a convenient javascript tool to validate all your BIDS # directories called the "BIDS-validator", available as a web version and a diff --git a/mne_bids/io.py b/mne_bids/io.py index 70f07c61c..4df8c3bde 100644 --- a/mne_bids/io.py +++ b/mne_bids/io.py @@ -41,8 +41,13 @@ def read_raw(raw_fname, electrode=None, hsp=None, hpi=None, config=None, # -------------- # KIT systems if ext in ['.con', '.sqd']: + if isinstance(hpi, list): + # No currently accepted way to name multiple marker files. See: + # https://github.com/bids-standard/bids-specification/issues/45 + raise ValueError('Only single marker coils supported currently') raw = io.read_raw_kit(raw_fname, elp=electrode, hsp=hsp, mrk=hpi, preload=False) + raw._filenames.append(hpi) # Neuromag or converted-to-fif systems elif ext in ['.fif']: diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index e9a1a82a3..be83dcc5b 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -401,8 +401,8 @@ def _coordsystem_json(raw, unit, orient, manufacturer, fname, return fname -def _sidecar_json(raw, task, manufacturer, fname, kind, eeg_ref=None, - eeg_gnd=None, overwrite=False, verbose=True): +def _sidecar_json(raw, task, manufacturer, fname, kind, overwrite=False, + verbose=True): """Create a sidecar json file depending on the kind and save it. The sidecar json file provides meta data about the data of a certain kind. @@ -420,11 +420,6 @@ def _sidecar_json(raw, task, manufacturer, fname, kind, eeg_ref=None, Filename to save the sidecar json to. kind : str Type of the data as in ALLOWED_KINDS. - eeg_ref : str - Description of the type of reference used and (when applicable) of - location of the reference electrode. Defaults to None. - eeg_gnd : str - Description of the location of the ground electrode. Defaults to None. overwrite : bool Whether to overwrite the existing file. Defaults to False. @@ -438,11 +433,6 @@ def _sidecar_json(raw, task, manufacturer, fname, kind, eeg_ref=None, warn('No line frequency found, defaulting to 50 Hz') powerlinefrequency = 50 - if not eeg_ref: - eeg_ref = 'n/a' - if not eeg_gnd: - eeg_gnd = 'n/a' - if isinstance(raw, BaseRaw): rec_type = 'continuous' elif isinstance(raw, Epochs): @@ -493,8 +483,8 @@ def _sidecar_json(raw, task, manufacturer, fname, kind, eeg_ref=None, ('MEGChannelCount', n_megchan), ('MEGREFChannelCount', n_megrefchan)] ch_info_json_eeg = [ - ('EEGReference', eeg_ref), - ('EEGGround', eeg_gnd), + ('EEGReference', 'n/a'), + ('EEGGround', 'n/a'), ('EEGPlacementScheme', _infer_eeg_placement_scheme(raw)), ('Manufacturer', manufacturer)] ch_info_json_ieeg = [ @@ -530,8 +520,7 @@ def _sidecar_json(raw, task, manufacturer, fname, kind, eeg_ref=None, def write_raw_bids(raw, bids_fname, output_path, events_data=None, - event_id=None, eeg_ref=None, eeg_gnd=None, overwrite=False, - verbose=True): + event_id=None, overwrite=False, verbose=True): """Walk over a folder of files and create BIDS compatible folder. Parameters @@ -541,8 +530,10 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, file. Otherwise it must be an instance of mne.Raw bids_fname : str The file path of the BIDS compatible raw file. In the case of multifile - systems (e.g., vhdr, .ds etc.), this is the path to a folder containing - all the files. + systems (e.g., .ds), this is the path to a folder containing + the files or (e.g., .vhdr or .set), the path to the principal file. + E.g.: sub-01_ses-01_task-testing_acq-01_run-01_meg.fif + Typically, this can be generated using make_bids_filename. output_path : str The path of the root of the BIDS compatible folder. The session and subject specific folders will be populated automatically by parsing @@ -553,11 +544,6 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, inferred from the stim channel using `mne.find_events`. event_id : dict | None The event id dict used to create a 'trial_type' column in events.tsv - eeg_ref : str - Description of the type of reference used and (when applicable) of - location of the reference electrode. Defaults to None. - eeg_gnd : str - Description of the location of the ground electrode. Defaults to None. overwrite : bool Whether to overwrite existing files or data in files. Defaults to False. @@ -658,8 +644,8 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, _events_tsv(events, raw, events_fname, event_id, overwrite, verbose) make_dataset_description(output_path, name=" ", verbose=verbose) - _sidecar_json(raw, task, manufacturer, sidecar_fname, kind, eeg_ref, - eeg_gnd, overwrite, verbose) + _sidecar_json(raw, task, manufacturer, sidecar_fname, kind, overwrite, + verbose) _channels_tsv(raw, channels_fname, overwrite, verbose) # set the raw file name to now be the absolute path to ensure the files @@ -709,16 +695,13 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, else: sh.copyfile(raw_fname, bids_fname) # KIT data requires the marker file to be copied over too - if hpi is not None: - if isinstance(hpi, list): - # No currently accepted way to name multiple marker files. See: - # https://github.com/bids-standard/bids-specification/issues/45 - raise ValueError('Only single marker coils supported currently') + if manufacturer == 'KIT/Yokogawa' and len(raw.filenames) > 1: + hpi = raw.filenames[-1] _, marker_ext = _parse_ext(hpi) marker_fname = make_bids_filename( subject=subject_id, session=session_id, task=task, run=run, acquisition=acquisition, suffix='markers%s' % marker_ext, - prefix=os.path.join(data_path, raw_folder)) + prefix=os.path.join(data_path, bids_raw_folder)) sh.copyfile(hpi, marker_fname) return output_path diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index d80d81504..ae9fa08f8 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -48,7 +48,7 @@ # MEG Tests # --------- def test_fif(): - """Test functionality of the raw_to_bids conversion for Neuromag data.""" + """Test functionality of the write_raw_bids conversion for fif.""" output_path = _TempDir() data_path = testing.data_path() raw_fname = op.join(data_path, 'MEG', 'sample', @@ -94,7 +94,7 @@ def test_fif(): def test_kit(): - """Test functionality of the raw_to_bids conversion for KIT data.""" + """Test functionality of the write_raw_bids conversion for KIT data.""" output_path = _TempDir() data_path = op.join(base_path, 'kit', 'tests', 'data') raw_fname = op.join(data_path, 'test.sqd') @@ -133,16 +133,12 @@ def test_kit(): # check for error if there are multiple marker coils specified with pytest.raises(ValueError): - raw_to_bids(subject_id=subject_id, session_id=session_id, run=run, - task=task, acquisition=acq, raw_file=raw_fname, - events_data=events_fname, event_id=event_id, - hpi=[hpi_fname, hpi_fname], electrode=electrode_fname, - hsp=headshape_fname, output_path=output_path, - overwrite=True) + raw = read_raw(raw_fname, hpi=[hpi_fname, hpi_fname], + electrode=electrode_fname, hsp=headshape_fname) def test_ctf(): - """Test functionality of the raw_to_bids conversion for CTF data.""" + """Test functionality of the write_raw_bids conversion for CTF data.""" output_path = _TempDir() data_path = op.join(testing.data_path(download=False), 'CTF') raw_fname = op.join(data_path, 'testdata_ctf.ds') @@ -163,7 +159,7 @@ def test_ctf(): def test_bti(): - """Test functionality of the raw_to_bids conversion for BTi data.""" + """Test functionality of the write_raw_bids conversion for BTi data.""" output_path = _TempDir() data_path = op.join(base_path, 'bti', 'tests', 'data') raw_fname = op.join(data_path, 'test_pdf_linux') @@ -183,7 +179,7 @@ def test_bti(): # EEG Tests # --------- def test_vhdr(): - """Test raw_to_bids conversion for BrainVision data.""" + """Test write_raw_bids conversion for BrainVision data.""" output_path = _TempDir() data_path = op.join(base_path, 'brainvision', 'tests', 'data') raw_fname = op.join(data_path, 'test.vhdr') @@ -204,7 +200,7 @@ def test_vhdr(): def test_edf(): - """Test raw_to_bids conversion for European Data Format data.""" + """Test write_raw_bids conversion for European Data Format data.""" output_path = _TempDir() data_path = op.join(testing.data_path(), 'EDF') raw_fname = op.join(data_path, 'test_reduced.edf') @@ -240,7 +236,7 @@ def test_edf(): def test_bdf(): - """Test raw_to_bids conversion for Biosemi data.""" + """Test write_raw_bids conversion for Biosemi data.""" output_path = _TempDir() data_path = op.join(base_path, 'edf', 'tests', 'data') raw_fname = op.join(data_path, 'test.bdf') @@ -255,7 +251,7 @@ def test_bdf(): @requires_version('mne', '0.17dev0') def test_set(): - """Test raw_to_bids conversion for EEGLAB data.""" + """Test write_raw_bids conversion for EEGLAB data.""" # standalone .set file output_path = _TempDir() data_path = op.join(testing.data_path(), 'EEGLAB') @@ -286,7 +282,7 @@ def test_set(): def test_cnt(): - """Test raw_to_bids conversion for Neuroscan data.""" + """Test write_raw_bids conversion for Neuroscan data.""" output_path = _TempDir() data_path = op.join(testing.data_path(), 'CNT') raw_fname = op.join(data_path, 'scan41_short.cnt') From 9aebb3151dd391c57b5adf38630feb70e953c01f Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Thu, 18 Oct 2018 22:15:10 -0400 Subject: [PATCH 09/38] FIX for rebase after overwrite PR --- mne_bids/tests/test_mne_bids.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index ae9fa08f8..f483c1868 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -151,9 +151,7 @@ def test_ctf(): # test to check that running again with overwrite == False raises an error with pytest.raises(OSError, match="already exists"): - raw_to_bids(subject_id=subject_id, session_id=session_id, run=run, - task=task, acquisition=acq, raw_file=raw_fname, - output_path=output_path, overwrite=False) + write_raw_bids(raw, bids_fname, output_path=output_path) assert op.exists(op.join(output_path, 'participants.tsv')) @@ -212,8 +210,8 @@ def test_edf(): raw.set_channel_types({'EMG': 'emg'}) bids_fname = partial_bids_fname + '_eeg.edf' - write_raw_bids(raw, bids_fname, output_path, overwrite=False) - bids_fname.replace('run-01', 'run-%02d' % run2) + write_raw_bids(raw, bids_fname, output_path) + bids_fname = bids_fname.replace('run-01', 'run-%s' % run2) write_raw_bids(raw, bids_fname, output_path, overwrite=True) cmd = ['bids-validator', '--bep006', output_path] @@ -265,9 +263,8 @@ def test_set(): run_subprocess(cmd, shell=shell) with pytest.raises(OSError, match="already exists"): - raw_to_bids(subject_id=subject_id, session_id=session_id, run=run, - task=task, acquisition=acq, raw_file=raw_fname, - output_path=output_path, overwrite=False, kind='eeg') + write_raw_bids(raw, bids_fname, output_path=output_path, + overwrite=False) # .set with associated .fdt output_path = _TempDir() From b56b1c9c174380273adb68a6cb775b971cd7405c Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Fri, 19 Oct 2018 13:57:40 -0400 Subject: [PATCH 10/38] TST improve coverage --- mne_bids/tests/test_mne_bids.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index f483c1868..f2dfa037d 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -87,6 +87,22 @@ def test_fif(): # now force the overwrite write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, event_id=event_id, overwrite=True) + + with pytest.raises(ValueError, match='raw_file must be'): + write_raw_bids('blah', bids_fname, output_path) + + bids_fname = partial_bids_fname + '_xyz.fif' + with pytest.raises(ValueError, match='as suffix'): + write_raw_bids(raw, bids_fname, output_path) + + bids_fname = 'sub-01_ses-01_xyz-01_run-01_meg.fif' + with pytest.raises(KeyError, match='Unexpected entity'): + write_raw_bids(raw, bids_fname, output_path) + + del raw._filenames + with pytest.raises(ValueError, match='raw.filenames is missing'): + write_raw_bids(raw, bids_fname, output_path) + cmd = ['bids-validator', output_path] run_subprocess(cmd, shell=shell) From 5a445c5856c146211409b3a715ae1529ded8bcf1 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Fri, 19 Oct 2018 15:39:54 -0400 Subject: [PATCH 11/38] TST change eeglab checks --- mne_bids/io.py | 2 +- mne_bids/mne_bids.py | 26 ++++++++------------------ mne_bids/tests/test_mne_bids.py | 4 ++-- 3 files changed, 11 insertions(+), 21 deletions(-) diff --git a/mne_bids/io.py b/mne_bids/io.py index 4df8c3bde..040d976d5 100644 --- a/mne_bids/io.py +++ b/mne_bids/io.py @@ -13,7 +13,7 @@ allowed_extensions_eeg = ['.vhdr', # BrainVision, accompanied by .vmrk, .eeg '.edf', # European Data Format '.bdf', # Biosemi - '.set', '.fdt', # EEGLAB + '.set', # EEGLAB '.cnt', # Neuroscan ] diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index be83dcc5b..c5c6bf9e2 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -106,14 +106,9 @@ def _channels_tsv(raw, fname, overwrite=False, verbose=True): # get the manufacturer from the file in the Raw object manufacturer = None - if hasattr(raw, 'filenames'): - # XXX: Hack for EEGLAB bug in MNE-Python 0.16; fixed in MNE-Python - # 0.17, ... remove the hack after upgrading dependencies in MNE-BIDS - if raw.filenames[0] is None: # hack - ext = '.set' # hack - else: - _, ext = _parse_ext(raw.filenames[0], verbose=verbose) - manufacturer = MANUFACTURERS[ext] + + _, ext = _parse_ext(raw.filenames[0], verbose=verbose) + manufacturer = MANUFACTURERS[ext] ignored_indexes = [raw.ch_names.index(ch_name) for ch_name in raw.ch_names if ch_name in @@ -568,20 +563,16 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, raise ValueError('raw_file must be an instance of BaseRaw, ' 'got %s' % type(raw)) - if not hasattr(raw, 'filenames'): + if not hasattr(raw, 'filenames') or raw.filenames[0] is None: raise ValueError('raw.filenames is missing. Please set raw.filenames' 'as a list with the full path of original raw file.') + _, ext = _parse_ext(bids_fname, verbose=verbose) raw_fname = raw.filenames[0] if '.ds' in op.dirname(raw.filenames[0]): raw_fname = op.dirname(raw.filenames[0]) - if '.eeg' in raw.filenames[0]: - raw_fname = raw_fname[:-4] + '.vhdr' - _, ext = _parse_ext(raw_fname, verbose=verbose) - - if ext in ('.set', '.fdt') and not check_version('mne', '0.17.dev'): - raise ValueError('MNE version > 0.17dev0 is required for converting' - ' set files. Please update your installation of mne') + raw_fname = raw_fname.replace('.eeg', '.vhdr') + raw_fname = raw_fname.replace('.fdt', '.set') params = _parse_bids_filename(bids_fname, verbose) subject_id, session_id, kind = params['sub'], params['ses'], params['kind'] @@ -616,8 +607,7 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, channels_fname = make_bids_filename( subject=subject_id, session=session_id, task=task, run=run, acquisition=acquisition, suffix='channels.tsv', prefix=data_path) - if ext not in ['.fif', '.ds', '.vhdr', '.edf', '.bdf', '.set', '.fdt', - '.cnt']: + if ext not in ['.fif', '.ds', '.vhdr', '.edf', '.bdf', '.set', '.cnt']: bids_raw_folder = bids_fname.split('.')[0] bids_fname = op.join(bids_raw_folder, bids_fname) diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index f2dfa037d..6fc571623 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -17,7 +17,7 @@ import pandas as pd import mne from mne.datasets import testing -from mne.utils import _TempDir, run_subprocess, requires_version +from mne.utils import _TempDir, run_subprocess from mne.io.constants import FIFF from mne_bids import make_bids_filename, make_bids_folders, write_raw_bids @@ -263,7 +263,6 @@ def test_bdf(): run_subprocess(cmd, shell=shell) -@requires_version('mne', '0.17dev0') def test_set(): """Test write_raw_bids conversion for EEGLAB data.""" # standalone .set file @@ -273,6 +272,7 @@ def test_set(): bids_fname = partial_bids_fname + '_eeg.set' raw = read_raw(raw_fname) + raw._filenames = [raw_fname] write_raw_bids(raw, bids_fname, output_path, overwrite=False) cmd = ['bids-validator', '--bep006', output_path] From 208b815dc1b100b99c244e4dce3e1bf6e8f6b4b7 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Sat, 20 Oct 2018 23:37:56 -0400 Subject: [PATCH 12/38] DOC: Update readme.rst --- doc/index.rst | 21 ++++++++++++++++++--- examples/convert_mne_sample.py | 4 ++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/doc/index.rst b/doc/index.rst index 89b89b1bd..55bed30a0 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -13,21 +13,36 @@ Installation We recommend the `Anaconda Python distribution `_. To install ``mne_bids``, you first need to install its dependencies:: - $ pip install pandas mne + $ pip install pandas mne Then install mne_bids:: - $ pip install git+https://github.com/mne-tools/mne-bids.git#egg=mne-bids + $ pip install git+https://github.com/mne-tools/mne-bids.git#egg=mne-bids If you do not have admin privileges on the computer, use the ``--user`` flag with `pip`. To upgrade, use the ``--upgrade`` flag provided by `pip`. To check if everything worked fine, you can do:: - $ python -c 'import mne_bids' + $ python -c 'import mne_bids' and it should not give any error messages. +Quickstart +========== + +Currently, we support writing of BIDS datasets for MEG and EEG. Support for +iEEG is experimental at the moment. + +.. code:: python + + >>> from mne_bids import read_raw, write_raw_bids + >>> raw = read_raw('my_old_file.fif') + >>> write_raw_bids(raw, 'sub-01_ses-01_run-05_meg.fif', + output_path='.') + +Reading of BIDS data will also be supported in the next version. + Bug reports =========== diff --git a/examples/convert_mne_sample.py b/examples/convert_mne_sample.py index ca8f945da..21ef50691 100644 --- a/examples/convert_mne_sample.py +++ b/examples/convert_mne_sample.py @@ -29,14 +29,14 @@ event_id = {'Auditory/Left': 1, 'Auditory/Right': 2, 'Visual/Left': 3, 'Visual/Right': 4, 'Smiley': 5, 'Button': 32} -fname = op.join(data_path, 'MEG', 'sample', 'sample_audvis_raw.fif') +raw_fname = op.join(data_path, 'MEG', 'sample', 'sample_audvis_raw.fif') events_data = op.join(data_path, 'MEG', 'sample', 'sample_audvis_raw-eve.fif') output_path = op.join(data_path, '..', 'MNE-sample-data-bids') ############################################################################### # Finally, we specify the raw_file and events_data -raw = read_raw(fname) +raw = read_raw(raw_fname) bids_fname = make_bids_filename(subject='01', run='01', session='01', task='audiovisual', suffix='meg.fif') write_raw_bids(raw, bids_fname, output_path, events_data=events_data, From c7e2ae466cddae41973de7cf18f7d1c915794654 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Sat, 20 Oct 2018 23:47:53 -0400 Subject: [PATCH 13/38] ENH: address sappelhoff comments --- mne_bids/mne_bids.py | 12 +++++++----- mne_bids/utils.py | 8 +++++--- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index c5c6bf9e2..144932df0 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -524,11 +524,12 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, The raw data. If a string, it is assumed to be the path to the raw data file. Otherwise it must be an instance of mne.Raw bids_fname : str - The file path of the BIDS compatible raw file. In the case of multifile - systems (e.g., .ds), this is the path to a folder containing - the files or (e.g., .vhdr or .set), the path to the principal file. - E.g.: sub-01_ses-01_task-testing_acq-01_run-01_meg.fif - Typically, this can be generated using make_bids_filename. + The file path of the BIDS compatible raw file. Typically, this can be + generated using make_bids_filename. In the case of multifile + systems, this either points to a folder containing + the files (e.g., .ds) or the path to the file containing the header + information (e.g., .vhdr or .set). + Example: sub-01_ses-01_task-testing_acq-01_run-01_meg.fif output_path : str The path of the root of the BIDS compatible folder. The session and subject specific folders will be populated automatically by parsing @@ -571,6 +572,7 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, raw_fname = raw.filenames[0] if '.ds' in op.dirname(raw.filenames[0]): raw_fname = op.dirname(raw.filenames[0]) + # point to file containing header info for multifile systems raw_fname = raw_fname.replace('.eeg', '.vhdr') raw_fname = raw_fname.replace('.fdt', '.set') diff --git a/mne_bids/utils.py b/mne_bids/utils.py index 08a79e932..60869b7d1 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -64,6 +64,8 @@ def _mkdir_p(path, overwrite=False, verbose=False): def _parse_bids_filename(fname, verbose): """Get dict from BIDS fname.""" + from .mne_bids import ALLOWED_KINDS + params = dict(sub=None, ses=None, task=None, acq=None, run=None, proc=None, recording=None, space=None) entities = fname.split('_') @@ -75,10 +77,10 @@ def _parse_bids_filename(fname, verbose): params['suffix'] = entities[-1] kind, ext = _parse_ext(entities[-1], verbose=verbose) params['kind'], params['ext'] = kind, ext - if kind not in ('eeg', 'meg', 'ieeg'): + if kind not in ALLOWED_KINDS: raise ValueError('Please check the filename of your BIDS file.' - 'It should have _eeg, _meg, or _ieeg as suffix.' - 'Got %s' % kind) + 'It should have one of %s as suffix.' + 'Got %s' % (kind, ', '.join(ALLOWED_KINDS))) return params From afab4e2faffe953d94cc067bc53cfa069dd5a86d Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Sun, 21 Oct 2018 00:08:32 -0400 Subject: [PATCH 14/38] FIX tests --- mne_bids/io.py | 2 +- mne_bids/pick.py | 2 -- mne_bids/tests/test_mne_bids.py | 1 + 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/mne_bids/io.py b/mne_bids/io.py index 040d976d5..2c827e578 100644 --- a/mne_bids/io.py +++ b/mne_bids/io.py @@ -13,7 +13,7 @@ allowed_extensions_eeg = ['.vhdr', # BrainVision, accompanied by .vmrk, .eeg '.edf', # European Data Format '.bdf', # Biosemi - '.set', # EEGLAB + '.set', # EEGLAB, potentially accompanied by .fdt '.cnt', # Neuroscan ] diff --git a/mne_bids/pick.py b/mne_bids/pick.py index 6a861bbc1..9afe71c16 100644 --- a/mne_bids/pick.py +++ b/mne_bids/pick.py @@ -27,8 +27,6 @@ def get_coil_types(): FIFF.FIFFV_COIL_VV_PLANAR_T2, FIFF.FIFFV_COIL_VV_PLANAR_T3), megmag=(FIFF.FIFFV_COIL_POINT_MAGNETOMETER, - FIFF.FIFFV_COIL_POINT_MAGNETOMETER_X, - FIFF.FIFFV_COIL_POINT_MAGNETOMETER_Y, FIFF.FIFFV_COIL_VV_MAG_W, FIFF.FIFFV_COIL_VV_MAG_T1, FIFF.FIFFV_COIL_VV_MAG_T2, diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index 6fc571623..8bf7dcd80 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -272,6 +272,7 @@ def test_set(): bids_fname = partial_bids_fname + '_eeg.set' raw = read_raw(raw_fname) + # XXX: remove hack below once mne v0.17 is released raw._filenames = [raw_fname] write_raw_bids(raw, bids_fname, output_path, overwrite=False) From cc14f9c672ff6ac2b6ba101f72ba136b730890b9 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Sun, 21 Oct 2018 00:29:03 -0400 Subject: [PATCH 15/38] ENH: address comments by Matt --- doc/index.rst | 2 +- examples/convert_group_studies.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/index.rst b/doc/index.rst index 55bed30a0..e21de8d29 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -39,7 +39,7 @@ iEEG is experimental at the moment. >>> from mne_bids import read_raw, write_raw_bids >>> raw = read_raw('my_old_file.fif') >>> write_raw_bids(raw, 'sub-01_ses-01_run-05_meg.fif', - output_path='.') + output_path='./bids_dataset') Reading of BIDS data will also be supported in the next version. diff --git a/examples/convert_group_studies.py b/examples/convert_group_studies.py index 0f65804c1..5566d1c92 100644 --- a/examples/convert_group_studies.py +++ b/examples/convert_group_studies.py @@ -25,8 +25,8 @@ import os.path as op -import mne from mne_bids import write_raw_bids, make_bids_filename +from mne_bids.io import read_raw from mne_bids.datasets import fetch_faces_data from mne_bids.utils import print_dir_tree @@ -67,8 +67,8 @@ for run in runs: raw_fname = op.join(data_path, repo, subject, 'MEG', 'run_%02d_raw.fif' % run) - raw = mne.io.read_raw_fif(raw_fname) + raw = read_raw(raw_fname) bids_fname = make_bids_filename(subject='%02d' % subject_id, run=run, session='01', task='VisualFaces', suffix='meg.fif') From ef9e116871a8e7687b3cce792d0d79772480f9cb Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Sun, 21 Oct 2018 00:59:59 -0400 Subject: [PATCH 16/38] TST ordering of key-value pairs in filename --- mne_bids/tests/test_mne_bids.py | 4 ++++ mne_bids/utils.py | 16 ++++++++++++---- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index 8bf7dcd80..d4217b653 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -99,6 +99,10 @@ def test_fif(): with pytest.raises(KeyError, match='Unexpected entity'): write_raw_bids(raw, bids_fname, output_path) + bids_fname = 'sub-01_run-01_task-auditory_meg.fif' + with pytest.raises(ValueError, match='ordered correctly'): + write_raw_bids(raw, bids_fname, output_path) + del raw._filenames with pytest.raises(ValueError, match='raw.filenames is missing'): write_raw_bids(raw, bids_fname, output_path) diff --git a/mne_bids/utils.py b/mne_bids/utils.py index 60869b7d1..c66313a45 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -66,13 +66,21 @@ def _parse_bids_filename(fname, verbose): """Get dict from BIDS fname.""" from .mne_bids import ALLOWED_KINDS - params = dict(sub=None, ses=None, task=None, acq=None, - run=None, proc=None, recording=None, space=None) + keys = ['sub', 'ses', 'task', 'acq', 'run', 'proc', 'run', 'space', + 'recording'] + params = {key: None for key in keys} entities = fname.split('_') + idx_key = 0 for entity in entities[:-1]: key, value = entity.split('-') - if key not in params: - raise KeyError('Unexpected entity found in filename %s' % entity) + if key not in keys: + raise KeyError('Unexpected entity ''%s'' found in filename ''%s''' + % (entity, fname)) + if keys.index(key) < idx_key: + raise ValueError('Entities in filename not ordered correctly.' + ' "%s" should have occured earlier in the ' + 'filename "%s"' % (key, fname)) + idx_key = keys.index(key) params[key] = value params['suffix'] = entities[-1] kind, ext = _parse_ext(entities[-1], verbose=verbose) From b402150f17ed836a480c726833d63e158db034e5 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Sun, 21 Oct 2018 12:26:30 -0400 Subject: [PATCH 17/38] ENH: add hpi param --- mne_bids/io.py | 1 - mne_bids/mne_bids.py | 12 +++++++----- mne_bids/tests/test_mne_bids.py | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/mne_bids/io.py b/mne_bids/io.py index 2c827e578..44ba20a44 100644 --- a/mne_bids/io.py +++ b/mne_bids/io.py @@ -47,7 +47,6 @@ def read_raw(raw_fname, electrode=None, hsp=None, hpi=None, config=None, raise ValueError('Only single marker coils supported currently') raw = io.read_raw_kit(raw_fname, elp=electrode, hsp=hsp, mrk=hpi, preload=False) - raw._filenames.append(hpi) # Neuromag or converted-to-fif systems elif ext in ['.fif']: diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 144932df0..c4ce5b8c2 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -515,14 +515,13 @@ def _sidecar_json(raw, task, manufacturer, fname, kind, overwrite=False, def write_raw_bids(raw, bids_fname, output_path, events_data=None, - event_id=None, overwrite=False, verbose=True): + event_id=None, hpi=None, overwrite=False, verbose=True): """Walk over a folder of files and create BIDS compatible folder. Parameters ---------- raw : instance of mne.Raw - The raw data. If a string, it is assumed to be the path to the raw data - file. Otherwise it must be an instance of mne.Raw + The raw data. It must be an instance of mne.Raw. bids_fname : str The file path of the BIDS compatible raw file. Typically, this can be generated using make_bids_filename. In the case of multifile @@ -540,6 +539,10 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, inferred from the stim channel using `mne.find_events`. event_id : dict | None The event id dict used to create a 'trial_type' column in events.tsv + hpi : None | str + Marker points representing the location of the marker coils with + respect to the MEG Sensors, or path to a marker file. Usually required + for KIT systems. overwrite : bool Whether to overwrite existing files or data in files. Defaults to False. @@ -687,8 +690,7 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, else: sh.copyfile(raw_fname, bids_fname) # KIT data requires the marker file to be copied over too - if manufacturer == 'KIT/Yokogawa' and len(raw.filenames) > 1: - hpi = raw.filenames[-1] + if hpi is not None: _, marker_ext = _parse_ext(hpi) marker_fname = make_bids_filename( subject=subject_id, session=session_id, task=task, run=run, diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index d4217b653..c94907d64 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -128,7 +128,7 @@ def test_kit(): raw = read_raw(raw_fname, hpi=hpi_fname, electrode=electrode_fname, hsp=headshape_fname) write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, - event_id=event_id, overwrite=False) + event_id=event_id, hpi=hpi_fname, overwrite=False) cmd = ['bids-validator', output_path] run_subprocess(cmd, shell=shell) assert op.exists(op.join(output_path, 'participants.tsv')) From e1386b8c2327b044c4897c44d0fb63abecf95008 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Sun, 21 Oct 2018 15:02:52 -0400 Subject: [PATCH 18/38] TST check kind --- mne_bids/mne_bids.py | 5 +++++ mne_bids/tests/test_mne_bids.py | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index c4ce5b8c2..1122c10bc 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -583,6 +583,11 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, subject_id, session_id, kind = params['sub'], params['ses'], params['kind'] acquisition, task, run = params['acq'], params['task'], params['run'] + ch_type = 'ecog' if kind == 'ieeg' else kind + if ch_type not in raw: + raise ValueError('Expected %s channel types in the data for ' + 'writing the file %s.' % (ch_type, bids_fname)) + data_path = make_bids_folders(subject=subject_id, session=session_id, kind=kind, root=output_path, overwrite=False, verbose=verbose) diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index c94907d64..04cbc60e8 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -103,6 +103,10 @@ def test_fif(): with pytest.raises(ValueError, match='ordered correctly'): write_raw_bids(raw, bids_fname, output_path) + bids_fname = 'sub-01_task-auditory_run-01_ieeg.fif' + with pytest.raises(ValueError, match='Expected ecog'): + write_raw_bids(raw, bids_fname, output_path) + del raw._filenames with pytest.raises(ValueError, match='raw.filenames is missing'): write_raw_bids(raw, bids_fname, output_path) From 2eaccda3762319c813fbf1dd57dfb81fb386b56c Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Mon, 22 Oct 2018 14:12:45 -0400 Subject: [PATCH 19/38] TST for keeping same filetype --- mne_bids/mne_bids.py | 10 +++++++++- mne_bids/tests/test_mne_bids.py | 4 ++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 1122c10bc..12c49e5ee 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -518,12 +518,15 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, event_id=None, hpi=None, overwrite=False, verbose=True): """Walk over a folder of files and create BIDS compatible folder. + .. warning:: The original files are simply copied over. This function + cannot convert data files from one format to another. + Parameters ---------- raw : instance of mne.Raw The raw data. It must be an instance of mne.Raw. bids_fname : str - The file path of the BIDS compatible raw file. Typically, this can be + The filename of the BIDS compatible raw file. Typically, this can be generated using make_bids_filename. In the case of multifile systems, this either points to a folder containing the files (e.g., .ds) or the path to the file containing the header @@ -578,6 +581,11 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, # point to file containing header info for multifile systems raw_fname = raw_fname.replace('.eeg', '.vhdr') raw_fname = raw_fname.replace('.fdt', '.set') + _, ext_orig = _parse_ext(raw_fname, verbose=verbose) + if ext_orig != ext: + raise ValueError('write_raw_bids cannot convert filetype.' + 'Extension %s in bids_fname does not match' + 'extension %s of original file' % (ext, ext_orig)) params = _parse_bids_filename(bids_fname, verbose) subject_id, session_id, kind = params['sub'], params['ses'], params['kind'] diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index 04cbc60e8..8b53a701c 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -103,6 +103,10 @@ def test_fif(): with pytest.raises(ValueError, match='ordered correctly'): write_raw_bids(raw, bids_fname, output_path) + bids_fname = 'sub-01_task-auditory_run-01_meg.edf' + with pytest.raises(ValueError, match='convert filetype'): + write_raw_bids(raw, bids_fname, output_path) + bids_fname = 'sub-01_task-auditory_run-01_ieeg.fif' with pytest.raises(ValueError, match='Expected ecog'): write_raw_bids(raw, bids_fname, output_path) From 1cf4e53b2ec6cf5a9555da0c7f00c49662e5390d Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Mon, 22 Oct 2018 14:38:54 -0400 Subject: [PATCH 20/38] DOC minor fixes --- doc/api.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/doc/api.rst b/doc/api.rst index 4021e0e2c..0134bb72c 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -15,6 +15,7 @@ MNE BIDS (:py:mod:`mne_bids.mne_bids`): write_raw_bids make_bids_filename + make_bids_folders IO (:py:mod:`mne_bids.io`): @@ -33,9 +34,9 @@ Utils (:py:mod:`mne_bids.utils`): :toctree: generated/ print_dir_tree - make_bids_folders make_dataset_description copyfile_brainvision + copyfile_eeglab Datasets (:py:mod:`mne_bids.datasets`): From 47f8e62bb5050c550e7faa0c45996475449379c5 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Tue, 23 Oct 2018 15:32:07 -0400 Subject: [PATCH 21/38] WIP: make_bids_basename --- doc/api.rst | 2 +- doc/index.rst | 2 +- mne_bids/__init__.py | 2 +- mne_bids/mne_bids.py | 56 +++++++++++++++--------------- mne_bids/tests/test_mne_bids.py | 60 ++++++++++++++------------------- mne_bids/utils.py | 32 +++++++++++------- 6 files changed, 77 insertions(+), 77 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index 0134bb72c..a4e0653b5 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -14,7 +14,7 @@ MNE BIDS (:py:mod:`mne_bids.mne_bids`): :toctree: generated/ write_raw_bids - make_bids_filename + make_bids_basename make_bids_folders IO (:py:mod:`mne_bids.io`): diff --git a/doc/index.rst b/doc/index.rst index e21de8d29..c9453a8b2 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -38,7 +38,7 @@ iEEG is experimental at the moment. >>> from mne_bids import read_raw, write_raw_bids >>> raw = read_raw('my_old_file.fif') - >>> write_raw_bids(raw, 'sub-01_ses-01_run-05_meg.fif', + >>> write_raw_bids(raw, 'sub-01_ses-01_run-05', output_path='./bids_dataset') Reading of BIDS data will also be supported in the next version. diff --git a/mne_bids/__init__.py b/mne_bids/__init__.py index 8638b010f..34bab9bbc 100644 --- a/mne_bids/__init__.py +++ b/mne_bids/__init__.py @@ -4,5 +4,5 @@ from .mne_bids import write_raw_bids # noqa -from .utils import make_bids_folders, make_bids_filename # noqa +from .utils import make_bids_folders, make_bids_basename # noqa from .config import BIDS_VERSION # noqa diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 12c49e5ee..09affed84 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -28,11 +28,12 @@ from warnings import warn from .pick import coil_type -from .utils import (make_bids_filename, make_bids_folders, +from .utils import (make_bids_basename, make_bids_folders, make_dataset_description, _write_json, _write_tsv, _read_events, _mkdir_p, age_on_date, copyfile_brainvision, copyfile_eeglab, - _infer_eeg_placement_scheme, _parse_bids_filename) + _infer_eeg_placement_scheme, _parse_bids_filename, + _handle_kind) from .io import (_parse_ext, ALLOWED_EXTENSIONS) @@ -526,12 +527,22 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, raw : instance of mne.Raw The raw data. It must be an instance of mne.Raw. bids_fname : str - The filename of the BIDS compatible raw file. Typically, this can be - generated using make_bids_filename. In the case of multifile - systems, this either points to a folder containing - the files (e.g., .ds) or the path to the file containing the header - information (e.g., .vhdr or .set). - Example: sub-01_ses-01_task-testing_acq-01_run-01_meg.fif + The base filename of the BIDS compatible files. Typically, this can be + generated using make_bids_basename. + Example: sub-01_ses-01_task-testing_acq-01_run-01 + This will write the following files in the correct subfolder + of output_path: + sub-01_ses-01_task-testing_acq-01_run-01_meg.fif + sub-01_ses-01_task-testing_acq-01_run-01_meg.json + sub-01_ses-01_task-testing_acq-01_run-01_channels.tsv + sub-01_ses-01_task-testing_acq-01_run-01_coordsystem.json + and the following one if events_data is not None + sub-01_ses-01_task-testing_acq-01_run-01_events.tsv + and add a line to the following files: + participants.tsv + scans.tsv + Note that the modality 'meg' is automatically inferred from the raw + object and extension '.fif' is copied from raw.filenames. output_path : str The path of the root of the BIDS compatible folder. The session and subject specific folders will be populated automatically by parsing @@ -574,27 +585,18 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, raise ValueError('raw.filenames is missing. Please set raw.filenames' 'as a list with the full path of original raw file.') - _, ext = _parse_ext(bids_fname, verbose=verbose) raw_fname = raw.filenames[0] if '.ds' in op.dirname(raw.filenames[0]): raw_fname = op.dirname(raw.filenames[0]) # point to file containing header info for multifile systems raw_fname = raw_fname.replace('.eeg', '.vhdr') raw_fname = raw_fname.replace('.fdt', '.set') - _, ext_orig = _parse_ext(raw_fname, verbose=verbose) - if ext_orig != ext: - raise ValueError('write_raw_bids cannot convert filetype.' - 'Extension %s in bids_fname does not match' - 'extension %s of original file' % (ext, ext_orig)) + _, ext = _parse_ext(raw.filenames[0], verbose=verbose) params = _parse_bids_filename(bids_fname, verbose) - subject_id, session_id, kind = params['sub'], params['ses'], params['kind'] + subject_id, session_id = params['sub'], params['ses'] acquisition, task, run = params['acq'], params['task'], params['run'] - - ch_type = 'ecog' if kind == 'ieeg' else kind - if ch_type not in raw: - raise ValueError('Expected %s channel types in the data for ' - 'writing the file %s.' % (ch_type, bids_fname)) + kind = _handle_kind(raw) data_path = make_bids_folders(subject=subject_id, session=session_id, kind=kind, root=output_path, @@ -607,22 +609,22 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, overwrite=False, verbose=verbose) # create filenames - scans_fname = make_bids_filename( + scans_fname = make_bids_basename( subject=subject_id, session=session_id, suffix='scans.tsv', prefix=ses_path) - participants_fname = make_bids_filename(prefix=output_path, + participants_fname = make_bids_basename(prefix=output_path, suffix='participants.tsv') - coordsystem_fname = make_bids_filename( + coordsystem_fname = make_bids_basename( subject=subject_id, session=session_id, acquisition=acquisition, suffix='coordsystem.json', prefix=data_path) - sidecar_fname = make_bids_filename( + sidecar_fname = make_bids_basename( subject=subject_id, session=session_id, task=task, run=run, acquisition=acquisition, suffix='%s.json' % kind, prefix=data_path) - events_fname = make_bids_filename( + events_fname = make_bids_basename( subject=subject_id, session=session_id, task=task, acquisition=acquisition, run=run, suffix='events.tsv', prefix=data_path) - channels_fname = make_bids_filename( + channels_fname = make_bids_basename( subject=subject_id, session=session_id, task=task, run=run, acquisition=acquisition, suffix='channels.tsv', prefix=data_path) if ext not in ['.fif', '.ds', '.vhdr', '.edf', '.bdf', '.set', '.cnt']: @@ -705,7 +707,7 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, # KIT data requires the marker file to be copied over too if hpi is not None: _, marker_ext = _parse_ext(hpi) - marker_fname = make_bids_filename( + marker_fname = make_bids_basename( subject=subject_id, session=session_id, task=task, run=run, acquisition=acquisition, suffix='markers%s' % marker_ext, prefix=os.path.join(data_path, bids_raw_folder)) diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index 8b53a701c..5098a2f5b 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -20,7 +20,7 @@ from mne.utils import _TempDir, run_subprocess from mne.io.constants import FIFF -from mne_bids import make_bids_filename, make_bids_folders, write_raw_bids +from mne_bids import make_bids_basename, make_bids_folders, write_raw_bids from mne_bids.io import read_raw base_path = op.join(op.dirname(mne.__file__), 'io') @@ -32,7 +32,7 @@ run2 = '02' task = 'testing' -partial_bids_fname = make_bids_filename( +bids_basename = make_bids_basename( subject=subject_id, session=session_id, run=run, acquisition=acq, task=task) @@ -60,8 +60,7 @@ def test_fif(): 'sample_audvis_trunc_raw-eve.fif') raw = mne.io.read_raw_fif(raw_fname) - bids_fname = partial_bids_fname + '_meg.fif' - write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, + write_raw_bids(raw, bids_basename, output_path, events_data=events_fname, event_id=event_id, overwrite=False) # give the raw object some fake participant data @@ -73,7 +72,7 @@ def test_fif(): raw_fname2 = op.join(data_path2, 'sample_audvis_raw.fif') raw.save(raw_fname2) - bids_fname = bids_fname.replace(subject_id, subject_id2) + bids_fname = bids_basename.replace(subject_id, subject_id2) write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, event_id=event_id, overwrite=False) # check that the overwrite parameters work correctly for the participant @@ -82,34 +81,27 @@ def test_fif(): raw.info['subject_info'] = {'his_id': subject_id2, 'birthday': (1994, 1, 26), 'sex': 2} with pytest.raises(OSError, match="already exists"): - write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, - event_id=event_id, overwrite=False) + write_raw_bids(raw, bids_fname, output_path, + events_data=events_fname, event_id=event_id, + overwrite=False) # now force the overwrite write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, event_id=event_id, overwrite=True) with pytest.raises(ValueError, match='raw_file must be'): - write_raw_bids('blah', bids_fname, output_path) - - bids_fname = partial_bids_fname + '_xyz.fif' - with pytest.raises(ValueError, match='as suffix'): - write_raw_bids(raw, bids_fname, output_path) + write_raw_bids('blah', bids_basename, output_path) - bids_fname = 'sub-01_ses-01_xyz-01_run-01_meg.fif' + bids_fname = 'sub-01_ses-01_xyz-01_run-01' with pytest.raises(KeyError, match='Unexpected entity'): write_raw_bids(raw, bids_fname, output_path) - bids_fname = 'sub-01_run-01_task-auditory_meg.fif' + bids_fname = 'sub-01_run-01_task-auditory' with pytest.raises(ValueError, match='ordered correctly'): - write_raw_bids(raw, bids_fname, output_path) + write_raw_bids(raw, bids_fname, output_path, overwrite=True) - bids_fname = 'sub-01_task-auditory_run-01_meg.edf' + bids_fname = 'sub-01_task-auditory_run-01' with pytest.raises(ValueError, match='convert filetype'): - write_raw_bids(raw, bids_fname, output_path) - - bids_fname = 'sub-01_task-auditory_run-01_ieeg.fif' - with pytest.raises(ValueError, match='Expected ecog'): - write_raw_bids(raw, bids_fname, output_path) + write_raw_bids(raw, bids_fname, output_path, overwrite=True) del raw._filenames with pytest.raises(ValueError, match='raw.filenames is missing'): @@ -132,7 +124,7 @@ def test_kit(): headshape_fname = op.join(data_path, 'test_hsp.txt') event_id = dict(cond=1) - bids_fname = partial_bids_fname + '_meg.sqd' + bids_fname = bids_basename + '_meg.sqd' raw = read_raw(raw_fname, hpi=hpi_fname, electrode=electrode_fname, hsp=headshape_fname) write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, @@ -142,7 +134,7 @@ def test_kit(): assert op.exists(op.join(output_path, 'participants.tsv')) # ensure the channels file has no STI 014 channel: - channels_tsv = make_bids_filename( + channels_tsv = make_bids_basename( subject=subject_id, session=session_id, task=task, run=run, suffix='channels.tsv', acquisition=acq, prefix=op.join(output_path, 'sub-01/ses-01/meg')) @@ -150,10 +142,10 @@ def test_kit(): assert not ('STI 014' in df['name'].values) # ensure the marker file is produced in the right place - raw_folder = make_bids_filename( + raw_folder = make_bids_basename( subject=subject_id, session=session_id, task=task, run=run, acquisition=acq, suffix='%s' % 'meg') - marker_fname = make_bids_filename( + marker_fname = make_bids_basename( subject=subject_id, session=session_id, task=task, run=run, acquisition=acq, suffix='markers.sqd', prefix=os.path.join(output_path, 'sub-01/ses-01/meg', raw_folder)) @@ -171,7 +163,7 @@ def test_ctf(): data_path = op.join(testing.data_path(download=False), 'CTF') raw_fname = op.join(data_path, 'testdata_ctf.ds') - bids_fname = partial_bids_fname + '_meg.ds' + bids_fname = bids_basename + '_meg.ds' raw = mne.io.read_raw_ctf(raw_fname) write_raw_bids(raw, bids_fname, output_path=output_path) cmd = ['bids-validator', output_path] @@ -192,7 +184,7 @@ def test_bti(): config_fname = op.join(data_path, 'test_config_linux') headshape_fname = op.join(data_path, 'test_hs_linux') - bids_fname = partial_bids_fname + '_meg.pdf' + bids_fname = bids_basename + '_meg.pdf' raw = read_raw(raw_fname, config=config_fname, hsp=headshape_fname) write_raw_bids(raw, bids_fname, output_path, verbose=True) @@ -210,7 +202,7 @@ def test_vhdr(): data_path = op.join(base_path, 'brainvision', 'tests', 'data') raw_fname = op.join(data_path, 'test.vhdr') - bids_fname = partial_bids_fname + '_eeg.vhdr' + bids_fname = bids_basename + '_eeg.vhdr' raw = read_raw(raw_fname) write_raw_bids(raw, bids_fname, output_path, overwrite=False) @@ -237,7 +229,7 @@ def test_edf(): raw.rename_channels({raw.info['ch_names'][1]: 'EMG'}) raw.set_channel_types({'EMG': 'emg'}) - bids_fname = partial_bids_fname + '_eeg.edf' + bids_fname = bids_basename + '_eeg.edf' write_raw_bids(raw, bids_fname, output_path) bids_fname = bids_fname.replace('run-01', 'run-%s' % run2) write_raw_bids(raw, bids_fname, output_path, overwrite=True) @@ -246,7 +238,7 @@ def test_edf(): run_subprocess(cmd, shell=shell) # ensure there is an EMG channel in the channels.tsv: - channels_tsv = make_bids_filename( + channels_tsv = make_bids_basename( subject=subject_id, session=session_id, task=task, run=run, suffix='channels.tsv', acquisition=acq, prefix=op.join(output_path, 'sub-01/ses-01/eeg')) @@ -254,7 +246,7 @@ def test_edf(): assert 'ElectroMyoGram' in df['description'].values # check that the scans list contains two scans - scans_tsv = make_bids_filename( + scans_tsv = make_bids_basename( subject=subject_id, session=session_id, suffix='scans.tsv', prefix=op.join(output_path, 'sub-01/ses-01')) df = pd.read_csv(scans_tsv, sep='\t') @@ -267,7 +259,7 @@ def test_bdf(): data_path = op.join(base_path, 'edf', 'tests', 'data') raw_fname = op.join(data_path, 'test.bdf') - bids_fname = partial_bids_fname + '_eeg.bdf' + bids_fname = bids_basename + '_eeg.bdf' raw = read_raw(raw_fname) write_raw_bids(raw, bids_fname, output_path, overwrite=False) @@ -282,7 +274,7 @@ def test_set(): data_path = op.join(testing.data_path(), 'EEGLAB') raw_fname = op.join(data_path, 'test_raw_onefile.set') - bids_fname = partial_bids_fname + '_eeg.set' + bids_fname = bids_basename + '_eeg.set' raw = read_raw(raw_fname) # XXX: remove hack below once mne v0.17 is released raw._filenames = [raw_fname] @@ -313,7 +305,7 @@ def test_cnt(): data_path = op.join(testing.data_path(), 'CNT') raw_fname = op.join(data_path, 'scan41_short.cnt') - bids_fname = partial_bids_fname + '_eeg.cnt' + bids_fname = bids_basename + '_eeg.cnt' raw = read_raw(raw_fname) write_raw_bids(raw, bids_fname, output_path) diff --git a/mne_bids/utils.py b/mne_bids/utils.py index c66313a45..367ae80ef 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -64,8 +64,6 @@ def _mkdir_p(path, overwrite=False, verbose=False): def _parse_bids_filename(fname, verbose): """Get dict from BIDS fname.""" - from .mne_bids import ALLOWED_KINDS - keys = ['sub', 'ses', 'task', 'acq', 'run', 'proc', 'run', 'space', 'recording'] params = {key: None for key in keys} @@ -82,24 +80,32 @@ def _parse_bids_filename(fname, verbose): 'filename "%s"' % (key, fname)) idx_key = keys.index(key) params[key] = value - params['suffix'] = entities[-1] - kind, ext = _parse_ext(entities[-1], verbose=verbose) - params['kind'], params['ext'] = kind, ext - if kind not in ALLOWED_KINDS: - raise ValueError('Please check the filename of your BIDS file.' - 'It should have one of %s as suffix.' - 'Got %s' % (kind, ', '.join(ALLOWED_KINDS))) return params -def make_bids_filename(subject=None, session=None, task=None, +def _handle_kind(raw): + """Get kind.""" + if 'meg' in raw: + kind = 'meg' + elif 'eeg' in raw: + kind = 'eeg' + elif 'ecog' in raw: + kind = 'ieeg' + else: + raise ValueError('Neither MEG/EEG/iEEG channels found in data.' + 'Please use raw.set_channel_types to set the ' + 'channel types in the data.') + return kind + + +def make_bids_basename(subject=None, session=None, task=None, acquisition=None, run=None, processing=None, recording=None, space=None, suffix=None, prefix=None): - """Create a BIDS filename from its component parts. + """Create a partial/full BIDS filename from its component parts. BIDS filename prefixes have one or more pieces of metadata in them. They must follow a particular order, which is followed by this function. This - will generate the *prefix* for a BIDS file name that can be used with many + will generate the *prefix* for a BIDS filename that can be used with many subsequent files, or you may also give a suffix that will then complete the file name. @@ -137,7 +143,7 @@ def make_bids_filename(subject=None, session=None, task=None, Examples -------- - >>> print(make_bids_filename(subject='test', session='two', task='mytask', suffix='data.csv')) # noqa + >>> print(make_bids_basename(subject='test', session='two', task='mytask', suffix='data.csv')) # noqa sub-test_ses-two_task-mytask_data.csv """ From 976aba0c235e7c8c13139217da3d5aab0d4ae26b Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Thu, 25 Oct 2018 15:09:39 -0400 Subject: [PATCH 22/38] WIP: don't make read_raw public --- doc/api.rst | 9 ----- mne_bids/io.py | 58 --------------------------------- mne_bids/tests/test_mne_bids.py | 25 +++++++------- 3 files changed, 11 insertions(+), 81 deletions(-) diff --git a/doc/api.rst b/doc/api.rst index a4e0653b5..0c1bd293b 100644 --- a/doc/api.rst +++ b/doc/api.rst @@ -17,15 +17,6 @@ MNE BIDS (:py:mod:`mne_bids.mne_bids`): make_bids_basename make_bids_folders -IO (:py:mod:`mne_bids.io`): - -.. current_module:: mne_bids.io - -.. autosummary:: - :toctree: generated/ - - read_raw - Utils (:py:mod:`mne_bids.utils`): .. currentmodule:: mne_bids.utils diff --git a/mne_bids/io.py b/mne_bids/io.py index 44ba20a44..dc7b1e4e8 100644 --- a/mne_bids/io.py +++ b/mne_bids/io.py @@ -6,7 +6,6 @@ # Stefan Appelhoff # # License: BSD (3-clause) -from mne import io import os allowed_extensions_meg = ['.con', '.sqd', '.fif', '.pdf', '.ds'] @@ -30,60 +29,3 @@ def _parse_ext(raw_fname, verbose=False): 'appending extension .pdf') ext = '.pdf' return fname, ext - - -def read_raw(raw_fname, electrode=None, hsp=None, hpi=None, config=None, - montage=None, verbose=None): - """Read a raw file into MNE, making inferences based on extension.""" - fname, ext = _parse_ext(raw_fname) - - # MEG File Types - # -------------- - # KIT systems - if ext in ['.con', '.sqd']: - if isinstance(hpi, list): - # No currently accepted way to name multiple marker files. See: - # https://github.com/bids-standard/bids-specification/issues/45 - raise ValueError('Only single marker coils supported currently') - raw = io.read_raw_kit(raw_fname, elp=electrode, hsp=hsp, - mrk=hpi, preload=False) - - # Neuromag or converted-to-fif systems - elif ext in ['.fif']: - raw = io.read_raw_fif(raw_fname, preload=False) - - # BTi systems - elif ext == '.pdf': - if os.path.isfile(raw_fname): - raw = io.read_raw_bti(raw_fname, config_fname=config, - head_shape_fname=hsp, - preload=False, verbose=verbose) - - # CTF systems - elif ext == '.ds': - raw = io.read_raw_ctf(raw_fname) - - # EEG File Types - # -------------- - # BrainVision format by Brain Products, links to a .eeg and a .vmrk file - elif ext == '.vhdr': - raw = io.read_raw_brainvision(raw_fname) - - # EDF (european data format) or BDF (biosemi) format - elif ext == '.edf' or ext == '.bdf': - raw = io.read_raw_edf(raw_fname, preload=True) - - # EEGLAB .set format, a .fdt file is potentially linked from the .set - elif ext == '.set': - raw = io.read_raw_eeglab(raw_fname) - - # Neuroscan .cnt format - elif ext == '.cnt': - raw = io.read_raw_cnt(raw_fname, montage=montage) - - # No supported data found ... - # --------------------------- - else: - raise ValueError("Raw file name extension must be one of %\n" - "Got %" % (ALLOWED_EXTENSIONS, ext)) - return raw diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index 5098a2f5b..ee10e2eae 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -15,13 +15,13 @@ import pytest import pandas as pd + import mne from mne.datasets import testing from mne.utils import _TempDir, run_subprocess from mne.io.constants import FIFF from mne_bids import make_bids_basename, make_bids_folders, write_raw_bids -from mne_bids.io import read_raw base_path = op.join(op.dirname(mne.__file__), 'io') subject_id = '01' @@ -125,8 +125,9 @@ def test_kit(): event_id = dict(cond=1) bids_fname = bids_basename + '_meg.sqd' - raw = read_raw(raw_fname, hpi=hpi_fname, electrode=electrode_fname, - hsp=headshape_fname) + raw = mne.io.read_raw_kit( + raw_fname, hpi=hpi_fname, electrode=electrode_fname, + hsp=headshape_fname) write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, event_id=event_id, hpi=hpi_fname, overwrite=False) cmd = ['bids-validator', output_path] @@ -151,11 +152,6 @@ def test_kit(): prefix=os.path.join(output_path, 'sub-01/ses-01/meg', raw_folder)) assert op.exists(marker_fname) - # check for error if there are multiple marker coils specified - with pytest.raises(ValueError): - raw = read_raw(raw_fname, hpi=[hpi_fname, hpi_fname], - electrode=electrode_fname, hsp=headshape_fname) - def test_ctf(): """Test functionality of the write_raw_bids conversion for CTF data.""" @@ -185,7 +181,8 @@ def test_bti(): headshape_fname = op.join(data_path, 'test_hs_linux') bids_fname = bids_basename + '_meg.pdf' - raw = read_raw(raw_fname, config=config_fname, hsp=headshape_fname) + raw = mne.io.read_raw_bti(raw_fname, config=config_fname, + hsp=headshape_fname) write_raw_bids(raw, bids_fname, output_path, verbose=True) assert op.exists(op.join(output_path, 'participants.tsv')) @@ -203,7 +200,7 @@ def test_vhdr(): raw_fname = op.join(data_path, 'test.vhdr') bids_fname = bids_basename + '_eeg.vhdr' - raw = read_raw(raw_fname) + raw = mne.io.read_raw_brainvision(raw_fname) write_raw_bids(raw, bids_fname, output_path, overwrite=False) cmd = ['bids-validator', '--bep006', output_path] @@ -260,7 +257,7 @@ def test_bdf(): raw_fname = op.join(data_path, 'test.bdf') bids_fname = bids_basename + '_eeg.bdf' - raw = read_raw(raw_fname) + raw = mne.io.read_raw_edf(raw_fname) write_raw_bids(raw, bids_fname, output_path, overwrite=False) cmd = ['bids-validator', '--bep006', output_path] @@ -275,7 +272,7 @@ def test_set(): raw_fname = op.join(data_path, 'test_raw_onefile.set') bids_fname = bids_basename + '_eeg.set' - raw = read_raw(raw_fname) + raw = mne.io.read_raw_eeglab(raw_fname) # XXX: remove hack below once mne v0.17 is released raw._filenames = [raw_fname] write_raw_bids(raw, bids_fname, output_path, overwrite=False) @@ -292,7 +289,7 @@ def test_set(): data_path = op.join(testing.data_path(), 'EEGLAB') raw_fname = op.join(data_path, 'test_raw.set') - raw = read_raw(raw_fname) + raw = mne.io.read_raw_eeglab(raw_fname) write_raw_bids(raw, bids_fname, output_path, overwrite=False) cmd = ['bids-validator', '--bep006', output_path] @@ -306,7 +303,7 @@ def test_cnt(): raw_fname = op.join(data_path, 'scan41_short.cnt') bids_fname = bids_basename + '_eeg.cnt' - raw = read_raw(raw_fname) + raw = mne.io.read_raw_cnt(raw_fname) write_raw_bids(raw, bids_fname, output_path) cmd = ['bids-validator', '--bep006', output_path] From 5afa95f21d9c5923e0f86118366409dabc660b2f Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Thu, 25 Oct 2018 15:16:35 -0400 Subject: [PATCH 23/38] WIP: raw.preload=False --- mne_bids/mne_bids.py | 11 +++++++-- mne_bids/tests/test_mne_bids.py | 43 ++++++++++++++------------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 09affed84..97256fab9 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -525,7 +525,8 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, Parameters ---------- raw : instance of mne.Raw - The raw data. It must be an instance of mne.Raw. + The raw data. It must be an instance of mne.Raw. The data should not be + loaded on disk, i.e., raw.preload must be False. bids_fname : str The base filename of the BIDS compatible files. Typically, this can be generated using make_bids_basename. @@ -585,19 +586,25 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, raise ValueError('raw.filenames is missing. Please set raw.filenames' 'as a list with the full path of original raw file.') + if raw.preload is not False: + raise ValueError('The data should not be preloaded') + + raw = raw.copy() + raw_fname = raw.filenames[0] if '.ds' in op.dirname(raw.filenames[0]): raw_fname = op.dirname(raw.filenames[0]) # point to file containing header info for multifile systems raw_fname = raw_fname.replace('.eeg', '.vhdr') raw_fname = raw_fname.replace('.fdt', '.set') - _, ext = _parse_ext(raw.filenames[0], verbose=verbose) + _, ext = _parse_ext(raw_fname, verbose=verbose) params = _parse_bids_filename(bids_fname, verbose) subject_id, session_id = params['sub'], params['ses'] acquisition, task, run = params['acq'], params['task'], params['run'] kind = _handle_kind(raw) + bids_fname = bids_fname + '_%s%s' % (kind, ext) data_path = make_bids_folders(subject=subject_id, session=session_id, kind=kind, root=output_path, overwrite=False, verbose=verbose) diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index ee10e2eae..a4831de16 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -124,11 +124,10 @@ def test_kit(): headshape_fname = op.join(data_path, 'test_hsp.txt') event_id = dict(cond=1) - bids_fname = bids_basename + '_meg.sqd' raw = mne.io.read_raw_kit( - raw_fname, hpi=hpi_fname, electrode=electrode_fname, + raw_fname, mrk=hpi_fname, elp=electrode_fname, hsp=headshape_fname) - write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, + write_raw_bids(raw, bids_basename, output_path, events_data=events_fname, event_id=event_id, hpi=hpi_fname, overwrite=False) cmd = ['bids-validator', output_path] run_subprocess(cmd, shell=shell) @@ -159,15 +158,14 @@ def test_ctf(): data_path = op.join(testing.data_path(download=False), 'CTF') raw_fname = op.join(data_path, 'testdata_ctf.ds') - bids_fname = bids_basename + '_meg.ds' raw = mne.io.read_raw_ctf(raw_fname) - write_raw_bids(raw, bids_fname, output_path=output_path) + write_raw_bids(raw, bids_basename, output_path=output_path) cmd = ['bids-validator', output_path] run_subprocess(cmd, shell=shell) # test to check that running again with overwrite == False raises an error with pytest.raises(OSError, match="already exists"): - write_raw_bids(raw, bids_fname, output_path=output_path) + write_raw_bids(raw, bids_basename, output_path=output_path) assert op.exists(op.join(output_path, 'participants.tsv')) @@ -180,10 +178,9 @@ def test_bti(): config_fname = op.join(data_path, 'test_config_linux') headshape_fname = op.join(data_path, 'test_hs_linux') - bids_fname = bids_basename + '_meg.pdf' - raw = mne.io.read_raw_bti(raw_fname, config=config_fname, - hsp=headshape_fname) - write_raw_bids(raw, bids_fname, output_path, verbose=True) + raw = mne.io.read_raw_bti(raw_fname, config_fname=config_fname, + head_shape_fname=headshape_fname) + write_raw_bids(raw, bids_basename, output_path, verbose=True) assert op.exists(op.join(output_path, 'participants.tsv')) @@ -199,9 +196,8 @@ def test_vhdr(): data_path = op.join(base_path, 'brainvision', 'tests', 'data') raw_fname = op.join(data_path, 'test.vhdr') - bids_fname = bids_basename + '_eeg.vhdr' raw = mne.io.read_raw_brainvision(raw_fname) - write_raw_bids(raw, bids_fname, output_path, overwrite=False) + write_raw_bids(raw, bids_basename, output_path, overwrite=False) cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) @@ -220,15 +216,15 @@ def test_edf(): data_path = op.join(testing.data_path(), 'EDF') raw_fname = op.join(data_path, 'test_reduced.edf') - raw = mne.io.read_raw_edf(raw_fname, preload=True) + raw = mne.io.read_raw_edf(raw_fname, stim_channel=None, + preload=False) raw.rename_channels({raw.info['ch_names'][0]: 'EOG'}) raw.info['chs'][0]['coil_type'] = FIFF.FIFFV_COIL_EEG_BIPOLAR raw.rename_channels({raw.info['ch_names'][1]: 'EMG'}) raw.set_channel_types({'EMG': 'emg'}) - bids_fname = bids_basename + '_eeg.edf' - write_raw_bids(raw, bids_fname, output_path) - bids_fname = bids_fname.replace('run-01', 'run-%s' % run2) + write_raw_bids(raw, bids_basename, output_path) + bids_fname = bids_basename.replace('run-01', 'run-%s' % run2) write_raw_bids(raw, bids_fname, output_path, overwrite=True) cmd = ['bids-validator', '--bep006', output_path] @@ -256,9 +252,8 @@ def test_bdf(): data_path = op.join(base_path, 'edf', 'tests', 'data') raw_fname = op.join(data_path, 'test.bdf') - bids_fname = bids_basename + '_eeg.bdf' raw = mne.io.read_raw_edf(raw_fname) - write_raw_bids(raw, bids_fname, output_path, overwrite=False) + write_raw_bids(raw, bids_basename, output_path, overwrite=False) cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) @@ -271,17 +266,16 @@ def test_set(): data_path = op.join(testing.data_path(), 'EEGLAB') raw_fname = op.join(data_path, 'test_raw_onefile.set') - bids_fname = bids_basename + '_eeg.set' raw = mne.io.read_raw_eeglab(raw_fname) # XXX: remove hack below once mne v0.17 is released raw._filenames = [raw_fname] - write_raw_bids(raw, bids_fname, output_path, overwrite=False) + write_raw_bids(raw, bids_basename, output_path, overwrite=False) cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) with pytest.raises(OSError, match="already exists"): - write_raw_bids(raw, bids_fname, output_path=output_path, + write_raw_bids(raw, bids_basename, output_path=output_path, overwrite=False) # .set with associated .fdt @@ -290,7 +284,7 @@ def test_set(): raw_fname = op.join(data_path, 'test_raw.set') raw = mne.io.read_raw_eeglab(raw_fname) - write_raw_bids(raw, bids_fname, output_path, overwrite=False) + write_raw_bids(raw, bids_basename, output_path, overwrite=False) cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) @@ -302,9 +296,8 @@ def test_cnt(): data_path = op.join(testing.data_path(), 'CNT') raw_fname = op.join(data_path, 'scan41_short.cnt') - bids_fname = bids_basename + '_eeg.cnt' - raw = mne.io.read_raw_cnt(raw_fname) - write_raw_bids(raw, bids_fname, output_path) + raw = mne.io.read_raw_cnt(raw_fname, montage=None) + write_raw_bids(raw, bids_basename, output_path) cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) From 2074a667360ae1ef33c4cf892ebaf812ae68f44e Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Fri, 26 Oct 2018 00:49:32 -0400 Subject: [PATCH 24/38] DOC: Update examples --- examples/convert_group_studies.py | 11 +++++------ examples/convert_mne_sample.py | 11 ++++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/examples/convert_group_studies.py b/examples/convert_group_studies.py index 5566d1c92..f21d6821a 100644 --- a/examples/convert_group_studies.py +++ b/examples/convert_group_studies.py @@ -25,8 +25,8 @@ import os.path as op -from mne_bids import write_raw_bids, make_bids_filename -from mne_bids.io import read_raw +import mne +from mne_bids import write_raw_bids from mne_bids.datasets import fetch_faces_data from mne_bids.utils import print_dir_tree @@ -68,10 +68,9 @@ raw_fname = op.join(data_path, repo, subject, 'MEG', 'run_%02d_raw.fif' % run) - raw = read_raw(raw_fname) - bids_fname = make_bids_filename(subject='%02d' % subject_id, run=run, - session='01', task='VisualFaces', - suffix='meg.fif') + raw = mne.io.read_raw_fif(raw_fname) + bids_fname = ('sub-%02d_ses-01_run-%d_task-VisualFaces' + % (subject_id, run)) write_raw_bids(raw, bids_fname, output_path, event_id=event_id, overwrite=True) diff --git a/examples/convert_mne_sample.py b/examples/convert_mne_sample.py index 21ef50691..4c34f5112 100644 --- a/examples/convert_mne_sample.py +++ b/examples/convert_mne_sample.py @@ -17,9 +17,11 @@ # Let us import mne_bids import os.path as op + +import mne from mne.datasets import sample -from mne_bids import write_raw_bids, make_bids_filename -from mne_bids.io import read_raw + +from mne_bids import write_raw_bids from mne_bids.utils import print_dir_tree ############################################################################### @@ -36,9 +38,8 @@ ############################################################################### # Finally, we specify the raw_file and events_data -raw = read_raw(raw_fname) -bids_fname = make_bids_filename(subject='01', run='01', session='01', - task='audiovisual', suffix='meg.fif') +raw = mne.io.read_raw_fif(raw_fname) +bids_fname = 'sub-01_ses-01_run-01_task-audiovisual' write_raw_bids(raw, bids_fname, output_path, events_data=events_data, event_id=event_id, overwrite=True) From 75b02c9b600259bede0b568d15ec4fc2345fcd56 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Fri, 26 Oct 2018 01:04:44 -0400 Subject: [PATCH 25/38] FIX test --- mne_bids/commands/mne_bids_raw_to_bids.py | 13 +++--- mne_bids/io.py | 54 +++++++++++++++++++++++ mne_bids/tests/test_io.py | 4 +- mne_bids/tests/test_mne_bids.py | 4 -- mne_bids/tests/test_utils.py | 10 ++--- mne_bids/utils.py | 3 +- 6 files changed, 69 insertions(+), 19 deletions(-) diff --git a/mne_bids/commands/mne_bids_raw_to_bids.py b/mne_bids/commands/mne_bids_raw_to_bids.py index 76844304b..c2a6c8da2 100644 --- a/mne_bids/commands/mne_bids_raw_to_bids.py +++ b/mne_bids/commands/mne_bids_raw_to_bids.py @@ -8,8 +8,8 @@ """ import os -from mne_bids import write_raw_bids, make_bids_filename -from mne_bids.io import read_raw +from mne_bids import write_raw_bids, make_bids_basename +from mne_bids.io import _read_raw def run(): @@ -62,12 +62,11 @@ def run(): opt, args = parser.parse_args() - suffix = '%s%s' % (opt.kind, os.path.splitext(opt.raw_fname)[1]) - bids_fname = make_bids_filename( + bids_fname = make_bids_basename( subject=opt.subject_id, session=opt.session_id, run=opt.run, - acquisition=opt.acq, task=opt.task, suffix=suffix) - raw = read_raw(opt.raw_fname, hpi=opt.hpi, electrode=opt.electrode, - hsp=opt.hsp, config=opt.config) + acquisition=opt.acq, task=opt.task) + raw = _read_raw(opt.raw_fname, hpi=opt.hpi, electrode=opt.electrode, + hsp=opt.hsp, config=opt.config) write_raw_bids(raw, bids_fname, opt.output_path, event_id=opt.event_id, events_data=opt.events_data, overwrite=opt.overwrite, verbose=True) diff --git a/mne_bids/io.py b/mne_bids/io.py index dc7b1e4e8..6fec70df7 100644 --- a/mne_bids/io.py +++ b/mne_bids/io.py @@ -6,6 +6,7 @@ # Stefan Appelhoff # # License: BSD (3-clause) +from mne import io import os allowed_extensions_meg = ['.con', '.sqd', '.fif', '.pdf', '.ds'] @@ -29,3 +30,56 @@ def _parse_ext(raw_fname, verbose=False): 'appending extension .pdf') ext = '.pdf' return fname, ext + + +def _read_raw(raw_fname, electrode=None, hsp=None, hpi=None, config=None, + montage=None, verbose=None): + """Read a raw file into MNE, making inferences based on extension.""" + fname, ext = _parse_ext(raw_fname) + + # MEG File Types + # -------------- + # KIT systems + if ext in ['.con', '.sqd']: + raw = io.read_raw_kit(raw_fname, elp=electrode, hsp=hsp, + mrk=hpi, preload=False) + + # Neuromag or converted-to-fif systems + elif ext in ['.fif']: + raw = io.read_raw_fif(raw_fname, preload=False) + + # BTi systems + elif ext == '.pdf': + if os.path.isfile(raw_fname): + raw = io.read_raw_bti(raw_fname, config_fname=config, + head_shape_fname=hsp, + preload=False, verbose=verbose) + + # CTF systems + elif ext == '.ds': + raw = io.read_raw_ctf(raw_fname) + + # EEG File Types + # -------------- + # BrainVision format by Brain Products, links to a .eeg and a .vmrk file + elif ext == '.vhdr': + raw = io.read_raw_brainvision(raw_fname) + + # EDF (european data format) or BDF (biosemi) format + elif ext == '.edf' or ext == '.bdf': + raw = io.read_raw_edf(raw_fname, preload=True) + + # EEGLAB .set format, a .fdt file is potentially linked from the .set + elif ext == '.set': + raw = io.read_raw_eeglab(raw_fname) + + # Neuroscan .cnt format + elif ext == '.cnt': + raw = io.read_raw_cnt(raw_fname, montage=montage) + + # No supported data found ... + # --------------------------- + else: + raise ValueError("Raw file name extension must be one of %\n" + "Got %" % (ALLOWED_EXTENSIONS, ext)) + return raw diff --git a/mne_bids/tests/test_io.py b/mne_bids/tests/test_io.py index 3f4b14f61..a907e16ca 100644 --- a/mne_bids/tests/test_io.py +++ b/mne_bids/tests/test_io.py @@ -4,7 +4,7 @@ # License: BSD (3-clause) import pytest -from mne_bids.io import _parse_ext, read_raw +from mne_bids.io import _parse_ext, _read_raw def test_parse_ext(): @@ -26,4 +26,4 @@ def test_read_raw(): # Use a file ending that does not exist f = 'file.bogus' with pytest.raises(ValueError): - read_raw(f) + _read_raw(f) diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index a4831de16..13938bb16 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -99,10 +99,6 @@ def test_fif(): with pytest.raises(ValueError, match='ordered correctly'): write_raw_bids(raw, bids_fname, output_path, overwrite=True) - bids_fname = 'sub-01_task-auditory_run-01' - with pytest.raises(ValueError, match='convert filetype'): - write_raw_bids(raw, bids_fname, output_path, overwrite=True) - del raw._filenames with pytest.raises(ValueError, match='raw.filenames is missing'): write_raw_bids(raw, bids_fname, output_path) diff --git a/mne_bids/tests/test_utils.py b/mne_bids/tests/test_utils.py index cc59583da..acb0b4c8d 100644 --- a/mne_bids/tests/test_utils.py +++ b/mne_bids/tests/test_utils.py @@ -13,7 +13,7 @@ from mne.datasets import testing from mne.utils import _TempDir -from mne_bids.utils import (make_bids_folders, make_bids_filename, +from mne_bids.utils import (make_bids_folders, make_bids_basename, _check_types, print_dir_tree, age_on_date, _get_brainvision_paths, copyfile_brainvision, copyfile_eeglab, _infer_eeg_placement_scheme) @@ -36,14 +36,14 @@ def test_make_filenames(): prefix_data = dict(subject='one', session='two', task='three', acquisition='four', run='five', processing='six', recording='seven', suffix='suffix.csv') - assert make_bids_filename(**prefix_data) == 'sub-one_ses-two_task-three_acq-four_run-five_proc-six_recording-seven_suffix.csv' # noqa + assert make_bids_basename(**prefix_data) == 'sub-one_ses-two_task-three_acq-four_run-five_proc-six_recording-seven_suffix.csv' # noqa # subsets of keys works - assert make_bids_filename(subject='one', task='three') == 'sub-one_task-three' # noqa - assert make_bids_filename(subject='one', task='three', suffix='hi.csv') == 'sub-one_task-three_hi.csv' # noqa + assert make_bids_basename(subject='one', task='three') == 'sub-one_task-three' # noqa + assert make_bids_basename(subject='one', task='three', suffix='hi.csv') == 'sub-one_task-three_hi.csv' # noqa with pytest.raises(ValueError): - make_bids_filename(subject='one-two', suffix='there.csv') + make_bids_basename(subject='one-two', suffix='there.csv') def test_make_folders(): diff --git a/mne_bids/utils.py b/mne_bids/utils.py index 367ae80ef..9fdaad9e5 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -69,7 +69,8 @@ def _parse_bids_filename(fname, verbose): params = {key: None for key in keys} entities = fname.split('_') idx_key = 0 - for entity in entities[:-1]: + for entity in entities: + assert '-' in entity key, value = entity.split('-') if key not in keys: raise KeyError('Unexpected entity ''%s'' found in filename ''%s''' From 003a8117d484a05f04e4bd522f4bbd455867ecbe Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Fri, 26 Oct 2018 01:21:21 -0400 Subject: [PATCH 26/38] DOC update quickstart --- doc/index.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/index.rst b/doc/index.rst index c9453a8b2..7c7ee5a3a 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -36,10 +36,10 @@ iEEG is experimental at the moment. .. code:: python - >>> from mne_bids import read_raw, write_raw_bids - >>> raw = read_raw('my_old_file.fif') - >>> write_raw_bids(raw, 'sub-01_ses-01_run-05', - output_path='./bids_dataset') + >>> from mne import io + >>> from mne_bids import write_raw_bids + >>> raw = io.read_raw_fif('my_old_file.fif') + >>> write_raw_bids(raw, 'sub-01_ses-01_run-05', output_path='./bids_dataset') Reading of BIDS data will also be supported in the next version. From cc5ad7825c2500a269040093e0243834d3b88a17 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Fri, 26 Oct 2018 02:28:28 -0400 Subject: [PATCH 27/38] add comment --- mne_bids/mne_bids.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 97256fab9..c07cbb33d 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -470,6 +470,7 @@ def _sidecar_json(raw, task, manufacturer, fname, kind, overwrite=False, ('PowerLineFrequency', powerlinefrequency), ('SamplingFrequency', sfreq), ('SoftwareFilters', 'n/a'), + # XXX: If user calls raw.crop, this will be screwed up ('RecordingDuration', raw.times[-1]), ('RecordingType', rec_type)] ch_info_json_meg = [ @@ -587,7 +588,7 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, 'as a list with the full path of original raw file.') if raw.preload is not False: - raise ValueError('The data should not be preloaded') + raise ValueError('The data should not be preloaded.') raw = raw.copy() From 24449906db0b028e35877d08b239cb906cbfbb6d Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Fri, 26 Oct 2018 16:08:29 -0400 Subject: [PATCH 28/38] edf example + raw.preload --- examples/convert_eeg_to_bids.py | 59 +++++++++++++++++++-------------- mne_bids/tests/test_mne_bids.py | 7 ++-- 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/examples/convert_eeg_to_bids.py b/examples/convert_eeg_to_bids.py index 330f7294f..9508e52cd 100644 --- a/examples/convert_eeg_to_bids.py +++ b/examples/convert_eeg_to_bids.py @@ -23,10 +23,11 @@ import os import shutil as sh +import mne from mne.datasets import eegbci -from mne.io import read_raw_edf +from mne.io.edf.edf import read_annotations_edf -from mne_bids import write_raw_bids, make_bids_filename +from mne_bids import write_raw_bids, make_bids_basename from mne_bids.utils import print_dir_tree ############################################################################### @@ -84,23 +85,31 @@ # -------------------------- # # Let's start by formatting a single subject. We are reading the data using -# MNE-Python's io module and the `read_raw_edf` function. -data_dir = os.path.join(data_dir, 'physiobank', 'database', 'eegmmidb') -edf_path = os.path.join(data_dir, 'S001', 'S001R02.edf') -raw = read_raw_edf(edf_path, preload=True) +# MNE-Python's io module and the `read_raw_edf` function. Note that we must +# use `preload=False`, the default in MNE-Python. It prevents the data from +# being loaded and modified when converting to BIDS. +edf_path = eegbci.load_data(subject=1, runs=2)[0] +raw = mne.io.read_raw_edf(edf_path, preload=False, stim_channel=None) + +############################################################################### +# The annotations stored in the file mut be read in separately and converted +# into a 2D numpy array of events that is compatible with MNE. +annot = read_annotations_edf(edf_path) +raw.set_annotations(annot) +events, event_id = mne.events_from_annotations(raw) + print(raw) ############################################################################### -# With this simple step we have everything to start a new BIDS directory using -# our data. To do that, we can use the high level function `raw_to_bids`, which -# is the core of MNE-BIDS. Generally, `raw_to_bids` tries to extract as much +# With this step, we have everything to start a new BIDS directory using +# our data. To do that, we can use the function `write_raw_bids` +# Generally, `write_raw_bids` tries to extract as much # meta data as possible from the raw data and then formats it in a BIDS -# compatible way. `raw_to_bids` takes a bunch of inputs, most of which are +# compatible way. `write_raw_bids` takes a bunch of inputs, most of which are # however optional. The required inputs are: # -# * subject_id -# * task -# * raw_file +# * raw +# * bids_fname # * output_path # # ... as you can see in the docstring: @@ -123,10 +132,9 @@ trial_type = {'rest': 0, 'imagine left fist': 1, 'imagine right fist': 2} # Now convert our data to be in a new BIDS dataset. -bids_fname = make_bids_filename(subject=subject_id, task=task, - suffix='eeg.edf') -write_raw_bids(raw_file, bids_fname, output_path, event_id=trial_type) - +bids_fname = make_bids_basename(subject=subject_id, task=task) +write_raw_bids(raw_file, bids_fname, output_path, event_id=trial_type, + events_data=events, overwrite=True) ############################################################################### # What does our fresh BIDS directory look like? print_dir_tree(output_path) @@ -152,15 +160,18 @@ for subj_idx in [1, 2]: for task_idx in [2, 4, 12]: # Load the data - edf_path = os.path.join(data_dir, - 'S{:03}'.format(subj_idx), - 'S{:03}R{:02}.edf'.format(subj_idx, task_idx)) - raw = read_raw_edf(edf_path, preload=True) - bids_fname = make_bids_filename( + edf_path = eegbci.load_data(subject=subj_idx, runs=task_idx)[0] + + raw = mne.io.read_raw_edf(edf_path, preload=False, stim_channel=None) + annot = read_annotations_edf(edf_path) + raw.set_annotations(annot) + events, event_id = mne.events_from_annotations(raw) + + make_bids_basename( subject='{:03}'.format(subj_idx), task=task_names[task_idx], - run=run_mapping[task_idx], suffix='eeg.edf') + run=run_mapping[task_idx]) write_raw_bids(raw, bids_fname, output_path, event_id=trial_type, - overwrite=True) + events_data=events, overwrite=True) ############################################################################### # Step 3: Check and compare with standard diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index 13938bb16..107d83ea1 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -212,8 +212,11 @@ def test_edf(): data_path = op.join(testing.data_path(), 'EDF') raw_fname = op.join(data_path, 'test_reduced.edf') - raw = mne.io.read_raw_edf(raw_fname, stim_channel=None, - preload=False) + raw = mne.io.read_raw_edf(raw_fname, preload=True) + # XXX: hack that should be fixed later. Annotation reading is + # broken for this file with preload=False and read_annotations_edf + raw.preload = False + raw.rename_channels({raw.info['ch_names'][0]: 'EOG'}) raw.info['chs'][0]['coil_type'] = FIFF.FIFFV_COIL_EEG_BIPOLAR raw.rename_channels({raw.info['ch_names'][1]: 'EMG'}) From 0ea50dd3b5251ba70ddb6d2a933535f16dcbacea Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Sat, 27 Oct 2018 01:02:50 -0400 Subject: [PATCH 29/38] TST make circle happy --- .circleci/config.yml | 2 +- examples/convert_group_studies.py | 2 +- examples/convert_mne_sample.py | 2 +- examples/create_bids_folder.py | 8 ++++---- mne_bids/commands/mne_bids_raw_to_bids.py | 1 - 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index d8f9eed14..75f8183df 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -20,7 +20,7 @@ jobs: source activate testenv conda install --yes --quiet numpy=1.13.1 scipy=0.19.1 matplotlib pandas=0.20.3 pip install sphinx numpydoc sphinx-gallery sphinx_bootstrap_theme pillow - pip install mne + pip install -U https://api.github.com/repos/mne-tools/mne-python/zipball/master python setup.py develop - run: name: Build the documentation diff --git a/examples/convert_group_studies.py b/examples/convert_group_studies.py index f21d6821a..368e20d8f 100644 --- a/examples/convert_group_studies.py +++ b/examples/convert_group_studies.py @@ -69,7 +69,7 @@ 'run_%02d_raw.fif' % run) raw = mne.io.read_raw_fif(raw_fname) - bids_fname = ('sub-%02d_ses-01_run-%d_task-VisualFaces' + bids_fname = ('sub-%02d_ses-01_task-VisualFaces_run-%d' % (subject_id, run)) write_raw_bids(raw, bids_fname, output_path, event_id=event_id, overwrite=True) diff --git a/examples/convert_mne_sample.py b/examples/convert_mne_sample.py index 4c34f5112..5e1e32a3d 100644 --- a/examples/convert_mne_sample.py +++ b/examples/convert_mne_sample.py @@ -39,7 +39,7 @@ # Finally, we specify the raw_file and events_data raw = mne.io.read_raw_fif(raw_fname) -bids_fname = 'sub-01_ses-01_run-01_task-audiovisual' +bids_fname = 'sub-01_ses-01_task-audiovisual_run-01' write_raw_bids(raw, bids_fname, output_path, events_data=events_data, event_id=event_id, overwrite=True) diff --git a/examples/create_bids_folder.py b/examples/create_bids_folder.py index 14bf53ae9..4601fc64d 100644 --- a/examples/create_bids_folder.py +++ b/examples/create_bids_folder.py @@ -19,18 +19,18 @@ ############################################################################### # We'll import the relevant functions from the utils module -from mne_bids import make_bids_folders, make_bids_filename +from mne_bids import make_bids_folders, make_bids_basename ############################################################################### # Creating file names for BIDS # ---------------------------- # # BIDS requires a specific ordering and structure for metadata fields in -# file paths, the function `filename_bids` allows you to specify many such +# file paths, the function `make_bids_basename` allows you to specify many such # pieces of metadata, ensuring that they are in the correct order in the # final file path. Omitted keys will not be included in the file path. -my_name = make_bids_filename(subject='test', session='two', task='mytask', +my_name = make_bids_basename(subject='test', session='two', task='mytask', suffix='data.csv') print(my_name) @@ -38,7 +38,7 @@ # You may also omit the suffix, which will result in *only* a prefix for a # file name. This could then prepended to many more files. -my_name = make_bids_filename(subject='test', task='mytask') +my_name = make_bids_basename(subject='test', task='mytask') print(my_name) ############################################################################### diff --git a/mne_bids/commands/mne_bids_raw_to_bids.py b/mne_bids/commands/mne_bids_raw_to_bids.py index c2a6c8da2..558d8142c 100644 --- a/mne_bids/commands/mne_bids_raw_to_bids.py +++ b/mne_bids/commands/mne_bids_raw_to_bids.py @@ -7,7 +7,6 @@ --raw_fname data.edf --output_path new_path """ -import os from mne_bids import write_raw_bids, make_bids_basename from mne_bids.io import _read_raw From 450310b71a83a32575abe96ad193e18ba817c793 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Mon, 29 Oct 2018 12:28:05 -0400 Subject: [PATCH 30/38] FIX --raw_fname -> --raw --- mne_bids/commands/mne_bids_raw_to_bids.py | 4 ++-- mne_bids/tests/test_commands.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mne_bids/commands/mne_bids_raw_to_bids.py b/mne_bids/commands/mne_bids_raw_to_bids.py index 558d8142c..c48356095 100644 --- a/mne_bids/commands/mne_bids_raw_to_bids.py +++ b/mne_bids/commands/mne_bids_raw_to_bids.py @@ -4,7 +4,7 @@ """Command line interface for mne_bids. example usage: $ mne_bids raw_to_bids --subject_id sub01 --task rest ---raw_fname data.edf --output_path new_path +--raw data.edf --output_path new_path """ from mne_bids import write_raw_bids, make_bids_basename @@ -24,7 +24,7 @@ def run(): parser.add_option('--task', dest='task', help='Name of the task the data is based on.', metavar='t') - parser.add_option('--raw_fname', dest='raw_fname', + parser.add_option('--raw', dest='raw_fname', help='The path to the raw MEG file.', metavar='r') parser.add_option('--output_path', dest='output_path', help='The path of the BIDS compatible folder.', diff --git a/mne_bids/tests/test_commands.py b/mne_bids/tests/test_commands.py index 2f9cafa0f..1ef631a3c 100644 --- a/mne_bids/tests/test_commands.py +++ b/mne_bids/tests/test_commands.py @@ -28,7 +28,7 @@ def test_raw_to_bids(): raw_fname = op.join(data_path, 'MEG', 'sample', 'sample_audvis_trunc_raw.fif') # check_usage(mne_bids_raw_to_bids) - with ArgvSetter(('--subject_id', subject_id, '--task', task, '--raw_fname', + with ArgvSetter(('--subject_id', subject_id, '--task', task, '--raw', raw_fname, '--output_path', output_path)): mne_bids_raw_to_bids.run() From 2989e65ae06e5b937cf9870c53988813d540d23c Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Mon, 29 Oct 2018 14:03:40 -0400 Subject: [PATCH 31/38] TST checks for making sure raw data does not change - remove test for single set file as it does not support preload=False - check raw has not been cropped --- doc/index.rst | 3 ++- mne_bids/mne_bids.py | 12 ++++++++++-- mne_bids/tests/test_mne_bids.py | 25 ++++++++++++------------- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/doc/index.rst b/doc/index.rst index 7c7ee5a3a..9be6095d5 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -13,7 +13,8 @@ Installation We recommend the `Anaconda Python distribution `_. To install ``mne_bids``, you first need to install its dependencies:: - $ pip install pandas mne + $ pip install pandas + $ pip install -U https://api.github.com/repos/mne-tools/mne-python/zipball/master Then install mne_bids:: diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index c07cbb33d..4b14ceabd 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -17,6 +17,8 @@ from collections import defaultdict, OrderedDict import numpy as np +from numpy.testing import assert_array_equal + from mne import Epochs from mne.io.constants import FIFF from mne.io.pick import channel_type @@ -34,7 +36,7 @@ copyfile_brainvision, copyfile_eeglab, _infer_eeg_placement_scheme, _parse_bids_filename, _handle_kind) -from .io import (_parse_ext, ALLOWED_EXTENSIONS) +from .io import _parse_ext, ALLOWED_EXTENSIONS, _read_raw ALLOWED_KINDS = ['meg', 'eeg', 'ieeg'] @@ -470,7 +472,6 @@ def _sidecar_json(raw, task, manufacturer, fname, kind, overwrite=False, ('PowerLineFrequency', powerlinefrequency), ('SamplingFrequency', sfreq), ('SoftwareFilters', 'n/a'), - # XXX: If user calls raw.crop, this will be screwed up ('RecordingDuration', raw.times[-1]), ('RecordingType', rec_type)] ch_info_json_meg = [ @@ -600,6 +601,13 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, raw_fname = raw_fname.replace('.fdt', '.set') _, ext = _parse_ext(raw_fname, verbose=verbose) + # XXX: don't have config_fname for BTi files + if ext != '.pdf': + raw_unmodified = _read_raw(raw_fname) + assert_array_equal(raw.times, raw_unmodified.times, + "raw.times should not have changed since reading" + " in from the file. It may have been cropped.") + params = _parse_bids_filename(bids_fname, verbose) subject_id, session_id = params['sub'], params['ses'] acquisition, task, run = params['acq'], params['task'], params['run'] diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index 107d83ea1..d40d5db1d 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -257,25 +257,16 @@ def test_bdf(): cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) + raw.crop(0, raw.times[-2]) + with pytest.raises(AssertionError, match='cropped'): + write_raw_bids(raw, bids_basename, output_path) + def test_set(): """Test write_raw_bids conversion for EEGLAB data.""" # standalone .set file output_path = _TempDir() data_path = op.join(testing.data_path(), 'EEGLAB') - raw_fname = op.join(data_path, 'test_raw_onefile.set') - - raw = mne.io.read_raw_eeglab(raw_fname) - # XXX: remove hack below once mne v0.17 is released - raw._filenames = [raw_fname] - write_raw_bids(raw, bids_basename, output_path, overwrite=False) - - cmd = ['bids-validator', '--bep006', output_path] - run_subprocess(cmd, shell=shell) - - with pytest.raises(OSError, match="already exists"): - write_raw_bids(raw, bids_basename, output_path=output_path, - overwrite=False) # .set with associated .fdt output_path = _TempDir() @@ -285,6 +276,10 @@ def test_set(): raw = mne.io.read_raw_eeglab(raw_fname) write_raw_bids(raw, bids_basename, output_path, overwrite=False) + with pytest.raises(OSError, match="already exists"): + write_raw_bids(raw, bids_basename, output_path=output_path, + overwrite=False) + cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) @@ -300,3 +295,7 @@ def test_cnt(): cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) + + raw = mne.io.read_raw_cnt(raw_fname, montage=None, preload=True) + with pytest.raises(ValueError, match='preload'): + write_raw_bids(raw, bids_basename, output_path) From a4712991a94fbf8d4087fc38a08ec85b832aebac Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Thu, 1 Nov 2018 01:19:04 -0400 Subject: [PATCH 32/38] ENH: Use raw._init_kwargs to check raw.times --- mne_bids/io.py | 36 +++++++++++------------------------- mne_bids/mne_bids.py | 12 +++++------- 2 files changed, 16 insertions(+), 32 deletions(-) diff --git a/mne_bids/io.py b/mne_bids/io.py index 6fec70df7..6c30037eb 100644 --- a/mne_bids/io.py +++ b/mne_bids/io.py @@ -19,6 +19,12 @@ ALLOWED_EXTENSIONS = allowed_extensions_meg + allowed_extensions_eeg +reader = {'.con': io.read_raw_kit, '.sqd': io.read_raw_kit, + '.fif': io.read_raw_fif, '.pdf': io.read_raw_bti, + '.ds': io.read_raw_ctf, '.vhdr': io.read_raw_brainvision, + '.edf': io.read_raw_edf, '.bdf': io.read_raw_edf, + '.set': io.read_raw_eeglab, '.cnt': io.read_raw_cnt} + def _parse_ext(raw_fname, verbose=False): """Split a filename into its name and extension.""" @@ -44,38 +50,18 @@ def _read_raw(raw_fname, electrode=None, hsp=None, hpi=None, config=None, raw = io.read_raw_kit(raw_fname, elp=electrode, hsp=hsp, mrk=hpi, preload=False) - # Neuromag or converted-to-fif systems - elif ext in ['.fif']: - raw = io.read_raw_fif(raw_fname, preload=False) - # BTi systems - elif ext == '.pdf': - if os.path.isfile(raw_fname): + elif ext == '.pdf' and os.path.isfile(raw_fname): raw = io.read_raw_bti(raw_fname, config_fname=config, head_shape_fname=hsp, preload=False, verbose=verbose) - # CTF systems - elif ext == '.ds': - raw = io.read_raw_ctf(raw_fname) - - # EEG File Types - # -------------- - # BrainVision format by Brain Products, links to a .eeg and a .vmrk file - elif ext == '.vhdr': - raw = io.read_raw_brainvision(raw_fname) + elif ext in ['.fif', '.ds', '.vhdr', '.set']: + raw = reader[ext](raw_fname) # EDF (european data format) or BDF (biosemi) format - elif ext == '.edf' or ext == '.bdf': - raw = io.read_raw_edf(raw_fname, preload=True) - - # EEGLAB .set format, a .fdt file is potentially linked from the .set - elif ext == '.set': - raw = io.read_raw_eeglab(raw_fname) - - # Neuroscan .cnt format - elif ext == '.cnt': - raw = io.read_raw_cnt(raw_fname, montage=montage) + elif ext in ['.edf', '.bdf']: + raw = reader[ext](raw_fname, preload=True) # No supported data found ... # --------------------------- diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 4b14ceabd..10bb52839 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -36,7 +36,7 @@ copyfile_brainvision, copyfile_eeglab, _infer_eeg_placement_scheme, _parse_bids_filename, _handle_kind) -from .io import _parse_ext, ALLOWED_EXTENSIONS, _read_raw +from .io import _parse_ext, ALLOWED_EXTENSIONS, reader ALLOWED_KINDS = ['meg', 'eeg', 'ieeg'] @@ -601,12 +601,10 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, raw_fname = raw_fname.replace('.fdt', '.set') _, ext = _parse_ext(raw_fname, verbose=verbose) - # XXX: don't have config_fname for BTi files - if ext != '.pdf': - raw_unmodified = _read_raw(raw_fname) - assert_array_equal(raw.times, raw_unmodified.times, - "raw.times should not have changed since reading" - " in from the file. It may have been cropped.") + raw_orig = reader[ext](**raw._init_kwargs) + assert_array_equal(raw.times, raw_orig.times, + "raw.times should not have changed since reading" + " in from the file. It may have been cropped.") params = _parse_bids_filename(bids_fname, verbose) subject_id, session_id = params['sub'], params['ses'] From 30df0b4338587b619a8b67865d897ab952d0090b Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Thu, 1 Nov 2018 21:48:07 -0400 Subject: [PATCH 33/38] DOC: update readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 4245a816a..df1b92791 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,8 @@ Next to `numpy`, `scipy`, and `matplotlib` that are included in the standard anaconda distribution, you will need to install the following dependencies to be able to use `mne_bids`: - $ pip install pandas mne + $ pip install pandas + $ pip install -U https://api.github.com/repos/mne-tools/mne-python/zipball/master Then install `mne_bids`: From 21c60523cee0b9cdfc1de9f448af504a57882950 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Fri, 2 Nov 2018 12:03:35 -0400 Subject: [PATCH 34/38] TST make travis and appveyor happy --- .travis.yml | 2 +- appveyor.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 15b40ec42..7ae9d1e47 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,7 +18,7 @@ install: - conda install --yes --quiet numpy=1.13.1 scipy=0.19.1 matplotlib pandas=0.20.3 - pip install flake8 pytest pytest-sugar pytest-faulthandler - | - git clone --depth 1 https://github.com/mne-tools/mne-python.git --branch maint/0.16 + git clone --depth 1 https://github.com/mne-tools/mne-python.git cd mne-python pip install --no-deps -e . cd ../ diff --git a/appveyor.yml b/appveyor.yml index 414896aad..328e80262 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -19,7 +19,7 @@ install: - "pip uninstall -yq mne" - |- cd .. - git clone --depth 1 https://github.com/mne-tools/mne-python.git --branch maint/0.16 + git clone --depth 1 https://github.com/mne-tools/mne-python.git cd mne-python python setup.py develop cd .. From 665fbf52a128c660c68ddbe41c31bba05899542c Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Fri, 2 Nov 2018 18:03:10 -0400 Subject: [PATCH 35/38] FIX: address comments by Teon --- mne_bids/commands/mne_bids_raw_to_bids.py | 4 ---- mne_bids/io.py | 2 ++ mne_bids/mne_bids.py | 9 +++------ mne_bids/tests/test_mne_bids.py | 2 +- mne_bids/utils.py | 6 +++--- 5 files changed, 9 insertions(+), 14 deletions(-) diff --git a/mne_bids/commands/mne_bids_raw_to_bids.py b/mne_bids/commands/mne_bids_raw_to_bids.py index c48356095..1e23b1933 100644 --- a/mne_bids/commands/mne_bids_raw_to_bids.py +++ b/mne_bids/commands/mne_bids_raw_to_bids.py @@ -17,7 +17,6 @@ def run(): parser = get_optparser(__file__) - parser.set_defaults(kind='meg') parser.add_option('--subject_id', dest='subject_id', help=('The subject name in BIDS compatible format', '(01,02, etc.)'), metavar='s') @@ -38,9 +37,6 @@ def run(): parser.add_option('--acq', dest='acq', help='The acquisition parameter.', metavar='acq') - parser.add_option('--kind', dest='kind', - help='The kind of data being converted.', - metavar='k') parser.add_option('--events_data', dest='events_data', help='The events file.', metavar='evt') parser.add_option('--event_id', dest='event_id', diff --git a/mne_bids/io.py b/mne_bids/io.py index 6c30037eb..30494376d 100644 --- a/mne_bids/io.py +++ b/mne_bids/io.py @@ -60,6 +60,8 @@ def _read_raw(raw_fname, electrode=None, hsp=None, hpi=None, config=None, raw = reader[ext](raw_fname) # EDF (european data format) or BDF (biosemi) format + # TODO: integrate with lines above once MNE can read + # annotations with preload=False elif ext in ['.edf', '.bdf']: raw = reader[ext](raw_fname, preload=True) diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 10bb52839..c94d3ba8c 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -518,7 +518,7 @@ def _sidecar_json(raw, task, manufacturer, fname, kind, overwrite=False, def write_raw_bids(raw, bids_fname, output_path, events_data=None, - event_id=None, hpi=None, overwrite=False, verbose=True): + event_id=None, overwrite=False, verbose=True): """Walk over a folder of files and create BIDS compatible folder. .. warning:: The original files are simply copied over. This function @@ -556,10 +556,6 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, inferred from the stim channel using `mne.find_events`. event_id : dict | None The event id dict used to create a 'trial_type' column in events.tsv - hpi : None | str - Marker points representing the location of the marker coils with - respect to the MEG Sensors, or path to a marker file. Usually required - for KIT systems. overwrite : bool Whether to overwrite existing files or data in files. Defaults to False. @@ -719,7 +715,8 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, else: sh.copyfile(raw_fname, bids_fname) # KIT data requires the marker file to be copied over too - if hpi is not None: + if 'mrk' in raw._init_kwargs: + hpi = raw._init_kwargs['mrk'] _, marker_ext = _parse_ext(hpi) marker_fname = make_bids_basename( subject=subject_id, session=session_id, task=task, run=run, diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index d40d5db1d..9ead403a9 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -124,7 +124,7 @@ def test_kit(): raw_fname, mrk=hpi_fname, elp=electrode_fname, hsp=headshape_fname) write_raw_bids(raw, bids_basename, output_path, events_data=events_fname, - event_id=event_id, hpi=hpi_fname, overwrite=False) + event_id=event_id, overwrite=False) cmd = ['bids-validator', output_path] run_subprocess(cmd, shell=shell) assert op.exists(op.join(output_path, 'participants.tsv')) diff --git a/mne_bids/utils.py b/mne_bids/utils.py index 9fdaad9e5..3b50acf73 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -101,7 +101,7 @@ def _handle_kind(raw): def make_bids_basename(subject=None, session=None, task=None, acquisition=None, run=None, processing=None, - recording=None, space=None, suffix=None, prefix=None): + recording=None, space=None, prefix=None, suffix=None): """Create a partial/full BIDS filename from its component parts. BIDS filename prefixes have one or more pieces of metadata in them. They @@ -131,11 +131,11 @@ def make_bids_basename(subject=None, session=None, task=None, The recording name for this item. Corresponds to "recording". space : str | None The coordinate space for an anatomical file. Corresponds to "space". - suffix : str | None - The suffix of a file that begins with this prefix. E.g., 'audio.wav'. prefix : str | None The prefix for the filename to be created. E.g., a path to the folder in which you wish to create a file with this name. + suffix : str | None + The suffix of a file that begins with this prefix. E.g., 'audio.wav'. Returns ------- From 6e5beeafcd778e110bf03e5dfbdfde60497babf4 Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Fri, 2 Nov 2018 21:28:44 -0400 Subject: [PATCH 36/38] bids_fname -> bids_basename --- examples/convert_eeg_to_bids.py | 8 ++++---- examples/convert_group_studies.py | 6 +++--- examples/convert_mne_sample.py | 4 ++-- examples/create_bids_folder.py | 14 +++++++------- mne_bids/commands/mne_bids_raw_to_bids.py | 4 ++-- mne_bids/mne_bids.py | 15 ++++++++------- mne_bids/tests/test_mne_bids.py | 18 +++++++++--------- 7 files changed, 35 insertions(+), 34 deletions(-) diff --git a/examples/convert_eeg_to_bids.py b/examples/convert_eeg_to_bids.py index 9508e52cd..b6bb932f1 100644 --- a/examples/convert_eeg_to_bids.py +++ b/examples/convert_eeg_to_bids.py @@ -109,7 +109,7 @@ # however optional. The required inputs are: # # * raw -# * bids_fname +# * bids_basename # * output_path # # ... as you can see in the docstring: @@ -132,8 +132,8 @@ trial_type = {'rest': 0, 'imagine left fist': 1, 'imagine right fist': 2} # Now convert our data to be in a new BIDS dataset. -bids_fname = make_bids_basename(subject=subject_id, task=task) -write_raw_bids(raw_file, bids_fname, output_path, event_id=trial_type, +bids_basename = make_bids_basename(subject=subject_id, task=task) +write_raw_bids(raw_file, bids_basename, output_path, event_id=trial_type, events_data=events, overwrite=True) ############################################################################### # What does our fresh BIDS directory look like? @@ -170,7 +170,7 @@ make_bids_basename( subject='{:03}'.format(subj_idx), task=task_names[task_idx], run=run_mapping[task_idx]) - write_raw_bids(raw, bids_fname, output_path, event_id=trial_type, + write_raw_bids(raw, bids_basename, output_path, event_id=trial_type, events_data=events, overwrite=True) ############################################################################### diff --git a/examples/convert_group_studies.py b/examples/convert_group_studies.py index 368e20d8f..97ccbf377 100644 --- a/examples/convert_group_studies.py +++ b/examples/convert_group_studies.py @@ -69,9 +69,9 @@ 'run_%02d_raw.fif' % run) raw = mne.io.read_raw_fif(raw_fname) - bids_fname = ('sub-%02d_ses-01_task-VisualFaces_run-%d' - % (subject_id, run)) - write_raw_bids(raw, bids_fname, output_path, event_id=event_id, + bids_basename = ('sub-%02d_ses-01_task-VisualFaces_run-%d' + % (subject_id, run)) + write_raw_bids(raw, bids_basename, output_path, event_id=event_id, overwrite=True) ############################################################################### diff --git a/examples/convert_mne_sample.py b/examples/convert_mne_sample.py index 5e1e32a3d..a49e65259 100644 --- a/examples/convert_mne_sample.py +++ b/examples/convert_mne_sample.py @@ -39,8 +39,8 @@ # Finally, we specify the raw_file and events_data raw = mne.io.read_raw_fif(raw_fname) -bids_fname = 'sub-01_ses-01_task-audiovisual_run-01' -write_raw_bids(raw, bids_fname, output_path, events_data=events_data, +bids_basename = 'sub-01_ses-01_task-audiovisual_run-01' +write_raw_bids(raw, bids_basename, output_path, events_data=events_data, event_id=event_id, overwrite=True) ############################################################################### diff --git a/examples/create_bids_folder.py b/examples/create_bids_folder.py index 4601fc64d..dc1e7c694 100644 --- a/examples/create_bids_folder.py +++ b/examples/create_bids_folder.py @@ -9,8 +9,8 @@ .. note:: - You may automatically convert Raw files to a BIDS-compatible folder with - `raw_to_bids`. This example is for manually creating files/folders. + You may automatically convert Raw objects to BIDS-compatible files with + `write_raw_bids`. This example is for manually creating files/folders. """ # Authors: Chris Holdgraf @@ -30,16 +30,16 @@ # pieces of metadata, ensuring that they are in the correct order in the # final file path. Omitted keys will not be included in the file path. -my_name = make_bids_basename(subject='test', session='two', task='mytask', - suffix='data.csv') -print(my_name) +bids_basename = make_bids_basename(subject='test', session='two', + task='mytask', suffix='data.csv') +print(bids_basename) ############################################################################### # You may also omit the suffix, which will result in *only* a prefix for a # file name. This could then prepended to many more files. -my_name = make_bids_basename(subject='test', task='mytask') -print(my_name) +bids_basename = make_bids_basename(subject='test', task='mytask') +print(bids_basename) ############################################################################### # Creating folders diff --git a/mne_bids/commands/mne_bids_raw_to_bids.py b/mne_bids/commands/mne_bids_raw_to_bids.py index 1e23b1933..83550923b 100644 --- a/mne_bids/commands/mne_bids_raw_to_bids.py +++ b/mne_bids/commands/mne_bids_raw_to_bids.py @@ -57,12 +57,12 @@ def run(): opt, args = parser.parse_args() - bids_fname = make_bids_basename( + bids_basename = make_bids_basename( subject=opt.subject_id, session=opt.session_id, run=opt.run, acquisition=opt.acq, task=opt.task) raw = _read_raw(opt.raw_fname, hpi=opt.hpi, electrode=opt.electrode, hsp=opt.hsp, config=opt.config) - write_raw_bids(raw, bids_fname, opt.output_path, event_id=opt.event_id, + write_raw_bids(raw, bids_basename, opt.output_path, event_id=opt.event_id, events_data=opt.events_data, overwrite=opt.overwrite, verbose=True) diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index c94d3ba8c..5d50af40d 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -517,19 +517,20 @@ def _sidecar_json(raw, task, manufacturer, fname, kind, overwrite=False, return fname -def write_raw_bids(raw, bids_fname, output_path, events_data=None, +def write_raw_bids(raw, bids_basename, output_path, events_data=None, event_id=None, overwrite=False, verbose=True): """Walk over a folder of files and create BIDS compatible folder. .. warning:: The original files are simply copied over. This function - cannot convert data files from one format to another. + cannot convert modify data files from one format to another. + Modification of the original data files is not allowed. Parameters ---------- raw : instance of mne.Raw The raw data. It must be an instance of mne.Raw. The data should not be loaded on disk, i.e., raw.preload must be False. - bids_fname : str + bids_basename : str The base filename of the BIDS compatible files. Typically, this can be generated using make_bids_basename. Example: sub-01_ses-01_task-testing_acq-01_run-01 @@ -549,7 +550,7 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, output_path : str The path of the root of the BIDS compatible folder. The session and subject specific folders will be populated automatically by parsing - bids_fname. + bids_basename. events_data : str | array | None The events file. If a string, a path to the events file. If an array, the MNE events array (shape n_events, 3). If None, events will be @@ -602,12 +603,12 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, "raw.times should not have changed since reading" " in from the file. It may have been cropped.") - params = _parse_bids_filename(bids_fname, verbose) + params = _parse_bids_filename(bids_basename, verbose) subject_id, session_id = params['sub'], params['ses'] acquisition, task, run = params['acq'], params['task'], params['run'] kind = _handle_kind(raw) - bids_fname = bids_fname + '_%s%s' % (kind, ext) + bids_fname = bids_basename + '_%s%s' % (kind, ext) data_path = make_bids_folders(subject=subject_id, session=session_id, kind=kind, root=output_path, overwrite=False, verbose=verbose) @@ -677,7 +678,7 @@ def write_raw_bids(raw, bids_fname, output_path, events_data=None, _mkdir_p(os.path.dirname(bids_fname)) if verbose: - print('Writing data files to %s' % bids_fname) + print('Copying data files to %s' % bids_fname) if ext not in ALLOWED_EXTENSIONS: raise ValueError('ext must be in %s, got %s' diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index 9ead403a9..69a2baa45 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -72,8 +72,8 @@ def test_fif(): raw_fname2 = op.join(data_path2, 'sample_audvis_raw.fif') raw.save(raw_fname2) - bids_fname = bids_basename.replace(subject_id, subject_id2) - write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, + bids_basename2 = bids_basename.replace(subject_id, subject_id2) + write_raw_bids(raw, bids_basename2, output_path, events_data=events_fname, event_id=event_id, overwrite=False) # check that the overwrite parameters work correctly for the participant # data @@ -81,27 +81,27 @@ def test_fif(): raw.info['subject_info'] = {'his_id': subject_id2, 'birthday': (1994, 1, 26), 'sex': 2} with pytest.raises(OSError, match="already exists"): - write_raw_bids(raw, bids_fname, output_path, + write_raw_bids(raw, bids_basename2, output_path, events_data=events_fname, event_id=event_id, overwrite=False) # now force the overwrite - write_raw_bids(raw, bids_fname, output_path, events_data=events_fname, + write_raw_bids(raw, bids_basename2, output_path, events_data=events_fname, event_id=event_id, overwrite=True) with pytest.raises(ValueError, match='raw_file must be'): write_raw_bids('blah', bids_basename, output_path) - bids_fname = 'sub-01_ses-01_xyz-01_run-01' + bids_basename2 = 'sub-01_ses-01_xyz-01_run-01' with pytest.raises(KeyError, match='Unexpected entity'): - write_raw_bids(raw, bids_fname, output_path) + write_raw_bids(raw, bids_basename2, output_path) - bids_fname = 'sub-01_run-01_task-auditory' + bids_basename2 = 'sub-01_run-01_task-auditory' with pytest.raises(ValueError, match='ordered correctly'): - write_raw_bids(raw, bids_fname, output_path, overwrite=True) + write_raw_bids(raw, bids_basename2, output_path, overwrite=True) del raw._filenames with pytest.raises(ValueError, match='raw.filenames is missing'): - write_raw_bids(raw, bids_fname, output_path) + write_raw_bids(raw, bids_basename2, output_path) cmd = ['bids-validator', output_path] run_subprocess(cmd, shell=shell) From 4c378c5a30a7c63defcc991371339b8aa96aebd7 Mon Sep 17 00:00:00 2001 From: Alexandre Gramfort Date: Sat, 3 Nov 2018 13:28:05 +0100 Subject: [PATCH 37/38] misc --- examples/convert_eeg_to_bids.py | 2 +- mne_bids/mne_bids.py | 2 +- setup.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/convert_eeg_to_bids.py b/examples/convert_eeg_to_bids.py index b6bb932f1..c61897bb1 100644 --- a/examples/convert_eeg_to_bids.py +++ b/examples/convert_eeg_to_bids.py @@ -92,7 +92,7 @@ raw = mne.io.read_raw_edf(edf_path, preload=False, stim_channel=None) ############################################################################### -# The annotations stored in the file mut be read in separately and converted +# The annotations stored in the file must be read in separately and converted # into a 2D numpy array of events that is compatible with MNE. annot = read_annotations_edf(edf_path) raw.set_annotations(annot) diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 5d50af40d..02fa10b4c 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -527,7 +527,7 @@ def write_raw_bids(raw, bids_basename, output_path, events_data=None, Parameters ---------- - raw : instance of mne.Raw + raw : instance of mne.io.Raw The raw data. It must be an instance of mne.Raw. The data should not be loaded on disk, i.e., raw.preload must be False. bids_basename : str diff --git a/setup.py b/setup.py index f47621237..681e03801 100644 --- a/setup.py +++ b/setup.py @@ -5,9 +5,9 @@ DISTNAME = 'mne-bids' DESCRIPTION = descr -MAINTAINER = 'Alexandre Gramfort' -MAINTAINER_EMAIL = 'alexandre.gramfort@telecom-paristech.fr' -URL = 'http://martinos.org/mne' +MAINTAINER = 'Mainak Jas' +MAINTAINER_EMAIL = 'mainakjas@gmail.com' +URL = 'https://mne-tools.github.io/mne-bids/' LICENSE = 'BSD (3-clause)' DOWNLOAD_URL = 'http://github.com/mne-tools/mne-bids' VERSION = '0.1.dev0' From 75b0cb6ab54e16133b02ee943378043693b5a78b Mon Sep 17 00:00:00 2001 From: Mainak Jas Date: Sat, 3 Nov 2018 16:43:06 -0400 Subject: [PATCH 38/38] root -> output_path and _age_on_date private --- examples/create_bids_folder.py | 2 +- mne_bids/mne_bids.py | 8 ++++---- mne_bids/tests/test_mne_bids.py | 2 +- mne_bids/tests/test_utils.py | 17 +++++++++-------- mne_bids/utils.py | 19 ++++++++++--------- 5 files changed, 25 insertions(+), 23 deletions(-) diff --git a/examples/create_bids_folder.py b/examples/create_bids_folder.py index dc1e7c694..463c9a46e 100644 --- a/examples/create_bids_folder.py +++ b/examples/create_bids_folder.py @@ -48,7 +48,7 @@ # You can also use MNE-BIDS to create folder hierarchies. path_folder = make_bids_folders('sub_01', session='mysession', - kind='meg', root='path/to/project', + kind='meg', output_path='path/to/project', make_dir=False) print(path_folder) diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index 02fa10b4c..de707ea3b 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -32,7 +32,7 @@ from .pick import coil_type from .utils import (make_bids_basename, make_bids_folders, make_dataset_description, _write_json, _write_tsv, - _read_events, _mkdir_p, age_on_date, + _read_events, _mkdir_p, _age_on_date, copyfile_brainvision, copyfile_eeglab, _infer_eeg_placement_scheme, _parse_bids_filename, _handle_kind) @@ -252,7 +252,7 @@ def _participants_tsv(raw, subject_id, group, fname, overwrite=False, if meas_date is not None and age is not None: bday = datetime(age[0], age[1], age[2]) meas_datetime = datetime.fromtimestamp(meas_date) - subject_age = age_on_date(bday, meas_datetime) + subject_age = _age_on_date(bday, meas_datetime) else: subject_age = "n/a" @@ -610,13 +610,13 @@ def write_raw_bids(raw, bids_basename, output_path, events_data=None, bids_fname = bids_basename + '_%s%s' % (kind, ext) data_path = make_bids_folders(subject=subject_id, session=session_id, - kind=kind, root=output_path, + kind=kind, output_path=output_path, overwrite=False, verbose=verbose) if session_id is None: ses_path = os.sep.join(data_path.split(os.sep)[:-1]) else: ses_path = make_bids_folders(subject=subject_id, session=session_id, - root=output_path, make_dir=False, + output_path=output_path, make_dir=False, overwrite=False, verbose=verbose) # create filenames diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index 69a2baa45..5a64914ef 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -201,7 +201,7 @@ def test_vhdr(): # create another bids folder with the overwrite command and check # no files are in the folder data_path = make_bids_folders(subject=subject_id, session=session_id, - kind='eeg', root=output_path, + kind='eeg', output_path=output_path, overwrite=True) assert len([f for f in os.listdir(data_path) if op.isfile(f)]) == 0 diff --git a/mne_bids/tests/test_utils.py b/mne_bids/tests/test_utils.py index acb0b4c8d..9619f777c 100644 --- a/mne_bids/tests/test_utils.py +++ b/mne_bids/tests/test_utils.py @@ -14,7 +14,7 @@ from mne.utils import _TempDir from mne_bids.utils import (make_bids_folders, make_bids_basename, - _check_types, print_dir_tree, age_on_date, + _check_types, print_dir_tree, _age_on_date, _get_brainvision_paths, copyfile_brainvision, copyfile_eeglab, _infer_eeg_placement_scheme) @@ -50,14 +50,15 @@ def test_make_folders(): """Test that folders are created and named properly.""" # Make sure folders are created properly output_path = _TempDir() - make_bids_folders(subject='hi', session='foo', kind='ba', root=output_path) + make_bids_folders(subject='hi', session='foo', kind='ba', + output_path=output_path) assert op.isdir(op.join(output_path, 'sub-hi', 'ses-foo', 'ba')) # If we remove a kwarg the folder shouldn't be created output_path = _TempDir() - make_bids_folders(subject='hi', kind='ba', root=output_path) + make_bids_folders(subject='hi', kind='ba', output_path=output_path) assert op.isdir(op.join(output_path, 'sub-hi', 'ba')) # check overwriting of folders - make_bids_folders(subject='hi', kind='ba', root=output_path, + make_bids_folders(subject='hi', kind='ba', output_path=output_path, overwrite=True, verbose=True) @@ -75,11 +76,11 @@ def test_age_on_date(): exp2 = datetime(2018, 1, 26) exp3 = datetime(2018, 1, 27) exp4 = datetime(1990, 1, 1) - assert age_on_date(bday, exp1) == 23 - assert age_on_date(bday, exp2) == 24 - assert age_on_date(bday, exp3) == 24 + assert _age_on_date(bday, exp1) == 23 + assert _age_on_date(bday, exp2) == 24 + assert _age_on_date(bday, exp3) == 24 with pytest.raises(ValueError): - age_on_date(bday, exp4) + _age_on_date(bday, exp4) def test_get_brainvision_paths(): diff --git a/mne_bids/utils.py b/mne_bids/utils.py index 3b50acf73..34a2ccfe6 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -180,12 +180,12 @@ def make_bids_basename(subject=None, session=None, task=None, return filename -def make_bids_folders(subject, session=None, kind=None, root=None, +def make_bids_folders(subject, session=None, kind=None, output_path=None, make_dir=True, overwrite=False, verbose=False): """Create a BIDS folder hierarchy. This creates a hierarchy of folders *within* a BIDS dataset. You should - plan to create these folders *inside* the root folder of the dataset. + plan to create these folders *inside* the output_path folder of the dataset. Parameters ---------- @@ -196,8 +196,8 @@ def make_bids_folders(subject, session=None, kind=None, root=None, "anat", "func", etc. session : str | None The session for a item. Corresponds to "ses". - root : str | None - The root for the folders to be created. If None, folders will be + output_path : str | None + The output_path for the folders to be created. If None, folders will be created in the current working directory. make_dir : bool Whether to actually create the folders specified. If False, only a @@ -219,11 +219,12 @@ def make_bids_folders(subject, session=None, kind=None, root=None, Examples -------- >>> print(make_bids_folders('sub_01', session='my_session', - kind='meg', root='path/to/project', make_dir=False)) # noqa + kind='meg', output_path='path/to/project', + make_dir=False)) # noqa path/to/project/sub-sub_01/ses-my_session/meg """ - _check_types((subject, kind, session, root)) + _check_types((subject, kind, session, output_path)) if session is not None: _check_key_val('ses', session) @@ -233,8 +234,8 @@ def make_bids_folders(subject, session=None, kind=None, root=None, if isinstance(kind, string_types): path.append(kind) path = op.join(*path) - if isinstance(root, string_types): - path = op.join(root, path) + if isinstance(output_path, string_types): + path = op.join(output_path, path) if make_dir is True: _mkdir_p(path, overwrite=overwrite, verbose=verbose) @@ -310,7 +311,7 @@ def make_dataset_description(path, name=None, data_license=None, _write_json(description, fname, overwrite=True, verbose=verbose) -def age_on_date(bday, exp_date): +def _age_on_date(bday, exp_date): """Calculate age from birthday and experiment date. Parameters