From ff78a1d21607eade64d75166c03044f50781a085 Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Fri, 9 Nov 2018 21:47:36 +0100 Subject: [PATCH 1/9] add ieeg formats to io --- mne_bids/io.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/mne_bids/io.py b/mne_bids/io.py index 30494376d..1978ea1d8 100644 --- a/mne_bids/io.py +++ b/mne_bids/io.py @@ -17,7 +17,16 @@ '.cnt', # Neuroscan ] -ALLOWED_EXTENSIONS = allowed_extensions_meg + allowed_extensions_eeg +allowed_extensions_ieeg = ['.vhdr', # BrainVision, accompanied by .vmrk, .eeg + '.edf', # European Data Format + '.set', # EEGLAB, potentially accompanied by .fdt + '.mef', # MEF: Multiscale Electrophysiology File + '.nwb', # Neurodata without borders + ] + +ALLOWED_EXTENSIONS = (allowed_extensions_meg + + allowed_extensions_eeg + + allowed_extensions_ieeg) reader = {'.con': io.read_raw_kit, '.sqd': io.read_raw_kit, '.fif': io.read_raw_fif, '.pdf': io.read_raw_bti, @@ -43,8 +52,6 @@ def _read_raw(raw_fname, electrode=None, hsp=None, hpi=None, config=None, """Read a raw file into MNE, making inferences based on extension.""" fname, ext = _parse_ext(raw_fname) - # MEG File Types - # -------------- # KIT systems if ext in ['.con', '.sqd']: raw = io.read_raw_kit(raw_fname, elp=electrode, hsp=hsp, @@ -65,6 +72,11 @@ def _read_raw(raw_fname, electrode=None, hsp=None, hpi=None, config=None, elif ext in ['.edf', '.bdf']: raw = reader[ext](raw_fname, preload=True) + # MEF and NWB are allowed, but not yet implemented + elif ext in ['.mef', '.nwb']: + raise ValueError("Got % as extension. This is an allowed extension but" + " there is no IO support for this file format yet.") + # No supported data found ... # --------------------------- else: From 6dd8cccfbc755154d4ff3845b2607da07ccbc8ea Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Fri, 9 Nov 2018 22:00:42 +0100 Subject: [PATCH 2/9] add ieeg fields to json and manufacturers --- mne_bids/mne_bids.py | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/mne_bids/mne_bids.py b/mne_bids/mne_bids.py index b28292a4b..af534295c 100644 --- a/mne_bids/mne_bids.py +++ b/mne_bids/mne_bids.py @@ -52,19 +52,24 @@ '.meg4': 'CTF'} eeg_manufacturers = {'.vhdr': 'BrainProducts', '.eeg': 'BrainProducts', - '.edf': 'Mixed', '.bdf': 'Biosemi', '.set': 'Mixed', - '.fdt': 'Mixed', '.cnt': 'Neuroscan'} + '.edf': 'n/a', '.bdf': 'Biosemi', '.set': 'n/a', + '.fdt': 'n/a', '.cnt': 'Neuroscan'} + +ieeg_manufacturers = {'.vhdr': 'BrainProducts', '.eeg': 'BrainProducts', + '.edf': 'n/a', '.set': 'n/a', '.fdt': 'n/a', + '.mef': 'n/a', '.nwb': 'n/a'} # Merge the manufacturer dictionaries in a python2 / python3 compatible way MANUFACTURERS = dict() MANUFACTURERS.update(meg_manufacturers) MANUFACTURERS.update(eeg_manufacturers) +MANUFACTURERS.update(ieeg_manufacturers) # List of synthetic channels by manufacturer that are to be excluded from the # channel list. Currently this is only for stimulus channels. IGNORED_CHANNELS = {'KIT/Yokogawa': ['STI 014'], 'BrainProducts': ['STI 014'], - 'Mixed': ['STI 014'], + 'n/a': ['STI 014'], # for unknown manufacturers, ignore it 'Biosemi': ['STI 014'], 'Neuroscan': ['STI 014']} @@ -135,10 +140,10 @@ def _channels_tsv(raw, fname, overwrite=False, verbose=True): ('name', raw.info['ch_names']), ('type', ch_type), ('units', units), - ('description', description), - ('sampling_frequency', np.full((n_channels), sfreq)), ('low_cutoff', np.full((n_channels), low_cutoff)), ('high_cutoff', np.full((n_channels), high_cutoff)), + ('description', description), + ('sampling_frequency', np.full((n_channels), sfreq)), ('status', status)])) df.drop(ignored_indexes, inplace=True) @@ -486,6 +491,7 @@ def _sidecar_json(raw, task, manufacturer, fname, kind, overwrite=False, ('EEGPlacementScheme', _infer_eeg_placement_scheme(raw)), ('Manufacturer', manufacturer)] ch_info_json_ieeg = [ + ('iEEGReference', 'n/a'), ('ECOGChannelCount', n_ecogchan), ('SEEGChannelCount', n_seegchan)] ch_info_ch_counts = [ @@ -646,8 +652,6 @@ def write_raw_bids(raw, bids_basename, output_path, events_data=None, orient = ORIENTATION.get(ext, 'n/a') unit = UNITS.get(ext, 'n/a') manufacturer = MANUFACTURERS.get(ext, 'n/a') - if manufacturer == 'Mixed': - manufacturer = 'n/a' # save all meta data _participants_tsv(raw, subject_id, "n/a", participants_fname, overwrite, @@ -700,9 +704,7 @@ def write_raw_bids(raw, bids_basename, output_path, events_data=None, 'Please contact MNE developers if you have ' 'any questions.') else: - # TODO insert arg `split_naming=split_naming` - # when MNE releases 0.17 - raw.save(bids_fname, overwrite=True) + raw.save(bids_fname, overwrite=True, split_naming='bids') # CTF data is saved in a directory elif ext == '.ds': From 18ffe8cf5e0974f925d6c93fc18625725fa636ce Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Fri, 9 Nov 2018 22:15:57 +0100 Subject: [PATCH 3/9] test ieeg code in io module --- mne_bids/tests/test_io.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/mne_bids/tests/test_io.py b/mne_bids/tests/test_io.py index a907e16ca..d1fb315a5 100644 --- a/mne_bids/tests/test_io.py +++ b/mne_bids/tests/test_io.py @@ -2,8 +2,11 @@ # Authors: Stefan Appelhoff # # License: BSD (3-clause) +import os.path as op import pytest +from mne.utils import _TempDir + from mne_bids.io import _parse_ext, _read_raw @@ -25,5 +28,17 @@ def test_read_raw(): """Test the raw reading.""" # Use a file ending that does not exist f = 'file.bogus' - with pytest.raises(ValueError): + with pytest.raises(ValueError, match='file name extension must be one of'): _read_raw(f) + + +def test_not_implemented(): + """Test the not yet implemented data formats raise an adequate error.""" + for not_implemented_ext in ['.mef', '.nwb']: + data_path = _TempDir() + raw_fname = op.join(data_path, 'test' + not_implemented_ext) + with open(raw_fname, 'w'): + pass + with pytest.raises(ValueError, match=('there is no IO support for ' + 'this file format yet')): + _read_raw(raw_fname) From 9c98b715286273321fb2955dbde025742878a2b3 Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Fri, 9 Nov 2018 22:38:24 +0100 Subject: [PATCH 4/9] test brainvision, edf and eeglab for ieeg modality --- mne_bids/tests/test_mne_bids.py | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index 5a64914ef..36fb2a05d 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -45,8 +45,6 @@ shell = True -# MEG Tests -# --------- def test_fif(): """Test functionality of the write_raw_bids conversion for fif.""" output_path = _TempDir() @@ -184,8 +182,6 @@ def test_bti(): run_subprocess(cmd, shell=shell) -# EEG Tests -# --------- def test_vhdr(): """Test write_raw_bids conversion for BrainVision data.""" output_path = _TempDir() @@ -205,6 +201,15 @@ def test_vhdr(): overwrite=True) assert len([f for f in os.listdir(data_path) if op.isfile(f)]) == 0 + # Also cover iEEG + # We use the same data and pretend that eeg channels are ecog + raw.set_channel_types({'FP1': 'ecog'}) + output_path = _TempDir() + write_raw_bids(raw, bids_basename, output_path, overwrite=False) + + cmd = ['bids-validator', '--bep010', output_path] + run_subprocess(cmd, shell=shell) + def test_edf(): """Test write_raw_bids conversion for European Data Format data.""" @@ -244,6 +249,15 @@ def test_edf(): df = pd.read_csv(scans_tsv, sep='\t') assert df.shape[0] == 2 + # Also cover iEEG + # We use the same data and pretend that eeg channels are ecog + raw.set_channel_types({'EMG': 'ecog'}) + output_path = _TempDir() + write_raw_bids(raw, bids_basename, output_path) + + cmd = ['bids-validator', '--bep010', output_path] + run_subprocess(cmd, shell=shell) + def test_bdf(): """Test write_raw_bids conversion for Biosemi data.""" @@ -283,6 +297,15 @@ def test_set(): cmd = ['bids-validator', '--bep006', output_path] run_subprocess(cmd, shell=shell) + # Also cover iEEG + # We use the same data and pretend that eeg channels are ecog + raw.set_channel_types({'EEG 000': 'ecog'}) + output_path = _TempDir() + write_raw_bids(raw, bids_basename, output_path) + + cmd = ['bids-validator', '--bep010', output_path] + run_subprocess(cmd, shell=shell) + def test_cnt(): """Test write_raw_bids conversion for Neuroscan data.""" From 0372208463d953ac8d2d5cea4e875e830d3a1d2a Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Fri, 9 Nov 2018 22:59:14 +0100 Subject: [PATCH 5/9] change order of kind extraction in favor of ieeg and add a test --- mne_bids/tests/test_utils.py | 25 ++++++++++++++++++++++++- mne_bids/utils.py | 4 ++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/mne_bids/tests/test_utils.py b/mne_bids/tests/test_utils.py index 9619f777c..7c18344f2 100644 --- a/mne_bids/tests/test_utils.py +++ b/mne_bids/tests/test_utils.py @@ -9,6 +9,7 @@ from datetime import datetime from scipy.io import savemat +from numpy.random import random import mne from mne.datasets import testing from mne.utils import _TempDir @@ -16,11 +17,33 @@ from mne_bids.utils import (make_bids_folders, make_bids_basename, _check_types, print_dir_tree, _age_on_date, _get_brainvision_paths, copyfile_brainvision, - copyfile_eeglab, _infer_eeg_placement_scheme) + copyfile_eeglab, _infer_eeg_placement_scheme, + _handle_kind) base_path = op.join(op.dirname(mne.__file__), 'io') +def test_handle_kind(): + """Test the automatic extraction of kind from the data.""" + # Create a dummy raw + n_channels = 1 + sampling_rate = 100 + data = random((n_channels, sampling_rate)) + channel_types = ['grad', 'eeg', 'ecog', 'misc'] + expected_kinds = ['meg', 'eeg', 'ieeg', 'error'] + # do it once for each type ... and once for "no type" + for chtype, kind in zip(channel_types, expected_kinds): + info = mne.create_info(n_channels, sampling_rate, ch_types=[chtype]) + raw = mne.io.RawArray(data, info) + if kind != 'error': + assert _handle_kind(raw) == kind + continue + + # if we cannot find a proper channel type, we raise an error + with pytest.raises(ValueError, match='Neither MEG/EEG/iEEG channels'): + _handle_kind(raw) + + def test_print_dir_tree(): """Test printing a dir tree.""" with pytest.raises(ValueError): diff --git a/mne_bids/utils.py b/mne_bids/utils.py index 34a2ccfe6..a95fe9715 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -88,10 +88,10 @@ def _handle_kind(raw): """Get kind.""" if 'meg' in raw: kind = 'meg' - elif 'eeg' in raw: - kind = 'eeg' elif 'ecog' in raw: kind = 'ieeg' + elif 'eeg' in raw: + kind = 'eeg' else: raise ValueError('Neither MEG/EEG/iEEG channels found in data.' 'Please use raw.set_channel_types to set the ' From ce4069881177a6e8470560b06b4b9ccb280ae8fc Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Fri, 9 Nov 2018 22:59:35 +0100 Subject: [PATCH 6/9] fix string formatting to {} --- mne_bids/io.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/mne_bids/io.py b/mne_bids/io.py index 1978ea1d8..34ebad689 100644 --- a/mne_bids/io.py +++ b/mne_bids/io.py @@ -74,12 +74,13 @@ def _read_raw(raw_fname, electrode=None, hsp=None, hpi=None, config=None, # MEF and NWB are allowed, but not yet implemented elif ext in ['.mef', '.nwb']: - raise ValueError("Got % as extension. This is an allowed extension but" - " there is no IO support for this file format yet.") + raise ValueError('Got "{}" as extension. This is an allowed extension ' + 'but there is no IO support for this file format yet.' + .format(ext)) # No supported data found ... # --------------------------- else: - raise ValueError("Raw file name extension must be one of %\n" - "Got %" % (ALLOWED_EXTENSIONS, ext)) + raise ValueError('Raw file name extension must be one of {}\n' + 'Got {}'.format(ALLOWED_EXTENSIONS, ext)) return raw From 86340c0e0fa82875ec5e48c4c1d13678e05ae0c5 Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Sat, 10 Nov 2018 09:23:32 +0100 Subject: [PATCH 7/9] take valueerror test out of the loop --- mne_bids/tests/test_utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/mne_bids/tests/test_utils.py b/mne_bids/tests/test_utils.py index 7c18344f2..3f3df03e0 100644 --- a/mne_bids/tests/test_utils.py +++ b/mne_bids/tests/test_utils.py @@ -29,19 +29,19 @@ def test_handle_kind(): n_channels = 1 sampling_rate = 100 data = random((n_channels, sampling_rate)) - channel_types = ['grad', 'eeg', 'ecog', 'misc'] - expected_kinds = ['meg', 'eeg', 'ieeg', 'error'] + channel_types = ['grad', 'eeg', 'ecog'] + expected_kinds = ['meg', 'eeg', 'ieeg'] # do it once for each type ... and once for "no type" for chtype, kind in zip(channel_types, expected_kinds): info = mne.create_info(n_channels, sampling_rate, ch_types=[chtype]) raw = mne.io.RawArray(data, info) - if kind != 'error': - assert _handle_kind(raw) == kind - continue + assert _handle_kind(raw) == kind - # if we cannot find a proper channel type, we raise an error - with pytest.raises(ValueError, match='Neither MEG/EEG/iEEG channels'): - _handle_kind(raw) + # if we cannot find a proper channel type, we raise an error + with pytest.raises(ValueError, match='Neither MEG/EEG/iEEG channels'): + info = mne.create_info(n_channels, sampling_rate, ch_types=['misc']) + raw = mne.io.RawArray(data, info) + _handle_kind(raw) def test_print_dir_tree(): From 5af805a9ec8e6c93eb63bbaa060ed7767304058e Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Sun, 11 Nov 2018 11:00:33 +0100 Subject: [PATCH 8/9] raise error if eeg AND ecog channels: ambiguous kind --- mne_bids/tests/test_mne_bids.py | 9 ++++++--- mne_bids/tests/test_utils.py | 7 +++++++ mne_bids/utils.py | 8 ++++++-- 3 files changed, 19 insertions(+), 5 deletions(-) diff --git a/mne_bids/tests/test_mne_bids.py b/mne_bids/tests/test_mne_bids.py index 36fb2a05d..235aadc9b 100644 --- a/mne_bids/tests/test_mne_bids.py +++ b/mne_bids/tests/test_mne_bids.py @@ -203,7 +203,8 @@ def test_vhdr(): # Also cover iEEG # We use the same data and pretend that eeg channels are ecog - raw.set_channel_types({'FP1': 'ecog'}) + raw.set_channel_types({raw.ch_names[i]: 'ecog' + for i in mne.pick_types(raw.info, eeg=True)}) output_path = _TempDir() write_raw_bids(raw, bids_basename, output_path, overwrite=False) @@ -251,7 +252,8 @@ def test_edf(): # Also cover iEEG # We use the same data and pretend that eeg channels are ecog - raw.set_channel_types({'EMG': 'ecog'}) + raw.set_channel_types({raw.ch_names[i]: 'ecog' + for i in mne.pick_types(raw.info, eeg=True)}) output_path = _TempDir() write_raw_bids(raw, bids_basename, output_path) @@ -299,7 +301,8 @@ def test_set(): # Also cover iEEG # We use the same data and pretend that eeg channels are ecog - raw.set_channel_types({'EEG 000': 'ecog'}) + raw.set_channel_types({raw.ch_names[i]: 'ecog' + for i in mne.pick_types(raw.info, eeg=True)}) output_path = _TempDir() write_raw_bids(raw, bids_basename, output_path) diff --git a/mne_bids/tests/test_utils.py b/mne_bids/tests/test_utils.py index 3f3df03e0..ef8534883 100644 --- a/mne_bids/tests/test_utils.py +++ b/mne_bids/tests/test_utils.py @@ -37,6 +37,13 @@ def test_handle_kind(): raw = mne.io.RawArray(data, info) assert _handle_kind(raw) == kind + # if the situation is ambiguous (EEG and iEEG channels both), raise error + with pytest.raises(ValueError, match='Both EEG and iEEG channels found'): + info = mne.create_info(2, sampling_rate, + ch_types=['eeg', 'ecog']) + raw = mne.io.RawArray(random((2, sampling_rate)), info) + _handle_kind(raw) + # if we cannot find a proper channel type, we raise an error with pytest.raises(ValueError, match='Neither MEG/EEG/iEEG channels'): info = mne.create_info(n_channels, sampling_rate, ch_types=['misc']) diff --git a/mne_bids/utils.py b/mne_bids/utils.py index a95fe9715..685935744 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -88,10 +88,14 @@ def _handle_kind(raw): """Get kind.""" if 'meg' in raw: kind = 'meg' - elif 'ecog' in raw: + elif 'ecog' in raw and 'eeg' not in raw: kind = 'ieeg' - elif 'eeg' in raw: + elif 'eeg' in raw and 'ecog' not in raw: kind = 'eeg' + elif 'eeg' in raw and 'ecog' in raw: + raise ValueError('Both EEG and iEEG channels found in data.' + 'There is currently no specification on how' + 'to handle this data. Please proceed manually.') else: raise ValueError('Neither MEG/EEG/iEEG channels found in data.' 'Please use raw.set_channel_types to set the ' From e8accaf71a05ab36fb39f855114cc6ddf0c51393 Mon Sep 17 00:00:00 2001 From: Stefan Appelhoff Date: Sun, 11 Nov 2018 21:20:03 +0100 Subject: [PATCH 9/9] simplify if-else logic in handle_kind --- mne_bids/utils.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mne_bids/utils.py b/mne_bids/utils.py index 685935744..6c4132ea5 100644 --- a/mne_bids/utils.py +++ b/mne_bids/utils.py @@ -86,16 +86,16 @@ def _parse_bids_filename(fname, verbose): def _handle_kind(raw): """Get kind.""" - if 'meg' in raw: - kind = 'meg' - elif 'ecog' in raw and 'eeg' not in raw: - kind = 'ieeg' - elif 'eeg' in raw and 'ecog' not in raw: - kind = 'eeg' - elif 'eeg' in raw and 'ecog' in raw: + if 'eeg' in raw and 'ecog' in raw: raise ValueError('Both EEG and iEEG channels found in data.' 'There is currently no specification on how' 'to handle this data. Please proceed manually.') + elif 'meg' in raw: + kind = 'meg' + elif 'ecog' in raw: + kind = 'ieeg' + elif 'eeg' in raw: + kind = 'eeg' else: raise ValueError('Neither MEG/EEG/iEEG channels found in data.' 'Please use raw.set_channel_types to set the '