From ac995cbc68bcdbb270a7a7200c418dc9bb44b010 Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Mon, 3 May 2021 18:15:30 +0200 Subject: [PATCH 01/11] Start CEdRawIO based on sonpy to read smrx files. --- neo/rawio/__init__.py | 7 ++ neo/rawio/cedrawio.py | 167 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 174 insertions(+) create mode 100644 neo/rawio/cedrawio.py diff --git a/neo/rawio/__init__.py b/neo/rawio/__init__.py index 9a27d93f5..571a49bf2 100644 --- a/neo/rawio/__init__.py +++ b/neo/rawio/__init__.py @@ -17,6 +17,7 @@ * :attr:`AxonRawIO` * :attr:`BlackrockRawIO` * :attr:`BrainVisionRawIO` +* :attr:`CedRawIO` * :attr:`ElanRawIO` * :attr:`IntanRawIO` * :attr:`MEArecRawIO` @@ -58,6 +59,10 @@ .. autoattribute:: extensions +.. autoclass:: neo.rawio.CedRawIO + + .. autoattribute:: extensions + .. autoclass:: neo.rawio.ElanRawIO .. autoattribute:: extensions @@ -142,6 +147,7 @@ from neo.rawio.axonrawio import AxonRawIO from neo.rawio.blackrockrawio import BlackrockRawIO from neo.rawio.brainvisionrawio import BrainVisionRawIO +from neo.rawio.cedrawio import CedRawIO from neo.rawio.elanrawio import ElanRawIO from neo.rawio.examplerawio import ExampleRawIO from neo.rawio.intanrawio import IntanRawIO @@ -169,6 +175,7 @@ AxonRawIO, BlackrockRawIO, BrainVisionRawIO, + CedRawIO, ElanRawIO, IntanRawIO, MicromedRawIO, diff --git a/neo/rawio/cedrawio.py b/neo/rawio/cedrawio.py new file mode 100644 index 000000000..cb62e71f5 --- /dev/null +++ b/neo/rawio/cedrawio.py @@ -0,0 +1,167 @@ +""" +Class for reading data from CED (Cambridge Electronic Design) +http://ced.co.uk/ + +This read *.smr and *.smrx from spike2 and signal software. + +Note Spike2RawIO/Spike2IO is the old implementation in neo. +It still works without any dependency. + +This implementation depend on SONPY package +https://pypi.org/project/sonpy/ + +Please note that the "sonpy" package IS NOT open source. + + +Author : Samuel Garcia +""" + +from .baserawio import (BaseRawIO, _signal_channel_dtype, _signal_stream_dtype, + _spike_channel_dtype, _event_channel_dtype) + +import numpy as np +from copy import deepcopy + +try: + import sonpy + HAVE_SONPY = True +except ImportError: + HAVE_SONPY = False + + + +class CedRawIO(BaseRawIO): + """ + Class for reading data from CED (Cambridge Electronic Design) spike2. + + This read smr and smrx. + """ + extensions = ['smr', 'smrx'] + rawmode = 'one-file' + + def __init__(self, filename=''): + BaseRawIO.__init__(self) + self.filename = filename + + def _source_name(self): + return self.filename + + def _parse_header(self): + assert HAVE_SONPY, 'sonpy must be installed' + + self.smrx_file = sonpy.lib.SonFile(sName=str(self.filename), bReadOnly=True) + smrx = self.smrx_file + + channel_infos = [] + sig_channels = [] + for chan_ind in range(smrx.MaxChannels()): + print() + print('chan_ind', chan_ind) + chan_type = smrx.ChannelType(chan_ind) + print(chan_type) + if chan_type == sonpy.lib.DataType.Adc: + physical_chan = smrx.PhysicalChannel(chan_ind) + #~ print(chan_type) + sr = smrx.GetIdealRate(chan_ind) + #~ print(sr) + divide = smrx.ChannelDivide(chan_ind) + #~ print(devide) + + max_time = smrx.ChannelMaxTime(chan_ind) + print('max_time', max_time) + first_time = smrx.FirstTime(chan_ind, 0, max_time) + print('first_time', first_time) + print('divide', divide) + print((max_time - first_time)/divide) + + # max_times is include so +1 + size_size = (max_time - first_time) /divide +1 + print('size_size', size_size) + channel_infos.append((first_time, max_time, divide, size_size)) + + #~ after_time = smrx.FirstTime(chan_ind, first_time+1, max_time) + #~ print('after_time', after_time) + + + + gain = smrx.GetChannelScale(chan_ind) + #~ print(gain) + offset = smrx.GetChannelOffset(chan_ind) + #~ print(offset) + units = smrx.GetChannelUnits(chan_ind) + #~ print(units) + + ch_name = smrx.GetChannelTitle(chan_ind) + #~ print(title) + + + chan_id = str(chan_ind) + dtype = 'int16' + stream_id = '0' + sig_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id)) + + sig_channels = np.array(sig_channels, dtype=_signal_channel_dtype) + + channel_infos = np.array(channel_infos, + dtype=[('first_time', 'i8'), ('max_time', 'i8'), ('divide', 'i8'), ('size_size', 'i8')]) + + # group depend on the signal size + print(sig_channels) + print(channel_infos) + + + signal_streams = np.array([('Signals', '0')], dtype=_signal_stream_dtype) + + + # spike channels not handled + spike_channels = [] + spike_channels = np.array([], dtype=_spike_channel_dtype) + + # event channels not handled + event_channels = [] + event_channels = np.array(event_channels, dtype=_event_channel_dtype) + + # TODO + self._t_stop = 10. + + self.header = {} + self.header['nb_block'] = 1 + self.header['nb_segment'] = [1] + self.header['signal_streams'] = signal_streams + self.header['signal_channels'] = sig_channels + self.header['spike_channels'] = spike_channels + self.header['event_channels'] = event_channels + + self._generate_minimal_annotations() + + def _segment_t_start(self, block_index, seg_index): + return 0. + + def _segment_t_stop(self, block_index, seg_index): + return self._t_stop + + def _get_signal_size(self, block_index, seg_index, stream_index): + return self._num_frames + + def _get_signal_t_start(self, block_index, seg_index, stream_index): + return 0. + + def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, + stream_index, channel_indexes): + if i_start is None: + i_start = 0 + if i_stop is None: + i_stop = self._num_frames + + #~ self.smrx_file.ReadInts(chan=chan_ind, + #~ nMax=int(f.ChannelMaxTime(smrx_ch_ind) / f.ChannelDivide(smrx_ch_ind)), + #~ tFrom=int(start_frame * f.ChannelDivide(smrx_ch_ind)), + #~ tUpto=int(end_frame * f.ChannelDivide(smrx_ch_ind)) + #~ ) + + #~ if channel_indexes is None: + #~ channel_indexes = slice(self._num_channels) + + #~ raw_signals = self._recgen.recordings[i_start:i_stop, channel_indexes] + #~ return raw_signals + From df6c70fd98d3cc869f6be92705f484d9899b7194 Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Mon, 3 May 2021 18:35:04 +0200 Subject: [PATCH 02/11] wip --- neo/rawio/cedrawio.py | 84 +++++++++++++++++++++++++------------------ 1 file changed, 50 insertions(+), 34 deletions(-) diff --git a/neo/rawio/cedrawio.py b/neo/rawio/cedrawio.py index cb62e71f5..63b04df17 100644 --- a/neo/rawio/cedrawio.py +++ b/neo/rawio/cedrawio.py @@ -53,12 +53,12 @@ def _parse_header(self): smrx = self.smrx_file channel_infos = [] - sig_channels = [] + signal_channels = [] for chan_ind in range(smrx.MaxChannels()): - print() - print('chan_ind', chan_ind) + #~ print() + #~ print('chan_ind', chan_ind) chan_type = smrx.ChannelType(chan_ind) - print(chan_type) + #~ print(chan_type) if chan_type == sonpy.lib.DataType.Adc: physical_chan = smrx.PhysicalChannel(chan_ind) #~ print(chan_type) @@ -68,15 +68,15 @@ def _parse_header(self): #~ print(devide) max_time = smrx.ChannelMaxTime(chan_ind) - print('max_time', max_time) + #~ print('max_time', max_time) first_time = smrx.FirstTime(chan_ind, 0, max_time) - print('first_time', first_time) - print('divide', divide) - print((max_time - first_time)/divide) + #~ print('first_time', first_time) + #~ print('divide', divide) + #~ print((max_time - first_time)/divide) # max_times is include so +1 size_size = (max_time - first_time) /divide +1 - print('size_size', size_size) + #~ print('size_size', size_size) channel_infos.append((first_time, max_time, divide, size_size)) #~ after_time = smrx.FirstTime(chan_ind, first_time+1, max_time) @@ -98,20 +98,24 @@ def _parse_header(self): chan_id = str(chan_ind) dtype = 'int16' stream_id = '0' - sig_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id)) + signal_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id)) - sig_channels = np.array(sig_channels, dtype=_signal_channel_dtype) + signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype) + # cahnnel are group into stream if same start/stop/size/divide channel_infos = np.array(channel_infos, - dtype=[('first_time', 'i8'), ('max_time', 'i8'), ('divide', 'i8'), ('size_size', 'i8')]) - - # group depend on the signal size - print(sig_channels) - print(channel_infos) - - - signal_streams = np.array([('Signals', '0')], dtype=_signal_stream_dtype) - + dtype=[('first_time', 'i8'), ('max_time', 'i8'), ('divide', 'i8'), ('size', 'i8')]) + unique_info = np.unique(channel_infos) + self.stream_info = unique_info + signal_streams = [] + for i, info in enumerate(unique_info): + stream_id = str(i) + mask = channel_infos == info + signal_channels['stream_id'][mask] = stream_id + num_chans = np.sum(mask) + stream_name = f'{stream_id} {num_chans}chans' + signal_streams.append((stream_name, stream_id)) + signal_streams = np.array(signal_streams, dtype=_signal_stream_dtype) # spike channels not handled spike_channels = [] @@ -128,7 +132,7 @@ def _parse_header(self): self.header['nb_block'] = 1 self.header['nb_segment'] = [1] self.header['signal_streams'] = signal_streams - self.header['signal_channels'] = sig_channels + self.header['signal_channels'] = signal_channels self.header['spike_channels'] = spike_channels self.header['event_channels'] = event_channels @@ -141,7 +145,8 @@ def _segment_t_stop(self, block_index, seg_index): return self._t_stop def _get_signal_size(self, block_index, seg_index, stream_index): - return self._num_frames + size = self.stream_info[stream_index]['size'] + return size def _get_signal_t_start(self, block_index, seg_index, stream_index): return 0. @@ -151,17 +156,28 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, if i_start is None: i_start = 0 if i_stop is None: - i_stop = self._num_frames - - #~ self.smrx_file.ReadInts(chan=chan_ind, - #~ nMax=int(f.ChannelMaxTime(smrx_ch_ind) / f.ChannelDivide(smrx_ch_ind)), - #~ tFrom=int(start_frame * f.ChannelDivide(smrx_ch_ind)), - #~ tUpto=int(end_frame * f.ChannelDivide(smrx_ch_ind)) - #~ ) + i_stop = self.stream_info[stream_index]['size'] - #~ if channel_indexes is None: - #~ channel_indexes = slice(self._num_channels) - - #~ raw_signals = self._recgen.recordings[i_start:i_stop, channel_indexes] - #~ return raw_signals + stream_id = self.header['signal_streams']['id'][stream_index] + signal_channels = self.header['signal_channels'] + mask = signal_channels['stream_id'] == stream_id + signal_channels = signal_channels[mask] + if channel_indexes is not None: + signal_channels = signal_channels[channel_indexes] + + num_chans = len(signal_channels) + + info = self.stream_info[stream_index] + + size = i_stop - i_start + tFrom = info['first_time'] + info['divide'] * i_start + tUpto = info['first_time'] + info['divide'] * (i_stop ) + sigs = np.zeros((size, num_chans), dtype='int16') + for i, chan_id in enumerate(signal_channels['id']): + chan_ind = int(chan_id) + sig = self.smrx_file.ReadInts(chan=chan_ind, + nMax=size, tFrom=tFrom, tUpto=tUpto ) + sigs[:, i] = sig + + return sigs From 497a97dea175a2652fd21537be6cde561c3a87c5 Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Tue, 4 May 2021 15:46:40 +0200 Subject: [PATCH 03/11] some clean --- neo/rawio/cedrawio.py | 69 ++++++++++++++++++------------------------- 1 file changed, 28 insertions(+), 41 deletions(-) diff --git a/neo/rawio/cedrawio.py b/neo/rawio/cedrawio.py index 63b04df17..8cd8c2e9d 100644 --- a/neo/rawio/cedrawio.py +++ b/neo/rawio/cedrawio.py @@ -2,15 +2,19 @@ Class for reading data from CED (Cambridge Electronic Design) http://ced.co.uk/ -This read *.smr and *.smrx from spike2 and signal software. +This read *.smrx (and *.smr) from spike2 and signal software. Note Spike2RawIO/Spike2IO is the old implementation in neo. -It still works without any dependency. +It still works without any dependency and should be faster. +Spike2IO only work for smr (32bit) and not for smrx (64bit). This implementation depend on SONPY package https://pypi.org/project/sonpy/ -Please note that the "sonpy" package IS NOT open source. +Please note that the SONPY package: + * IS NOT open source. + * use internally list (and not numpy.ndarray) which can make the data read very slow + * is maintain by CED Author : Samuel Garcia @@ -55,48 +59,23 @@ def _parse_header(self): channel_infos = [] signal_channels = [] for chan_ind in range(smrx.MaxChannels()): - #~ print() - #~ print('chan_ind', chan_ind) chan_type = smrx.ChannelType(chan_ind) - #~ print(chan_type) if chan_type == sonpy.lib.DataType.Adc: physical_chan = smrx.PhysicalChannel(chan_ind) - #~ print(chan_type) sr = smrx.GetIdealRate(chan_ind) - #~ print(sr) divide = smrx.ChannelDivide(chan_ind) - #~ print(devide) - max_time = smrx.ChannelMaxTime(chan_ind) - #~ print('max_time', max_time) first_time = smrx.FirstTime(chan_ind, 0, max_time) - #~ print('first_time', first_time) - #~ print('divide', divide) - #~ print((max_time - first_time)/divide) - # max_times is include so +1 size_size = (max_time - first_time) /divide +1 - #~ print('size_size', size_size) - channel_infos.append((first_time, max_time, divide, size_size)) - - #~ after_time = smrx.FirstTime(chan_ind, first_time+1, max_time) - #~ print('after_time', after_time) - - - + channel_infos.append((first_time, max_time, divide, size_size, sr)) gain = smrx.GetChannelScale(chan_ind) - #~ print(gain) offset = smrx.GetChannelOffset(chan_ind) - #~ print(offset) units = smrx.GetChannelUnits(chan_ind) - #~ print(units) - ch_name = smrx.GetChannelTitle(chan_ind) - #~ print(title) - - chan_id = str(chan_ind) dtype = 'int16' + # set later after groping stream_id = '0' signal_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id)) @@ -104,7 +83,7 @@ def _parse_header(self): # cahnnel are group into stream if same start/stop/size/divide channel_infos = np.array(channel_infos, - dtype=[('first_time', 'i8'), ('max_time', 'i8'), ('divide', 'i8'), ('size', 'i8')]) + dtype=[('first_time', 'i8'), ('max_time', 'i8'), ('divide', 'i8'), ('size', 'i8'), ('sampling_rate', 'f8')]) unique_info = np.unique(channel_infos) self.stream_info = unique_info signal_streams = [] @@ -125,8 +104,12 @@ def _parse_header(self): event_channels = [] event_channels = np.array(event_channels, dtype=_event_channel_dtype) - # TODO - self._t_stop = 10. + + self._seg_t_start = np.inf + self._seg_t_stop = -np.inf + for info in self.stream_info: + self._seg_t_start = min(self._seg_t_start, info['first_time'] / info['sampling_rate']) + self._seg_t_stop = max(self._seg_t_stop, info['max_time'] / info['sampling_rate']) self.header = {} self.header['nb_block'] = 1 @@ -139,40 +122,44 @@ def _parse_header(self): self._generate_minimal_annotations() def _segment_t_start(self, block_index, seg_index): - return 0. + return self._seg_t_start def _segment_t_stop(self, block_index, seg_index): - return self._t_stop + return self._seg_t_stop def _get_signal_size(self, block_index, seg_index, stream_index): size = self.stream_info[stream_index]['size'] return size def _get_signal_t_start(self, block_index, seg_index, stream_index): - return 0. + info = self.stream_info[stream_index] + t_start = info['first_time'] / info['sampling_rate'] + return t_start def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, stream_index, channel_indexes): + if i_start is None: i_start = 0 if i_stop is None: i_stop = self.stream_info[stream_index]['size'] - + stream_id = self.header['signal_streams']['id'][stream_index] signal_channels = self.header['signal_channels'] mask = signal_channels['stream_id'] == stream_id signal_channels = signal_channels[mask] if channel_indexes is not None: signal_channels = signal_channels[channel_indexes] - + num_chans = len(signal_channels) - info = self.stream_info[stream_index] - size = i_stop - i_start + sigs = np.zeros((size, num_chans), dtype='int16') + + info = self.stream_info[stream_index] tFrom = info['first_time'] + info['divide'] * i_start tUpto = info['first_time'] + info['divide'] * (i_stop ) - sigs = np.zeros((size, num_chans), dtype='int16') + for i, chan_id in enumerate(signal_channels['id']): chan_ind = int(chan_id) sig = self.smrx_file.ReadInts(chan=chan_ind, From 5321483836a72e853a8f4812e796cb635cbdef77 Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Wed, 5 May 2021 17:25:32 +0200 Subject: [PATCH 04/11] wip ced (smrx) with sonpy --- neo/io/__init__.py | 7 +++++ neo/io/cedio.py | 10 ++++++++ neo/rawio/cedrawio.py | 40 ++++++++++++++--------------- neo/test/iotest/test_cedio.py | 18 +++++++++++++ neo/test/rawiotest/test_cedrawio.py | 19 ++++++++++++++ 5 files changed, 74 insertions(+), 20 deletions(-) create mode 100644 neo/io/cedio.py create mode 100644 neo/test/iotest/test_cedio.py create mode 100644 neo/test/rawiotest/test_cedrawio.py diff --git a/neo/io/__init__.py b/neo/io/__init__.py index ed8796b21..0b5f4e8e2 100644 --- a/neo/io/__init__.py +++ b/neo/io/__init__.py @@ -28,6 +28,7 @@ * :attr:`BrainwareDamIO` * :attr:`BrainwareF32IO` * :attr:`BrainwareSrcIO` +* :attr:`CedIO` * :attr:`ElanIO` * :attr:`IgorIO` * :attr:`IntanIO` @@ -114,6 +115,10 @@ .. autoattribute:: extensions +.. autoclass:: neo.io.CedIO + + .. autoattribute:: extensions + .. autoclass:: neo.io.ElanIO .. autoattribute:: extensions @@ -264,6 +269,7 @@ from neo.io.brainwaredamio import BrainwareDamIO from neo.io.brainwaref32io import BrainwareF32IO from neo.io.brainwaresrcio import BrainwareSrcIO +from neo.io.cedio import CedIO from neo.io.elanio import ElanIO # from neo.io.elphyio import ElphyIO from neo.io.exampleio import ExampleIO @@ -310,6 +316,7 @@ BrainwareDamIO, BrainwareF32IO, BrainwareSrcIO, + CedIO, ElanIO, # ElphyIO, ExampleIO, diff --git a/neo/io/cedio.py b/neo/io/cedio.py new file mode 100644 index 000000000..602e49c41 --- /dev/null +++ b/neo/io/cedio.py @@ -0,0 +1,10 @@ +from neo.io.basefromrawio import BaseFromRaw +from neo.rawio.cedrawio import CedRawIO + + +class CedIO(CedRawIO, BaseFromRaw): + __doc__ = CedRawIO.__doc__ + + def __init__(self, filename, entfile=None, posfile=None): + CedRawIO.__init__(self, filename=filename) + BaseFromRaw.__init__(self, filename) diff --git a/neo/rawio/cedrawio.py b/neo/rawio/cedrawio.py index 8cd8c2e9d..434a5a870 100644 --- a/neo/rawio/cedrawio.py +++ b/neo/rawio/cedrawio.py @@ -33,11 +33,11 @@ HAVE_SONPY = False - class CedRawIO(BaseRawIO): """ Class for reading data from CED (Cambridge Electronic Design) spike2. - + This use internally the sonpy package which is close source. + This read smr and smrx. """ extensions = ['smr', 'smrx'] @@ -55,7 +55,7 @@ def _parse_header(self): self.smrx_file = sonpy.lib.SonFile(sName=str(self.filename), bReadOnly=True) smrx = self.smrx_file - + channel_infos = [] signal_channels = [] for chan_ind in range(smrx.MaxChannels()): @@ -67,7 +67,7 @@ def _parse_header(self): max_time = smrx.ChannelMaxTime(chan_ind) first_time = smrx.FirstTime(chan_ind, 0, max_time) # max_times is include so +1 - size_size = (max_time - first_time) /divide +1 + size_size = (max_time - first_time) / divide + 1 channel_infos.append((first_time, max_time, divide, size_size, sr)) gain = smrx.GetChannelScale(chan_ind) offset = smrx.GetChannelOffset(chan_ind) @@ -77,13 +77,15 @@ def _parse_header(self): dtype = 'int16' # set later after groping stream_id = '0' - signal_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id)) - + signal_channels.append((ch_name, chan_id, sr, dtype, + units, gain, offset, stream_id)) + signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype) - + # cahnnel are group into stream if same start/stop/size/divide channel_infos = np.array(channel_infos, - dtype=[('first_time', 'i8'), ('max_time', 'i8'), ('divide', 'i8'), ('size', 'i8'), ('sampling_rate', 'f8')]) + dtype=[('first_time', 'i8'), ('max_time', 'i8'), + ('divide', 'i8'), ('size', 'i8'), ('sampling_rate', 'f8')]) unique_info = np.unique(channel_infos) self.stream_info = unique_info signal_streams = [] @@ -99,18 +101,17 @@ def _parse_header(self): # spike channels not handled spike_channels = [] spike_channels = np.array([], dtype=_spike_channel_dtype) - + # event channels not handled event_channels = [] event_channels = np.array(event_channels, dtype=_event_channel_dtype) - - + self._seg_t_start = np.inf self._seg_t_stop = -np.inf - for info in self.stream_info: + for info in self.stream_info: self._seg_t_start = min(self._seg_t_start, info['first_time'] / info['sampling_rate']) self._seg_t_stop = max(self._seg_t_stop, info['max_time'] / info['sampling_rate']) - + self.header = {} self.header['nb_block'] = 1 self.header['nb_segment'] = [1] @@ -152,19 +153,18 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, signal_channels = signal_channels[channel_indexes] num_chans = len(signal_channels) - + size = i_stop - i_start sigs = np.zeros((size, num_chans), dtype='int16') - + info = self.stream_info[stream_index] tFrom = info['first_time'] + info['divide'] * i_start - tUpto = info['first_time'] + info['divide'] * (i_stop ) + tUpto = info['first_time'] + info['divide'] * i_stop for i, chan_id in enumerate(signal_channels['id']): chan_ind = int(chan_id) - sig = self.smrx_file.ReadInts(chan=chan_ind, - nMax=size, tFrom=tFrom, tUpto=tUpto ) + sig = self.smrx_file.ReadInts(chan=chan_ind, + nMax=size, tFrom=tFrom, tUpto=tUpto) sigs[:, i] = sig - - return sigs + return sigs diff --git a/neo/test/iotest/test_cedio.py b/neo/test/iotest/test_cedio.py new file mode 100644 index 000000000..34584b4f6 --- /dev/null +++ b/neo/test/iotest/test_cedio.py @@ -0,0 +1,18 @@ +import unittest + +from neo.io import CedIO +from neo.test.iotest.common_io_test import BaseTestIO + + +class TestCedIO(BaseTestIO, unittest.TestCase, ): + ioclass = CedIO + entities_to_test = [ + 'spike2/m365_1sec.smrx' + ] + entities_to_download = [ + 'spike2/m365_1sec.smrx' + ] + + +if __name__ == "__main__": + unittest.main() diff --git a/neo/test/rawiotest/test_cedrawio.py b/neo/test/rawiotest/test_cedrawio.py new file mode 100644 index 000000000..d5d05d706 --- /dev/null +++ b/neo/test/rawiotest/test_cedrawio.py @@ -0,0 +1,19 @@ +import unittest + +from neo.rawio.cedrawio import CedRawIO + +from neo.test.rawiotest.common_rawio_test import BaseTestRawIO + + +class TestCedRawIO(BaseTestRawIO, unittest.TestCase, ): + rawioclass = CedRawIO + entities_to_test = [ + 'spike2/m365_1sec.smrx' + ] + entities_to_download = [ + 'spike2/m365_1sec.smrx' + ] + + +if __name__ == "__main__": + unittest.main() From 95ed3e78d701e28415adf1774fe3c672c9acad4a Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Wed, 12 May 2021 08:59:26 +0200 Subject: [PATCH 05/11] Fix for t_start/_t_stop --- neo/rawio/cedrawio.py | 6 +++--- requirements_testing.txt | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/neo/rawio/cedrawio.py b/neo/rawio/cedrawio.py index 434a5a870..b3d214e90 100644 --- a/neo/rawio/cedrawio.py +++ b/neo/rawio/cedrawio.py @@ -109,8 +109,8 @@ def _parse_header(self): self._seg_t_start = np.inf self._seg_t_stop = -np.inf for info in self.stream_info: - self._seg_t_start = min(self._seg_t_start, info['first_time'] / info['sampling_rate']) - self._seg_t_stop = max(self._seg_t_stop, info['max_time'] / info['sampling_rate']) + self._seg_t_start = min(self._seg_t_start, info['first_time'] / info['sampling_rate'] / info['divide']) + self._seg_t_stop = max(self._seg_t_stop, info['max_time'] / info['sampling_rate'] / info['divide']) self.header = {} self.header['nb_block'] = 1 @@ -134,7 +134,7 @@ def _get_signal_size(self, block_index, seg_index, stream_index): def _get_signal_t_start(self, block_index, seg_index, stream_index): info = self.stream_info[stream_index] - t_start = info['first_time'] / info['sampling_rate'] + t_start = info['first_time'] / info['sampling_rate'] / info['divide'] return t_start def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, diff --git a/requirements_testing.txt b/requirements_testing.txt index 8d6b6a112..24536ae8b 100644 --- a/requirements_testing.txt +++ b/requirements_testing.txt @@ -12,3 +12,4 @@ ipython coverage coveralls pillow +sonpy From 30d079ad4df9453d34560420ed7bd5e0239c78d6 Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Mon, 14 Jun 2021 21:27:39 +0200 Subject: [PATCH 06/11] Similarly to spike2io implemente the take_ideal_sampling_rate option. --- neo/rawio/cedrawio.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/neo/rawio/cedrawio.py b/neo/rawio/cedrawio.py index 434a5a870..b64d62eeb 100644 --- a/neo/rawio/cedrawio.py +++ b/neo/rawio/cedrawio.py @@ -43,9 +43,11 @@ class CedRawIO(BaseRawIO): extensions = ['smr', 'smrx'] rawmode = 'one-file' - def __init__(self, filename=''): + def __init__(self, filename='', take_ideal_sampling_rate=False, ): BaseRawIO.__init__(self) self.filename = filename + + self.take_ideal_sampling_rate = take_ideal_sampling_rate def _source_name(self): return self.filename @@ -62,8 +64,11 @@ def _parse_header(self): chan_type = smrx.ChannelType(chan_ind) if chan_type == sonpy.lib.DataType.Adc: physical_chan = smrx.PhysicalChannel(chan_ind) - sr = smrx.GetIdealRate(chan_ind) divide = smrx.ChannelDivide(chan_ind) + if self.take_ideal_sampling_rate: + sr = smrx.GetIdealRate(chan_ind) + else: + sr = 1. / (smrx.GetTimeBase() * divide) max_time = smrx.ChannelMaxTime(chan_ind) first_time = smrx.FirstTime(chan_ind, 0, max_time) # max_times is include so +1 From 1b7aa381bff1022c5b1a573f64ab72ec13e8928f Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Mon, 14 Jun 2021 21:30:26 +0200 Subject: [PATCH 07/11] pep8 --- neo/rawio/cedrawio.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/neo/rawio/cedrawio.py b/neo/rawio/cedrawio.py index ac6f47dcf..de468ac25 100644 --- a/neo/rawio/cedrawio.py +++ b/neo/rawio/cedrawio.py @@ -46,7 +46,7 @@ class CedRawIO(BaseRawIO): def __init__(self, filename='', take_ideal_sampling_rate=False, ): BaseRawIO.__init__(self) self.filename = filename - + self.take_ideal_sampling_rate = take_ideal_sampling_rate def _source_name(self): @@ -114,8 +114,10 @@ def _parse_header(self): self._seg_t_start = np.inf self._seg_t_stop = -np.inf for info in self.stream_info: - self._seg_t_start = min(self._seg_t_start, info['first_time'] / info['sampling_rate'] / info['divide']) - self._seg_t_stop = max(self._seg_t_stop, info['max_time'] / info['sampling_rate'] / info['divide']) + self._seg_t_start = min(self._seg_t_start, + info['first_time'] / info['sampling_rate']) + self._seg_t_stop = max(self._seg_t_stop, + info['max_time'] / info['sampling_rate']) self.header = {} self.header['nb_block'] = 1 @@ -139,7 +141,7 @@ def _get_signal_size(self, block_index, seg_index, stream_index): def _get_signal_t_start(self, block_index, seg_index, stream_index): info = self.stream_info[stream_index] - t_start = info['first_time'] / info['sampling_rate'] / info['divide'] + t_start = info['first_time'] / info['sampling_rate'] return t_start def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, From e7dd88f5bdb0bf7599d5feac90f6d68c9ee1029d Mon Sep 17 00:00:00 2001 From: Garcia Samuel Date: Tue, 15 Jun 2021 10:21:21 +0200 Subject: [PATCH 08/11] Julia suggestions Co-authored-by: Julia Sprenger --- neo/rawio/cedrawio.py | 30 ++++++++++++++--------------- neo/test/iotest/test_cedio.py | 2 +- neo/test/rawiotest/test_cedrawio.py | 2 +- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/neo/rawio/cedrawio.py b/neo/rawio/cedrawio.py index de468ac25..b212ec607 100644 --- a/neo/rawio/cedrawio.py +++ b/neo/rawio/cedrawio.py @@ -6,15 +6,15 @@ Note Spike2RawIO/Spike2IO is the old implementation in neo. It still works without any dependency and should be faster. -Spike2IO only work for smr (32bit) and not for smrx (64bit). +Spike2IO only works for smr (32 bit) and not for smrx (64 bit) files. -This implementation depend on SONPY package +This implementation depends on the SONPY package: https://pypi.org/project/sonpy/ Please note that the SONPY package: - * IS NOT open source. - * use internally list (and not numpy.ndarray) which can make the data read very slow - * is maintain by CED + * is NOT open source + * internally uses a list instead of numpy.ndarray, potentially causing slow data reading + * is maintained by CED Author : Samuel Garcia @@ -36,9 +36,9 @@ class CedRawIO(BaseRawIO): """ Class for reading data from CED (Cambridge Electronic Design) spike2. - This use internally the sonpy package which is close source. + This internally uses the sonpy package which is closed source. - This read smr and smrx. + This IO reads smr and smrx files """ extensions = ['smr', 'smrx'] rawmode = 'one-file' @@ -71,23 +71,23 @@ def _parse_header(self): sr = 1. / (smrx.GetTimeBase() * divide) max_time = smrx.ChannelMaxTime(chan_ind) first_time = smrx.FirstTime(chan_ind, 0, max_time) - # max_times is include so +1 - size_size = (max_time - first_time) / divide + 1 - channel_infos.append((first_time, max_time, divide, size_size, sr)) + # max_times is included so +1 + time_size = (max_time - first_time) / divide + 1 + channel_infos.append((first_time, max_time, divide, time_size, sr)) gain = smrx.GetChannelScale(chan_ind) offset = smrx.GetChannelOffset(chan_ind) units = smrx.GetChannelUnits(chan_ind) ch_name = smrx.GetChannelTitle(chan_ind) chan_id = str(chan_ind) dtype = 'int16' - # set later after groping + # set later after grouping stream_id = '0' signal_channels.append((ch_name, chan_id, sr, dtype, units, gain, offset, stream_id)) signal_channels = np.array(signal_channels, dtype=_signal_channel_dtype) - # cahnnel are group into stream if same start/stop/size/divide + # channels are grouped into stream if they have a common start, stop, size, divide and sampling_rate channel_infos = np.array(channel_infos, dtype=[('first_time', 'i8'), ('max_time', 'i8'), ('divide', 'i8'), ('size', 'i8'), ('sampling_rate', 'f8')]) @@ -165,13 +165,13 @@ def _get_analogsignal_chunk(self, block_index, seg_index, i_start, i_stop, sigs = np.zeros((size, num_chans), dtype='int16') info = self.stream_info[stream_index] - tFrom = info['first_time'] + info['divide'] * i_start - tUpto = info['first_time'] + info['divide'] * i_stop + t_from = info['first_time'] + info['divide'] * i_start + t_upto = info['first_time'] + info['divide'] * i_stop for i, chan_id in enumerate(signal_channels['id']): chan_ind = int(chan_id) sig = self.smrx_file.ReadInts(chan=chan_ind, - nMax=size, tFrom=tFrom, tUpto=tUpto) + nMax=size, tFrom=t_from, tUpto=t_upto) sigs[:, i] = sig return sigs diff --git a/neo/test/iotest/test_cedio.py b/neo/test/iotest/test_cedio.py index 34584b4f6..41da943ef 100644 --- a/neo/test/iotest/test_cedio.py +++ b/neo/test/iotest/test_cedio.py @@ -10,7 +10,7 @@ class TestCedIO(BaseTestIO, unittest.TestCase, ): 'spike2/m365_1sec.smrx' ] entities_to_download = [ - 'spike2/m365_1sec.smrx' + 'spike2' ] diff --git a/neo/test/rawiotest/test_cedrawio.py b/neo/test/rawiotest/test_cedrawio.py index d5d05d706..688bb55ad 100644 --- a/neo/test/rawiotest/test_cedrawio.py +++ b/neo/test/rawiotest/test_cedrawio.py @@ -11,7 +11,7 @@ class TestCedRawIO(BaseTestRawIO, unittest.TestCase, ): 'spike2/m365_1sec.smrx' ] entities_to_download = [ - 'spike2/m365_1sec.smrx' + 'spike2' ] From 260154634448809a0fb7a496df1b223f1cdf3162 Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Tue, 15 Jun 2021 10:48:18 +0200 Subject: [PATCH 09/11] Add also smr file in tests --- neo/test/iotest/test_cedio.py | 4 +++- neo/test/rawiotest/test_cedrawio.py | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/neo/test/iotest/test_cedio.py b/neo/test/iotest/test_cedio.py index 41da943ef..3b5795207 100644 --- a/neo/test/iotest/test_cedio.py +++ b/neo/test/iotest/test_cedio.py @@ -7,7 +7,9 @@ class TestCedIO(BaseTestIO, unittest.TestCase, ): ioclass = CedIO entities_to_test = [ - 'spike2/m365_1sec.smrx' + 'spike2/m365_1sec.smrx', + 'spike2/File_spike2_1.smr', + 'spike2/Two-mice-bigfile-test000.smr' ] entities_to_download = [ 'spike2' diff --git a/neo/test/rawiotest/test_cedrawio.py b/neo/test/rawiotest/test_cedrawio.py index 688bb55ad..50a0e4beb 100644 --- a/neo/test/rawiotest/test_cedrawio.py +++ b/neo/test/rawiotest/test_cedrawio.py @@ -8,7 +8,9 @@ class TestCedRawIO(BaseTestRawIO, unittest.TestCase, ): rawioclass = CedRawIO entities_to_test = [ - 'spike2/m365_1sec.smrx' + 'spike2/m365_1sec.smrx', + 'spike2/File_spike2_1.smr', + 'spike2/Two-mice-bigfile-test000.smr' ] entities_to_download = [ 'spike2' From 0e0f88501dafafc531ca76d72e0ce5c3fb5b6871 Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Wed, 16 Jun 2021 16:09:16 +0200 Subject: [PATCH 10/11] Fix the gain. Debug by Eduarda. --- neo/rawio/cedrawio.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/rawio/cedrawio.py b/neo/rawio/cedrawio.py index b212ec607..6b74e45b3 100644 --- a/neo/rawio/cedrawio.py +++ b/neo/rawio/cedrawio.py @@ -74,7 +74,7 @@ def _parse_header(self): # max_times is included so +1 time_size = (max_time - first_time) / divide + 1 channel_infos.append((first_time, max_time, divide, time_size, sr)) - gain = smrx.GetChannelScale(chan_ind) + gain = smrx.GetChannelScale(chan_ind) / 6553.6 offset = smrx.GetChannelOffset(chan_ind) units = smrx.GetChannelUnits(chan_ind) ch_name = smrx.GetChannelTitle(chan_ind) From fe63e4e46fb5a93efb20f73ef8efd7f4a6db79dd Mon Sep 17 00:00:00 2001 From: Samuel Garcia Date: Fri, 18 Jun 2021 10:54:11 +0200 Subject: [PATCH 11/11] mention CedRawIO in Spike2Rawio. --- neo/rawio/spike2rawio.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/neo/rawio/spike2rawio.py b/neo/rawio/spike2rawio.py index 220553335..374f7f3a5 100644 --- a/neo/rawio/spike2rawio.py +++ b/neo/rawio/spike2rawio.py @@ -27,7 +27,8 @@ class Spike2RawIO(BaseRawIO): """ - + This implementation in neo read only old smr files. + For smrx files you need to use CedRawIO which is based on sonpy. """ extensions = ['smr'] rawmode = 'one-file'