diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e6709d9..91a57a7f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Contributions - BIDS - - Bundling support and refactor of `convert` module. See `demo_convert.py` #362 + - Bundling support and refactor of `convert` module. See `demo_convert.py` #362 Add support for 1020 channels and eye tracker data #369 - Library Refactor - Refactor `helpers` into `io` and `core` #362 - Dependencies diff --git a/README.md b/README.md index abd962ae..3a4de644 100644 --- a/README.md +++ b/README.md @@ -190,7 +190,7 @@ This a list of the major modules and their functionality. Each module will conta - `feedback`: feedback mechanisms for sound and visual stimuli. - `gui`: end-user interface into registered bci tasks and parameter editing. See BCInterface.py. - `helpers`: helpful functions needed for interactions between modules and general utility. -- `io`: load, save, and convert data files. Ex. BrainVision, EDF, MNE, CSV, JSON, etc. +- `io`: load, save, and convert data files. Ex. BIDS, BrainVision, EDF, MNE, CSV, JSON, etc. - `language`: gives probabilities of next symbols during typing. - `main`: executor of experiments. Main entry point into the application - `parameters`: location of json parameters. This includes parameters.json (main experiment / app configuration) and device.json (device registry and configuration). diff --git a/bcipy/core/demo/demo_report.py b/bcipy/core/demo/demo_report.py index 862e08e2..3a2771c7 100644 --- a/bcipy/core/demo/demo_report.py +++ b/bcipy/core/demo/demo_report.py @@ -31,7 +31,10 @@ # if no path is provided, prompt for one using a GUI path = args.path if not path: - path = load_experimental_data() + path = load_experimental_data( + message="Select the folder containing the raw_data.csv, parameters.json and triggers.txt", + strict=True + ) trial_window = (0, 1.0) diff --git a/bcipy/core/demo/demo_session_tools.py b/bcipy/core/demo/demo_session_tools.py index a6721a7e..b3c06dd5 100644 --- a/bcipy/core/demo/demo_session_tools.py +++ b/bcipy/core/demo/demo_session_tools.py @@ -39,7 +39,7 @@ def main(data_dir: str): args = parser.parse_args() path = args.path if not path: - path = ask_directory() + path = ask_directory(strict=True) if args.db or args.csv or args.charts: session = read_session(Path(path, SESSION_DATA_FILENAME)) diff --git a/bcipy/core/raw_data.py b/bcipy/core/raw_data.py index d94ac217..297ac1e1 100644 --- a/bcipy/core/raw_data.py +++ b/bcipy/core/raw_data.py @@ -402,3 +402,34 @@ def sample_data(rows: int = 1000, data.append([timestamp] + channel_data + [trg]) return data + + +def get_1020_channels() -> List[str]: + """Returns the standard 10-20 channel names. + + Note: The 10-20 system is a standard for EEG electrode placement. The following is not a complete list of all + possible channels, but the most common ones used in BCI research. This excludes the reference and ground channels. + + Returns + ------- + list of channel names + """ + return [ + 'Fp1', 'Fp2', 'F7', 'F3', 'Fz', 'F4', 'F8', 'T3', 'C3', 'Cz', 'C4', + 'T4', 'T5', 'P3', 'Pz', 'P4', 'T6', 'O1', 'O2' + ] + + +def get_1020_channel_map(channels_name: List[str]) -> List[int]: + """Returns a list of 1s and 0s indicating if the channel name is in the 10-20 system. + + Parameters + ---------- + channels_name : list of channel names + + Returns + ------- + list of 1s and 0s indicating if the channel name is in the 10-20 system + """ + valid_channels = get_1020_channels() + return [1 if name in valid_channels else 0 for name in channels_name] diff --git a/bcipy/core/tests/test_raw_data.py b/bcipy/core/tests/test_raw_data.py index 71093157..2ba537c7 100644 --- a/bcipy/core/tests/test_raw_data.py +++ b/bcipy/core/tests/test_raw_data.py @@ -12,7 +12,8 @@ from bcipy.exceptions import BciPyCoreException from bcipy.core.raw_data import (RawData, RawDataReader, RawDataWriter, - load, sample_data, settings, write) + load, sample_data, settings, write, + get_1020_channel_map, get_1020_channels) class TestRawData(unittest.TestCase): @@ -377,5 +378,25 @@ def test_data_by_channel_map_applies_transformation(self): verify(RawData, times=1).by_channel(transform) +class Test1020(unittest.TestCase): + """Tests for 10-20 channel mapping functions.""" + + def test_get_1020_channels(self): + """Tests that the 10-20 channel map is correctly generated.""" + channels = get_1020_channels() + self.assertEqual(19, len(channels)) + self.assertTrue(isinstance(channels[0], str)) + + def test_get_1020_channel_map(self): + """Tests that the 10-20 channel map is correctly generated.""" + # all but the last channel are valid 10-20 channels + channels = ['Fp1', 'Fp2', 'F3', 'F4', 'C3', 'C4', 'P3', 'P4', 'O1', 'invalid'] + channel_map = get_1020_channel_map(channels) + self.assertEqual(10, len(channel_map)) + self.assertEqual(0, channel_map[-1]) + for i in range(len(channels) - 1): + self.assertEqual(1, channel_map[i]) + + if __name__ == '__main__': unittest.main() diff --git a/bcipy/gui/file_dialog.py b/bcipy/gui/file_dialog.py index 715b223b..f8271fc6 100644 --- a/bcipy/gui/file_dialog.py +++ b/bcipy/gui/file_dialog.py @@ -1,11 +1,13 @@ # pylint: disable=no-name-in-module,missing-docstring,too-few-public-methods import sys from pathlib import Path +from typing import Union from PyQt6 import QtGui from PyQt6.QtWidgets import QApplication, QFileDialog, QWidget from bcipy.preferences import preferences +from bcipy.exceptions import BciPyCoreException DEFAULT_FILE_TYPES = "All Files (*)" @@ -61,18 +63,22 @@ def ask_directory(self, directory: str = "", prompt: str = "Select Directory") - def ask_filename( file_types: str = DEFAULT_FILE_TYPES, directory: str = "", - prompt: str = "Select File") -> str: - """Prompt for a file. + prompt: str = "Select File", + strict: bool = False) -> Union[str, BciPyCoreException]: + """Prompt for a file using a GUI. Parameters ---------- - file_types : optional file type filters; Examples: 'Text files (*.txt)' or 'Image files (*.jpg *.gif)' or '*.csv;;*.pkl' - directory : optional directory + - prompt : optional prompt message to display to users + - strict : optional flag to raise an exception if the user cancels the dialog. Default is False. + If False, an empty string is returned. Returns ------- - path to file or None if the user cancelled the dialog. + path to file or raises an exception if the user cancels the dialog. """ app = QApplication(sys.argv) dialog = FileDialog() @@ -84,18 +90,29 @@ def ask_filename( if filename and path.is_file(): preferences.last_directory = str(path.parent) - # Alternatively, we could use `app.closeAllWindows()` - app.quit() + # Alternatively, we could use `app.closeAllWindows()` + app.quit() + + return filename + + if strict: + raise BciPyCoreException('No file selected.') + + return '' - return filename +def ask_directory(prompt: str = "Select Directory", strict: bool = False) -> Union[str, BciPyCoreException]: + """Prompt for a directory using a GUI. -def ask_directory(prompt: str = "Select Directory") -> str: - """Prompt for a directory. + Parameters + ---------- + prompt : optional prompt message to display to users + strict : optional flag to raise an exception if the user cancels the dialog. Default is False. + If False, an empty string is returned. Returns ------- - path to directory or None if the user cancelled the dialog. + path to directory or raises an exception if the user cancels the dialog. """ app = QApplication(sys.argv) @@ -107,7 +124,12 @@ def ask_directory(prompt: str = "Select Directory") -> str: if name and Path(name).is_dir(): preferences.last_directory = name - # Alternatively, we could use `app.closeAllWindows()` - app.quit() + # Alternatively, we could use `app.closeAllWindows()` + app.quit() + + return name + + if strict: + raise BciPyCoreException('No directory selected.') - return name + return '' diff --git a/bcipy/helpers/demo/demo_visualization.py b/bcipy/helpers/demo/demo_visualization.py index 2cfc3a04..192a0b01 100644 --- a/bcipy/helpers/demo/demo_visualization.py +++ b/bcipy/helpers/demo/demo_visualization.py @@ -42,7 +42,10 @@ path = args.path if not path: - path = load_experimental_data() + path = load_experimental_data( + message="Select the folder containing the raw_data.csv, parameters.json and triggers.txt", + strict=True + ) parameters = load_json_parameters(f'{path}/{DEFAULT_PARAMETERS_FILENAME}', value_cast=True) diff --git a/bcipy/helpers/offset.py b/bcipy/helpers/offset.py index 09b8814c..5e543e56 100644 --- a/bcipy/helpers/offset.py +++ b/bcipy/helpers/offset.py @@ -340,7 +340,7 @@ def extract_data_latency_calculation( args = parser.parse_args() data_path = args.data_path if not data_path: - data_path = ask_directory() + data_path = ask_directory(prompt="Please select a BciPy time test directory..", strict=True) # grab the stim length from the data directory parameters stim_length = load_json_parameters(f'{data_path}/{DEFAULT_PARAMETERS_FILENAME}', value_cast=True)['stim_length'] diff --git a/bcipy/io/README.md b/bcipy/io/README.md index 38e1541d..8b04bd53 100644 --- a/bcipy/io/README.md +++ b/bcipy/io/README.md @@ -2,6 +2,6 @@ The BciPy IO module contains functionality for loading and saving data in various formats. This includes the ability to convert data to BIDS format, load and save raw data, and load and save triggers. -- `convert`: functionality for converting the bcipy raw data output to other formats (currently, EDF) -- `load`: methods for loading most BciPy data. For loading of triggers, see triggers.py -- `save`: methods for saving BciPy data in supported formats. For saving of triggers, see triggers.py +- `convert`: functionality for converting the bcipy raw data output to other formats (currently, BrainVision and EDF), and for converting the raw data to BIDS format. +- `load`: methods for loading most BciPy data formats, including raw data and triggers. +- `save`: methods for saving BciPy data in supported formats. diff --git a/bcipy/io/convert.py b/bcipy/io/convert.py index 33adbe2a..03c03e0e 100644 --- a/bcipy/io/convert.py +++ b/bcipy/io/convert.py @@ -4,6 +4,7 @@ import os import tarfile from typing import List, Optional, Tuple +import glob import mne from enum import Enum @@ -12,13 +13,12 @@ from tqdm import tqdm from bcipy.acquisition.devices import preconfigured_device -from bcipy.config import (DEFAULT_PARAMETERS_FILENAME, RAW_DATA_FILENAME, +from bcipy.config import (RAW_DATA_FILENAME, TRIGGER_FILENAME, SESSION_LOG_FILENAME) -from bcipy.io.load import load_json_parameters, load_raw_data -from bcipy.core.raw_data import RawData +from bcipy.io.load import load_raw_data +from bcipy.core.raw_data import RawData, get_1020_channel_map from bcipy.core.triggers import trigger_decoder from bcipy.signal.process import Composition -# from bcipy.signal.process import get_default_transform logger = logging.getLogger(SESSION_LOG_FILENAME) @@ -52,7 +52,9 @@ def convert_to_bids( output_dir: str, task_name: Optional[str] = None, line_frequency: float = 60, - format: ConvertFormat = ConvertFormat.BV) -> str: + format: ConvertFormat = ConvertFormat.BV, + label_duration: float = 0.5, + full_labels: bool = True) -> str: """Convert to BIDS. Convert the raw data to the Brain Imaging Data Structure (BIDS) format. @@ -65,13 +67,21 @@ def convert_to_bids( ---------- data_dir - path to the directory containing the raw data, triggers, and parameters participant_id - the participant ID + session_id - the session ID + run_id - the run ID output_dir - the directory to save the BIDS formatted data + task_name - the name of the task + line_frequency - the line frequency of the data (50 or 60 Hz) + format - the format to convert the data to (BrainVision, EDF, FIF, or EEGLAB) + label_duration - the duration of the trigger labels in seconds. Default is 0.5 seconds. + full_labels - if True, include the full trigger labels in the BIDS data. Default is True. If False, only include + the targetness labels (target/non-target). Returns ------- The path to the BIDS formatted data """ - # validate the inputs + # validate the inputs before proceeding if not os.path.exists(data_dir): raise FileNotFoundError(f"Data directory={data_dir} does not exist") if not os.path.exists(output_dir): @@ -87,24 +97,13 @@ def convert_to_bids( # create file paths for raw data, triggers, and parameters raw_data_file = os.path.join(data_dir, f'{RAW_DATA_FILENAME}.csv') trigger_file = os.path.join(data_dir, TRIGGER_FILENAME) - parameters_file = os.path.join(data_dir, DEFAULT_PARAMETERS_FILENAME) - # load the raw data + # load the raw data and specifications for the device used to collect the data raw_data = load_raw_data(raw_data_file) + channel_map = get_1020_channel_map(raw_data.channels) device_spec = preconfigured_device(raw_data.daq_type) volts = True if device_spec.channel_specs[0].units == 'volts' else False - # load the parameters - parameters = load_json_parameters(parameters_file, value_cast=True) - trial_window = parameters.get("trial_window", (0.0, 0.5)) - - if task_name is None: - task_name = parameters.get("task") - if task_name is None: - raise ValueError("Task name must be provided or specified in the parameters") - - window_length = trial_window[1] - trial_window[0] - # load the triggers without removing any triggers other than system triggers (default) trigger_targetness, trigger_timing, trigger_labels = trigger_decoder( trigger_path=trigger_file, @@ -113,19 +112,28 @@ def convert_to_bids( ) # convert the raw data to MNE format - mne_data = convert_to_mne(raw_data, volts=volts, remove_system_channels=True) + mne_data = convert_to_mne(raw_data, volts=volts, channel_map=channel_map) + + # add the trigger annotations to the MNE data targetness_annotations = mne.Annotations( onset=trigger_timing, - duration=[window_length] * len(trigger_timing), + duration=[label_duration] * len(trigger_timing), description=trigger_targetness, ) - label_annotations = mne.Annotations( - onset=trigger_timing, - duration=[window_length] * len(trigger_timing), - description=trigger_labels, - ) - mne_data.set_annotations(targetness_annotations + label_annotations) - mne_data.info["line_freq"] = line_frequency # set the line frequency to 60 Hz + + if full_labels: + label_annotations = mne.Annotations( + onset=trigger_timing, + duration=[label_duration] * len(trigger_timing), + description=trigger_labels, + ) + mne_data.set_annotations(targetness_annotations + label_annotations) + else: + mne_data.set_annotations(targetness_annotations) + # add the line frequency to the MNE data + mne_data.info["line_freq"] = line_frequency + + # create the BIDS path for the data bids_path = BIDSPath( subject=participant_id, session=session_id, @@ -143,7 +151,73 @@ def convert_to_bids( allow_preload=True, overwrite=True) - return bids_path.root + return bids_path.directory + + +def convert_eyetracking_to_bids( + raw_data_path, + output_dir, + participant_id, + session_id, + run_id, + task_name) -> str: + """Converts the raw eye tracking data to BIDS format. + + There is currently no standard for eye tracking data in BIDS. This function will write the raw eye tracking data + to a tsv file in a BIDS-style format. + + Parameters + ---------- + raw_data_path : str + Path to the raw eye tracking data + output_dir : str + Path to the output directory. + This should be where other BIDS formatted data is stored for the participant, session, and run. + participant_id : str + Participant ID, e.g. 'S01' + session_id : str + Session ID, e.g. '01' + run_id : str + Run ID, e.g. '01' + task_name : str + Task name. Example: 'RSVPCalibration' + + Returns + ------- + str + Path to the BIDS formatted eye tracking data + """ + # check that the raw data path exists + if not os.path.exists(raw_data_path): + raise FileNotFoundError(f"Raw eye tracking data path={raw_data_path} does not exist") + + if not os.path.exists(output_dir): + raise FileNotFoundError(f"Output directory={output_dir} does not exist") + + found_files = glob.glob(f"{raw_data_path}/eyetracker*.csv") + if len(found_files) == 0: + raise FileNotFoundError(f"No raw eye tracking data found in directory={raw_data_path}") + if len(found_files) > 1: + raise ValueError(f"Multiple raw eye tracking data files found in directory={raw_data_path}") + + eye_tracking_file = found_files[0] + logger.info(f"Found raw eye tracking data file={eye_tracking_file}") + + # load the raw eye tracking data + raw_data = load_raw_data(eye_tracking_file) + # get the data as a pandas DataFrame + data = raw_data.dataframe + + # make the et subdirectory + et_dir = os.path.join(output_dir, 'et') + os.makedirs(et_dir, exist_ok=True) + + # write the dataframe as a tsv file to the output directory + output_filename = f'sub-{participant_id}_ses-{session_id}_task-{task_name}_run-{run_id}_eyetracking.tsv' + output_path = os.path.join(et_dir, output_filename) + data.to_csv(output_path, sep='\t', index=False) + logger.info(f"Eye tracking data saved to {output_path}") + return output_path def compress(tar_file_name: str, members: List[str]) -> None: @@ -297,7 +371,7 @@ def convert_to_mne( transform: Optional[Composition] = None, montage: str = 'standard_1020', volts: bool = False, - remove_system_channels: bool = False) -> RawArray: + remove_system_channels: bool = True) -> RawArray: """Convert to MNE. Returns BciPy RawData as an MNE RawArray. This assumes all channel names @@ -338,6 +412,7 @@ def convert_to_mne( # if no channel types provided, assume all channels are eeg if not channel_types: + logger.warning("No channel types provided. Assuming all channels are EEG.") channel_types = ['eeg'] * len(channels) # check that number of channel types matches number of channels in the case custom channel types are provided diff --git a/bcipy/io/demo/demo_convert.py b/bcipy/io/demo/demo_convert.py index 37e98821..fd08b71b 100644 --- a/bcipy/io/demo/demo_convert.py +++ b/bcipy/io/demo/demo_convert.py @@ -4,33 +4,80 @@ `python bcipy/io/demo/demo_convert.py -d "path://to/bcipy/data/folder"` """ -from typing import Optional -from bcipy.io.convert import convert_to_bids, ConvertFormat +from typing import Optional, List +from pathlib import Path +from bcipy.io.convert import convert_to_bids, ConvertFormat, convert_eyetracking_to_bids from bcipy.gui.file_dialog import ask_directory -from bcipy.io.load import load_bcipy_data +from bcipy.io.load import BciPySessionTaskData, load_bcipy_data EXCLUDED_TASKS = ['Report', 'Offline', 'Intertask', 'BAD'] +def load_historical_bcipy_data(directory: str, experiment_id: str, + task_name: str = 'MatrixCalibration') -> List[BciPySessionTaskData]: + """Load the data from the given directory. + + The expected directory structure is: + + directory/ + user1/ + task_run/ + raw_data.csv + user2/ + task_run/ + raw_data.csv + """ + + data = Path(directory) + experiment_data = [] + for participant in data.iterdir(): + + # Skip files + if not participant.is_dir(): + continue + + # pull out the user id. This is the name of the folder + user_id = participant.name + + for task_run in participant.iterdir(): + if not task_run.is_dir(): + continue + + session_data = BciPySessionTaskData( + path=task_run, + user_id=user_id, + experiment_id=experiment_id, + session_id=1, + run=1, + task_name=task_name + ) + experiment_data.append(session_data) + return experiment_data + + def convert_experiment_to_bids( directory: str, experiment_id: str, format: ConvertFormat = ConvertFormat.BV, - output_dir: Optional[str] = None) -> None: + output_dir: Optional[str] = None, + include_eye_tracker: bool = False) -> None: """Converts the data in the study folder to BIDS format.""" - experiment_data = load_bcipy_data( - directory, - experiment_id=experiment_id, - excluded_tasks=EXCLUDED_TASKS, - anonymize=False) + # Use for data pre-2.0rc4 + # experiment_data = load_historical_bcipy_data( + # directory, + # experiment_id=experiment_id) + + # Use for data post-2.0rc4 + experiment_data = load_bcipy_data(directory, experiment_id, excluded_tasks=EXCLUDED_TASKS) + if not output_dir: output_dir = directory errors = [] for data in experiment_data: try: - convert_to_bids( + bids_path = convert_to_bids( data_dir=data.path, participant_id=data.user_id, session_id=data.session_id, @@ -39,6 +86,24 @@ def convert_experiment_to_bids( output_dir=f'{output_dir}/bids_{experiment_id}/', format=format ) + + if include_eye_tracker: + # Convert the eye tracker data + # The eye tracker data is in the same folder as the EEG data, but should be saved in a different folder + bids_path = Path(bids_path).parent + try: + convert_eyetracking_to_bids( + raw_data_path=data.path, + participant_id=data.user_id, + session_id=data.session_id, + run_id=data.run, + output_dir=bids_path, + task_name=data.task_name + ) + except Exception as e: + print(f"Error converting eye tracker data for {data.path} - {e}") + errors.append(f"Error converting eye tracker data for {data.path}") + except Exception as e: print(f"Error converting {data.path} - {e}") errors.append(str(data.path)) @@ -64,14 +129,26 @@ def convert_experiment_to_bids( '-e', '--experiment', help='Experiment ID to convert', - default='SCRD', + default='Matrix Multimodal Experiment', + ) + parser.add_argument( + '-et', + '--eye_tracker', + help='Include eye tracker data', + default=False, + action='store_true', ) args = parser.parse_args() path = args.directory if not path: - path = ask_directory("Select the directory with data to be converted") + path = ask_directory("Select the directory with data to be converted", strict=True) # convert a study to BIDS format - convert_experiment_to_bids(path, args.experiment, ConvertFormat.BV) + convert_experiment_to_bids( + path, + args.experiment, + ConvertFormat.BV, + output_dir=".", + include_eye_tracker=args.eye_tracker) diff --git a/bcipy/io/load.py b/bcipy/io/load.py index 691bc747..9bbf1e5a 100644 --- a/bcipy/io/load.py +++ b/bcipy/io/load.py @@ -10,7 +10,6 @@ from bcipy.config import (DEFAULT_ENCODING, DEFAULT_EXPERIMENT_PATH, DEFAULT_FIELD_PATH, DEFAULT_PARAMETERS_PATH, - DEFAULT_PARAMETERS_FILENAME, EXPERIMENT_FILENAME, FIELD_FILENAME, SIGNAL_MODEL_FILE_SUFFIX, SESSION_LOG_FILENAME) from bcipy.gui.file_dialog import ask_directory, ask_filename @@ -156,8 +155,8 @@ def load_json_parameters(path: str, value_cast: bool = False) -> Parameters: return Parameters(source=path, cast_values=value_cast) -def load_experimental_data() -> str: - filename = ask_directory() # show dialog box and return the path +def load_experimental_data(message='', strict=False) -> str: + filename = ask_directory(prompt=message, strict=strict) # show dialog box and return the path log.info("Loaded Experimental Data From: %s" % filename) return filename @@ -307,10 +306,10 @@ def fast_scandir(directory_name: str, return_path: bool = True) -> List[str]: class BciPySessionTaskData: """Session Task Data. - This class is used to represent a single task data session. It is used to store the + This class is used to represent a single task session. It is used to store the path to the task data, as well as the parameters and other information about the task. - ////// + // protocol.json / parameters.json @@ -322,39 +321,32 @@ def __init__( self, path: str, user_id: str, - date: str, experiment_id: str, - date_time: str, - task: str, + date_time: Optional[str] = None, + date: Optional[str] = None, + task_name: Optional[str] = None, + session_id: int = 1, run: int = 1) -> None: self.user_id = user_id - self.date = date self.experiment_id = experiment_id.replace('_', '') - self.date_time = date_time.replace("_", "").replace("-", "") - self.session_id = f'{self.experiment_id}{self.date_time}' + self.session_id = f'0{str(session_id)}' if session_id < 10 else str(session_id) self.date_time = date_time - self.task = task - self.run = str(run) + self.date = date + self.run = f'0{str(run)}' if run < 10 else str(run) self.path = path - self.parameters = self.get_parameters() - self.task_name = self.parameters.get('task', 'unknown').replace(' ', '') + self.task_name = task_name self.info = { - 'user_id': user_id, - 'date': date, + 'user_id': self.user_id, 'experiment_id': self.experiment_id, - 'date_time': self.date_time, - 'task': task, 'task_name': self.task_name, - 'run': run, - 'path': path + 'session_id': self.session_id, + 'run': self.run, + 'date': self.date, + 'date_time': self.date_time, + 'path': self.path } - def get_parameters(self) -> Parameters: - return load_json_parameters( - f'{self.path}/{DEFAULT_PARAMETERS_FILENAME}', - value_cast=True) - def __str__(self): return f'BciPySessionTaskData: {self.info=}' @@ -549,15 +541,16 @@ def load_tasks(self) -> None: date = task_path.parts[-4] experiment_id = task_path.parts[-3] date_time = task_path.parts[-2] + task_name = task.split(date)[0].strip('_') self.session_task_data.append( BciPySessionTaskData( path=task_path, user_id=user_id, + date_time=date_time, date=date, experiment_id=experiment_id, - date_time=date_time, run=run, - task=task + task_name=task_name ) ) run += 1 @@ -578,7 +571,7 @@ def load_bcipy_data( Walks a data directory and returns a list of data paths for the given experiment id, user id, and date. The BciPy data directory is structured as follows: - data_directory/ + data/ user_ids/ dates/ experiment_ids/ diff --git a/bcipy/io/tests/test_convert.py b/bcipy/io/tests/test_convert.py index 3898e580..d9e6d8d7 100644 --- a/bcipy/io/tests/test_convert.py +++ b/bcipy/io/tests/test_convert.py @@ -18,7 +18,8 @@ convert_to_mne, decompress, norm_to_tobii, - tobii_to_norm + tobii_to_norm, + convert_eyetracking_to_bids ) from bcipy.core.parameters import Parameters from bcipy.core.raw_data import RawData, sample_data, write @@ -244,7 +245,7 @@ def setUp(self): 'notch_filter_frequency': 60, 'down_sampling_rate': 3 } - self.channels = ['timestamp', 'O1', 'O2', 'Pz'] + self.channels = ['timestamp', 'O1', 'O2', 'Pz', 'TRG', 'lsl_timestamp'] self.raw_data = RawData('SampleDevice', self.sample_rate, self.channels) devices.register(devices.DeviceSpec('SampleDevice', channels=self.channels, sample_rate=self.sample_rate)) @@ -264,26 +265,46 @@ def test_convert_to_mne_defaults(self): data = convert_to_mne(self.raw_data) self.assertTrue(len(data) > 0) - self.assertEqual(data.ch_names, self.channels[1:]) + self.assertEqual(data.ch_names, self.channels[1:-2]) self.assertEqual(data.info['sfreq'], self.sample_rate) def test_convert_to_mne_with_channel_map(self): """Test the convert_to_mne function with channel mapping""" # here we know only three channels are generated, using the channel map let's only use the last one - channel_map = [0, 0, 1] + channel_map = [0, 0, 1, 0, 0] data = convert_to_mne(self.raw_data, channel_map=channel_map) self.assertTrue(len(data) > 0) self.assertTrue(len(data.ch_names) == 1) # this is the main assertion! self.assertEqual(data.info['sfreq'], self.sample_rate) + def test_convert_to_mne_with_remove_system_channels(self): + """Test the convert_to_mne function with system channels removed""" + data = convert_to_mne(self.raw_data, remove_system_channels=True) + + self.assertTrue(len(data) > 0) + self.assertEqual(data.ch_names, self.channels[1:-2]) + self.assertEqual(data.info['sfreq'], self.sample_rate) + + def test_convert_to_mne_without_remove_system_channels_throws_error(self): + """Test the convert_to_mne function with system channels removed raises an error. + + This is due to MNE requiring the channels be EEG. The system channels are not EEG. + """ + with self.assertRaises(ValueError): + data = convert_to_mne(self.raw_data, remove_system_channels=False) + + self.assertTrue(len(data) > 0) + self.assertEqual(data.ch_names, self.channels[1:]) + self.assertEqual(data.info['sfreq'], self.sample_rate) + def test_convert_to_mne_with_channel_types(self): """Test the convert_to_mne function with channel types""" channel_types = ['eeg', 'eeg', 'seeg'] data = convert_to_mne(self.raw_data, channel_types=channel_types) self.assertTrue(len(data) > 0) - self.assertEqual(data.ch_names, self.channels[1:]) + self.assertEqual(data.ch_names, self.channels[1:-2]) self.assertEqual(data.info['sfreq'], self.sample_rate) self.assertTrue(data.get_channel_types()[2] == 'seeg') @@ -297,7 +318,7 @@ def transform(x, fs): data = convert_to_mne(self.raw_data, transform=transform, volts=True) self.assertTrue(len(data) > 0) - self.assertEqual(data.ch_names, self.channels[1:]) + self.assertEqual(data.ch_names, self.channels[1:-2]) self.assertEqual(data.info['sfreq'], self.sample_rate) # apply the transform to the first data point and compare to data returned @@ -309,7 +330,7 @@ def test_convert_to_mne_with_mv_conversion(self): data = convert_to_mne(self.raw_data, volts=False) self.assertTrue(len(data) > 0) - self.assertEqual(data.ch_names, self.channels[1:]) + self.assertEqual(data.ch_names, self.channels[1:-2]) self.assertEqual(data.info['sfreq'], self.sample_rate) # apply the transform to the first data point and compare to data returned @@ -325,7 +346,7 @@ def test_convert_to_mne_with_custom_montage(self): data = convert_to_mne(self.raw_data, montage=montage_type) self.assertTrue(len(data) > 0) - self.assertEqual(data.ch_names, self.channels[1:]) + self.assertEqual(data.ch_names, self.channels[1:-2]) self.assertEqual(data.info['sfreq'], self.sample_rate) @@ -445,5 +466,151 @@ def test_norm_to_tobii_raises_error_with_invalid_units(self): norm_to_tobii(norm_data) +class TestConvertETBIDS(unittest.TestCase): + + def setUp(self): + self.temp_dir = tempfile.mkdtemp() + self.trg_data, self.data, self.params = create_bcipy_session_artifacts(self.temp_dir, channels=3) + self.eyetracking_data = sample_data( + ch_names=[ + 'timestamp', + 'x', + 'y', + 'pupil'], + daq_type='Gaze', + sample_rate=60, + rows=5000) + devices.register(devices.DeviceSpec('Gaze', channels=['timestamp', 'x', 'y', 'pupil'], sample_rate=60)) + + write(self.eyetracking_data, Path(self.temp_dir, 'eyetracker.csv')) + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + def test_convert_eyetracking_to_bids_generates_bids_strucutre(self): + """Test the convert_eyetracking_to_bids function""" + response = convert_eyetracking_to_bids( + f"{self.temp_dir}/", + participant_id='01', + session_id='01', + run_id='01', + task_name='TestTask', + output_dir=self.temp_dir, + ) + self.assertTrue(os.path.exists(response)) + # Assert the session directory was created with et + self.assertTrue(os.path.exists(f"{self.temp_dir}/et/")) + # Assert the et tsv file was created with the correct name + self.assertTrue(os.path.exists(f"{self.temp_dir}/et/sub-01_ses-01_task-TestTask_run-01_eyetracking.tsv")) + + def test_convert_eyetracking_to_bids_reflects_participant_id(self): + """Test the convert_eyetracking_to_bids function with a participant id""" + response = convert_eyetracking_to_bids( + f"{self.temp_dir}/", + participant_id='100', + session_id='01', + run_id='01', + task_name='TestTask', + output_dir=self.temp_dir, + ) + self.assertTrue(os.path.exists(response)) + # Assert the et tsv file was created with the correct name + self.assertTrue(os.path.exists(f"{self.temp_dir}/et/sub-100_ses-01_task-TestTask_run-01_eyetracking.tsv")) + + def test_convert_eyetracking_to_bids_reflects_session_id(self): + """Test the convert_eyetracking_to_bids function with a session id""" + response = convert_eyetracking_to_bids( + f"{self.temp_dir}/", + participant_id='01', + session_id='100', + run_id='01', + task_name='TestTask', + output_dir=self.temp_dir, + ) + self.assertTrue(os.path.exists(response)) + # Assert the et tsv file was created with the correct name + self.assertTrue(os.path.exists(f"{self.temp_dir}/et/sub-01_ses-100_task-TestTask_run-01_eyetracking.tsv")) + + def test_convert_eyetracking_to_bids_reflects_run_id(self): + """Test the convert_eyetracking_to_bids function with a run id""" + response = convert_eyetracking_to_bids( + f"{self.temp_dir}/", + participant_id='01', + session_id='01', + run_id='100', + task_name='TestTask', + output_dir=self.temp_dir, + ) + self.assertTrue(os.path.exists(response)) + # Assert the et tsv file was created with the correct name + self.assertTrue(os.path.exists(f"{self.temp_dir}/et/sub-01_ses-01_task-TestTask_run-100_eyetracking.tsv")) + + def test_convert_eyetracking_to_bids_reflects_task_name(self): + """Test the convert_eyetracking_to_bids function with a task name""" + response = convert_eyetracking_to_bids( + f"{self.temp_dir}/", + participant_id='01', + session_id='01', + run_id='01', + task_name='TestTaskEtc', + output_dir=self.temp_dir, + ) + self.assertTrue(os.path.exists(response)) + # Assert the et tsv file was created with the correct name + self.assertTrue(os.path.exists(f"{self.temp_dir}/et/sub-01_ses-01_task-TestTaskEtc_run-01_eyetracking.tsv")) + + def test_convert_et_raises_error_with_invalid_data_dir(self): + """Test the convert_eyetracking_to_bids function raises an error with invalid output directory""" + with self.assertRaises(FileNotFoundError): + convert_eyetracking_to_bids( + 'invalid_data_dir', + participant_id='01', + session_id='01', + run_id='01', + task_name='TestTask', + output_dir=self.temp_dir + ) + + def test_convert_et_raises_error_with_output_dir_not_exist(self): + """Test the convert_eyetracking_to_bids function raises an error with invalid output directory""" + with self.assertRaises(FileNotFoundError): + convert_eyetracking_to_bids( + f"{self.temp_dir}/", + participant_id='01', + session_id='01', + run_id='01', + task_name='TestTask', + output_dir='invalid_output_dir' + ) + + def test_convert_et_raises_error_with_no_data_file(self): + """Test the convert_eyetracking_to_bids function raises an error with no data file""" + # remove the csv file + os.remove(f"{self.temp_dir}/eyetracker.csv") + with self.assertRaises(FileNotFoundError): + convert_eyetracking_to_bids( + f"{self.temp_dir}/", + participant_id='01', + session_id='01', + run_id='01', + task_name='TestTask', + output_dir=self.temp_dir, + ) + + def test_convert_et_raises_error_with_multiple_data_files(self): + """Test the convert_eyetracking_to_bids function raises an error with multiple data files""" + # create a second data file + write(self.eyetracking_data, Path(self.temp_dir, 'eyetracker_2.csv')) + with self.assertRaises(ValueError): + convert_eyetracking_to_bids( + f"{self.temp_dir}/", + participant_id='01', + session_id='01', + run_id='01', + task_name='TestTask', + output_dir=self.temp_dir, + ) + + if __name__ == '__main__': unittest.main() diff --git a/bcipy/signal/evaluate/artifact.py b/bcipy/signal/evaluate/artifact.py index afffaebc..3c09e1d0 100644 --- a/bcipy/signal/evaluate/artifact.py +++ b/bcipy/signal/evaluate/artifact.py @@ -566,7 +566,8 @@ def write_mne_annotations( # if no path is provided, prompt for one using a GUI path = args.path if not path: - path = load_experimental_data() + path = load_experimental_data( + message='Select the directory with the sessions to analyze for artifacts', strict=True) positions = None for session in Path(path).iterdir(): diff --git a/bcipy/signal/model/offline_analysis.py b/bcipy/signal/model/offline_analysis.py index 22d611b2..785df678 100644 --- a/bcipy/signal/model/offline_analysis.py +++ b/bcipy/signal/model/offline_analysis.py @@ -510,7 +510,9 @@ def offline_analysis( """ assert parameters, "Parameters are required for offline analysis." if not data_folder: - data_folder = load_experimental_data() + data_folder = load_experimental_data( + message="Select the folder containing the data to be analyzed.", + strict=True) # Load default devices which are used for training the model with different channels, etc. devices_by_name = devices.load( diff --git a/bcipy/task/actions.py b/bcipy/task/actions.py index b2429a94..2937a35f 100644 --- a/bcipy/task/actions.py +++ b/bcipy/task/actions.py @@ -217,7 +217,8 @@ def __init__( if not protocol_path: protocol_path = ask_directory( - prompt="Select BciPy protocol directory with calibration data...") + prompt="Select BciPy protocol directory with calibration data...", + strict=True) self.protocol_path = protocol_path self.last_task_dir = last_task_dir self.trial_window = (-0.2, 1.0) diff --git a/requirements.txt b/requirements.txt index b41684b2..819a1c90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,6 +24,7 @@ Pillow==9.4.0 py-cpuinfo==9.0.0 pyopengl==3.1.7 PyQt6==6.7.1 +PyQt6-sip==13.8 pywavelets==1.4.1 tqdm==4.62.2 reportlab==4.2.0