Skip to content

Commit

Permalink
Add MATLAB traversal without changing input folder content
Browse files Browse the repository at this point in the history
  • Loading branch information
@dissagaliyeva
dissagaliyeva committed May 4, 2023
1 parent fa5c074 commit 714037e
Show file tree
Hide file tree
Showing 7 changed files with 208 additions and 123 deletions.
101 changes: 48 additions & 53 deletions app_templates/example.ipynb
View file

Large diffs are not rendered by default.

147 changes: 105 additions & 42 deletions sim2bids/app/app.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@
import pandas as pd
import panel as pn
import lems.api as lems
from scipy.io import loadmat
import mat73
import scipy.io as sio

# import local packages
import requests
Expand Down Expand Up @@ -56,19 +59,21 @@
SESSIONS = False
H5_CONTENT = dict()
MISSING = []
SUBJECTS = None
ADDED_FILES = []

# store different time series accounting for 'times'
TIMES = []

# define all accepted files
ACCEPTED = ['weight', 'distance', 'tract_length', 'delay', 'speeds', # Network (net)
'nodes', 'labels', 'centre', 'area', 'hemisphere', 'cortical', # Coordinates (coord)
'orientation', 'average_orientation', 'normals', 'times', 'bold_ts', 'vertices', # Coordinates (coord)
'faces', 'vnormal', 'fnormal', 'sensor', 'volume', 'map', # Coordinates (coord)
'cartesian2d', 'cartesian3d', 'polar2d', 'polar3d', # Coordinates (coord)
'vars', 'stimuli', 'noise', 'spike', 'raster', 'ts', 'event', # Timeseries (ts)
'emp', 'bold_times', 'hrf' # Timeseries (ts)
'emp_fc', 'fc'] # Spatial (spatial)
ACCEPTED = ['weight', 'distance', 'tract_length', 'delay', 'speeds', # Network (net)
'nodes', 'labels', 'centre', 'area', 'hemisphere', 'cortical', # Coordinates (coord)
'orientation', 'average_orientation', 'normals', 'times', 'bold_ts', 'vertices', # Coordinates (coord)
'faces', 'vnormal', 'fnormal', 'sensor', 'volume', 'map', # Coordinates (coord)
'cartesian2d', 'cartesian3d', 'polar2d', 'polar3d', # Coordinates (coord)
'vars', 'stimuli', 'noise', 'spike', 'raster', 'ts', 'event', # Timeseries (ts)
'emp', 'bold_times', 'hrf' # Timeseries (ts)
'emp_fc', 'fc'] # Spatial (spatial)

TO_EXTRACT = ['weights.txt', 'centres.txt', 'distances.txt', # folder "net"
'areas.txt', 'average_orientations.txt', 'cortical.txt', # folder "coord"
Expand Down Expand Up @@ -106,13 +111,14 @@ def main(path: str, files: list, subs: dict = None, save: bool = False, layout:
:param input_path:
"""
global MODEL_NAME, INPUT, INPUT_TRANSFERRED
global MODEL_NAME, INPUT, INPUT_TRANSFERRED, SUBJECTS

# whether to generate layout
if layout:
# if no subjects are passed, define them
if subs is None:
subs = subjects.Files(path, files).subs
SUBJECTS = subs
# results = convert.check_centres()
# if results:
# convert.IGNORE_CENTRE = results
Expand Down Expand Up @@ -154,44 +160,79 @@ def main(path: str, files: list, subs: dict = None, save: bool = False, layout:
return None


def preprocess_input(path, input_files):
to_preprocess = {}

for root, dirs, files in os.walk(path):
for file in files:
if file in input_files:
continue

if file.endswith('mat') and 'csf' not in root and 'cspeed' not in root and root != path:
if root not in to_preprocess:
to_preprocess[root] = []
to_preprocess[root].append(file)
def get_mat_files(input_path):
matches = []
for root, dirnames, filenames in os.walk(input_path):
for filename in filenames:
if filename.endswith('.mat'):
matches.append(os.path.join(root, filename))
return matches


def open_mat(file):
try:
matfile = loadmat(file, squeeze_me=True)
except NotImplementedError:
return mat73.loadmat(file)
except sio.matlab._miobase.MatReadError:
pn.state.notifications.error(f'File `{file}` is empty! Aborting...')
return None
except FileNotFoundError:
return None
else:
return matfile

for root, content in to_preprocess.items():
for c in content:
cspeed = re.findall(r'speed_[0-9]+', c)

if cspeed:
cspeed = cspeed[0].replace('speed_', '')
csf = re.findall(r'csf_[0-9\.]+', c)[0].replace('csf_', '')
def preprocess_input(path, input_files):
mat_files = []

# check if a folder is passed
if len(input_files) > 0:
# and os.path.isdir(os.path.join(path, input_files[0]))
# iterate over input files
for file in input_files:
fpath = os.path.join(path, file)

# check if it's a directory, if yes get all matlab contents
if os.path.isdir(fpath):
mat_files += get_mat_files(fpath)
elif fpath.endswith('.mat'):
mat_files.append(fpath)

if len(mat_files) > 0:
# iterate over matlab files and get the content
for file in mat_files:
mat = open_mat(file)

if not isinstance(mat, dict):
continue
else:
# iterate over keys and save content
path, fname = os.path.dirname(file), os.path.basename(file).replace('.mat', '')

path = os.path.join(root, f'{COND_SPEED}{cspeed}')
if not os.path.exists(path):
os.mkdir(path)
for array in mat.keys():
if array.startswith('__') or array.endswith('__'): continue

path = os.path.join(path, f'csf {csf}')
if not os.path.exists(path):
os.mkdir(path)
# get new path
new_path = os.path.join(path, f'{fname}_{array}.txt')

# copy file
shutil.move(os.path.join(root, c), path)
# save array in the same directory as the original matlab file
with open(new_path, 'w') as fout:
try:
fout.write(mat[array])
except TypeError:
try:
np.savetxt(new_path, mat[array])
except ValueError:
np.savetxt(new_path, mat[array][:, 0, :])


def save_params():
global MODEL_NAME

# verify path exists
path = os.path.join(OUTPUT, 'param')

if not os.path.exists(path):
os.mkdir(path)

Expand Down Expand Up @@ -291,6 +332,7 @@ def save_missing(path, files):
with open(os.path.join(OUTPUT, 'dataset_description.json'), 'w') as f:
json.dump(json_file, f)


def save_output(subs):
"""
Expand Down Expand Up @@ -435,12 +477,14 @@ def save_code():
-------
"""
global SoftwareVersion

path = 'https://github.com/the-virtual-brain/tvb-root/archive/refs/tags/{}.zip'

if SoftwareName == 'TVB':
if SoftwareVersion == 1.5:
path = path.replace(str(SoftwareVersion), '1.5.10')
SoftwareVersion = '1.5.10'
path = 'https://github.com/the-virtual-brain/tvb-root/archive/refs/tags/1.5.10.zip'
elif SoftwareVersion:
path = path.format(str(SoftwareVersion))

Expand All @@ -461,7 +505,7 @@ def save_code():
pn.state.notifications.error('Please check the TVB version!', duration=5000)

if isinstance(CODE, str) and CODE.endswith('.py'):
template = f'code.py'
template = f'desc-{DESC}_code.py'
path = os.path.join(OUTPUT, 'code', template)

shutil.copy(CODE, path)
Expand Down Expand Up @@ -489,6 +533,7 @@ def save_code():
def transfer_xml():
# transfer results to appropriate folders
path = os.path.join(OUTPUT, 'param', '_eq.xml')

if os.path.exists(path):
shutil.move(path, os.path.join(OUTPUT, 'eq'))

Expand Down Expand Up @@ -516,8 +561,18 @@ def save():
if len(temp.struct[ftype]['required']) > 0:
file.update(get_dict('required'))

eq = '../eq/eq.xml'
file['SourceCode'] = f'../code/code.py'
print(f'Ftype: {ftype}')
eq = None

if SESSIONS:
if ftype.lower() not in ['code', 'param']:
if ftype.lower() != 'eq':
eq = f'../../eq/desc-{DESC}_eq.xml'
else:
eq = f'../eq/desc-{DESC}_eq.xml'

if ftype.lower() != 'code':
file['SourceCode'] = f'../code/desc-{DESC}_code.py'

for key, val in zip(['SoftwareVersion', 'SoftwareRepository', 'SoftwareName'],
[SoftwareVersion, SoftwareRepository, SoftwareName]):
Expand All @@ -528,10 +583,12 @@ def save():
file[key] = val

if ftype == 'code':
file['ModelEq'] = eq
if eq:
file['ModelEq'] = eq
file['Description'] = 'The source code to reproduce results.'
elif ftype == 'param':
file['ModelEq'] = eq
if eq:
file['ModelEq'] = eq
file['Description'] = f'These are the parameters for the {MODEL_NAME} model.'
elif ftype == 'eq':
file['Description'] = f'These are the equations to simulate the time series with the {MODEL_NAME} model.'
Expand Down Expand Up @@ -582,7 +639,13 @@ def check_output():

if os.path.exists(path) and len(os.listdir(path)) > 0:
for file in os.listdir(path):
fpath = os.path.join(path, file)

if file.startswith('tvb'): continue

if f'desc-{DESC}_' not in fpath:
os.rename(fpath, os.path.join(path, f'desc-{DESC}_{file}'))

if file.startswith('sub-'):
match = re.match('sub-[0-9a-zA-Z\_]+', file)[0]
os.replace(os.path.join(path, file), os.path.join(path, file.replace(match, '').strip('_')))

24 changes: 12 additions & 12 deletions sim2bids/app/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,11 +59,11 @@ def recursive_walk(path: str, basename: bool = False) -> list:
os.remove(os.path.join(root, file))
continue

if file.endswith('.h5'):
content += extract_h5(os.path.join(root, file))
# if file.endswith('.h5'):
# content += extract_h5(os.path.join(root, file))

if file.endswith('.mat'):
content += extract_mat(os.path.join(root, file), path)
# if file.endswith('.mat'):
# content += extract_mat(os.path.join(root, file), path)

if 'times' in file:
app.TIMES.append(subj.accepted(file)[-1])
Expand Down Expand Up @@ -135,7 +135,7 @@ def get_content(path: str, files: [str, list], basename: bool = False) -> list:

# traverse files
for file in files:
if '.ipynb_checkpoints' in file:
if '.ipynb_checkpoints' in file or file.endswith('.mat'):
continue

# combine path
Expand All @@ -161,13 +161,13 @@ def get_content(path: str, files: [str, list], basename: bool = False) -> list:
# get the file's extension
ext = os.path.basename(file).split('.')[-1]

if ext == 'h5':
extract_h5(file_path)
continue

if file.endswith('.mat'):
contents += extract_mat(os.path.join(path, file), path)
continue
# if ext == 'h5':
# extract_h5(file_path)
# continue
#
# if file.endswith('.mat'):
# contents += extract_mat(os.path.join(path, file), path)
# continue

# check if it's among the accepted extensions
if ext in app.ACCEPTED_EXT:
Expand Down
43 changes: 30 additions & 13 deletions sim2bids/convert/convert.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ def save_files(sub: dict, folder: str, content, type: str = 'default', centres:
"""
global COORDS

name = sub['name'].replace('.txt', '')
name = f'desc-{app.DESC}_' + sub['name'].replace('.txt', '')

if type == 'default':
json_file = os.path.join(folder, DEFAULT_TEMPLATE.format(sub['sid'], name, 'json'))
Expand All @@ -378,7 +378,7 @@ def save_files(sub: dict, folder: str, content, type: str = 'default', centres:
nodes = json_file.replace(sub['name'], 'nodes')

if COORDS is None:
if IGNORE_CENTRE or not app.MULTI_INPUT:
if IGNORE_CENTRE or not app.MULTI_INPUT and not app.SESSIONS:
COORDS = [f'../coord/desc-{app.DESC}_labels.json', f'../coord/desc-{app.DESC}_nodes.json']
else:
COORDS = [labels.replace(app.OUTPUT, '../..').replace('\\', '/'),
Expand Down Expand Up @@ -644,46 +644,63 @@ def to_json(path, shape, desc, key, **kwargs):
# ===========================================================
coord = None

def one_centre():
for k, v in app.SUBJECTS.items():
for k2, v2 in v.items():
if 'centre' in v2 or 'center' in v2 or 'centres' in v2:
print('Center file is in subject-level instance')
return False
return True

# check IGNORE_CENTRE again
nodes, labels = f'desc-{app.DESC}_nodes.json', f'desc-{app.DESC}_labels.json'

if 'CoordsRows' in out.keys() or key in ['wd', 'ts', 'spatial']:
if app.SESSIONS and IGNORE_CENTRE:
coord = ['../../../coord/nodes.json', '../../../coord/labels.json']
coord = [f'../../../coord/{nodes}', f'../../../coord/{labels}']
elif app.SESSIONS and not IGNORE_CENTRE:
coord = ['../coord/nodes.json', '../coord/labels.json']
coord = [f'../coord/{nodes}', f'../coord/{labels}']
elif not app.SESSIONS and IGNORE_CENTRE:
coord = ['../../coord/nodes.json', '../../coord/labels.json']
coord = [f'../../coord/{nodes}', f'../../coord/{labels}']
elif app.MULTI_INPUT:
# check if centres exist on subject-level
if one_centre():
coord = [f'../../coord/{nodes}', f'../../coord/{labels}']
else:
coord = ['../coord/nodes.json', '../coord/labels.json']
coord = [f'../coord/{nodes}', f'../coord/{labels}']

if key != 'wd':

# ===========================================================
# TAKE CARE OF EQUATIONS
# ===========================================================
eq_name = f'desc-{app.DESC}_eq.xml'

if 'ModelEq' in out.keys() or 'ModelEq' in temp.struct[key]['recommend']:
if 'param' in path or 'code' in path:
out['ModelEq'] = '../eq/eq.xml'
out['ModelEq'] = f'../eq/{eq_name}'
else:
if app.SESSIONS:
out['ModelEq'] = '../../../eq/eq.xml'
out['ModelEq'] = f'../../../eq/{eq_name}'
else:
out['ModelEq'] = '../../eq/eq.xml'
out['ModelEq'] = f'../../eq/{eq_name}'

# ===========================================================
# TAKE CARE OF PARAMETERS
# ===========================================================
param_name = f'desc-{app.DESC}_param.xml'

if 'ModelParam' in out.keys() or 'ModelParam' in temp.struct[key]['recommend']:
if 'eq' in path or 'code' in path:
out['ModelParam'] = '../param/param.xml'
out['ModelParam'] = f'../param/{param_name}'
else:
if app.SESSIONS:
out['ModelParam'] = '../../../param/param.xml'
out['ModelParam'] = f'../../../param/{param_name}'
else:
out['ModelParam'] = '../../param/param.xml'
out['ModelParam'] = f'../../param/{param_name}'

params = {
'SourceCode': f'/tvb-framework-{app.SoftwareVersion}' if app.SoftwareCode == 'MISSING' else app.SoftwareCode,
'SourceCode': f'../code/tvb-framework-{app.SoftwareVersion}' if app.SoftwareCode == 'MISSING' else app.SoftwareCode,
'SoftwareVersion': app.SoftwareVersion if app.SoftwareVersion else None,
'SoftwareRepository': app.SoftwareRepository if app.SoftwareRepository else None,
'SourceCodeVersion': app.SoftwareVersion if app.SoftwareVersion else None,
Expand Down
Loading

0 comments on commit 714037e

Please sign in to comment.