diff --git a/CerebNet/datasets/utils.py b/CerebNet/datasets/utils.py index 4d848f4a..10cda76a 100644 --- a/CerebNet/datasets/utils.py +++ b/CerebNet/datasets/utils.py @@ -14,15 +14,16 @@ # IMPORTS -from typing import Tuple, Union, Sequence, Optional, TypeVar +from typing import Tuple, Union, Sequence, Optional, TypeVar, TypedDict, Iterable, Type +from pathlib import Path import nibabel as nib import numpy as np +from numpy import typing as npt import torch from FastSurferCNN.data_loader.conform import getscale, scalecrop -# class names for network training and validation/testing CLASS_NAMES = { "Background": 0, "Left_I_IV": 1, @@ -54,11 +55,38 @@ "Right_Corpus_Medullare": 38, } +# class names for network training and validation/testing subseg_labels = {"cereb_subseg": np.array(list(CLASS_NAMES.values()))} AT = TypeVar("AT", np.ndarray, torch.Tensor) +class LTADict(TypedDict): + type: int + nxforms: int + mean: list[float] + sigma: float + lta: npt.NDArray[float] + src_valid: int + src_filename: str + src_volume: list[int] + src_voxelsize: list[float] + src_xras: list[float] + src_yras: list[float] + src_zras: list[float] + src_cras: list[float] + dst_valid: int + dst_filename: str + dst_volume: list[int] + dst_voxelsize: list[float] + dst_xras: list[float] + dst_yras: list[float] + dst_zras: list[float] + dst_cras: list[float] + src: npt.NDArray[float] + dst: npt.NDArray[float] + + def define_size(mov_dim, ref_dim): new_dim = np.zeros(len(mov_dim), dtype=int) borders = np.zeros((len(mov_dim), 2), dtype=int) @@ -167,7 +195,7 @@ def bounding_volume_offset( if isinstance(img, np.ndarray): from FastSurferCNN.data_loader.data_utils import bbox_3d - bbox = bbox_3d(img != 0) + bbox = bbox_3d(np.not_equal(img, 0)) bbox = bbox[::2] + bbox[1::2] else: bbox = img @@ -325,237 +353,78 @@ def apply_warp_field(dform_field, img, interpol_order=3): return deformed_img -def readLTA(file): +def read_lta(file: Path | str) -> LTADict: + """Read the LTA info.""" import re + from functools import partial import numpy as np + parameter_pattern = re.compile("^\s*([^=]+)\s*=\s*([^#]*)\s*(#.*)") + vol_info_pattern = re.compile("^(.*) volume info$") + shape_pattern = re.compile("^(\s*\d+)+$") + matrix_pattern = re.compile("^(-?\d+\.\S+\s+)+$") + + _Type = TypeVar("_Type", bound=Type) + + def _vector(_a: str, dtype: Type[_Type] = float, count: int = -1) -> list[_Type]: + return np.fromstring(_a, dtype=dtype, count=count, sep=" ").tolist() + + parameters = { + "type": int, + "nxforms": int, + "mean": partial(_vector, dtype=float, count=3), + "sigma": float, + "subject": str, + "fscale": float, + } + vol_info_par = { + "valid": int, + "filename": str, + "volume": partial(_vector, dtype=int, count=3), + "voxelsize": partial(_vector, dtype=float, count=3), + **{f"{c}ras": partial(_vector, dtype=float) for c in "xyzc"} + } with open(file, "r") as f: - lta = f.readlines() - d = dict() - i = 0 - while i < len(lta): - if re.match("type", lta[i]) is not None: - d["type"] = int( - re.sub("=", "", re.sub("[a-z]+", "", re.sub("#.*", "", lta[i]))).strip() - ) - i += 1 - elif re.match("nxforms", lta[i]) is not None: - d["nxforms"] = int( - re.sub("=", "", re.sub("[a-z]+", "", re.sub("#.*", "", lta[i]))).strip() - ) - i += 1 - elif re.match("mean", lta[i]) is not None: - d["mean"] = [ - float(x) - for x in re.split( - " +", - re.sub( - "=", "", re.sub("[a-z]+", "", re.sub("#.*", "", lta[i])) - ).strip(), - ) - ] - i += 1 - elif re.match("sigma", lta[i]) is not None: - d["sigma"] = float( - re.sub("=", "", re.sub("[a-z]+", "", re.sub("#.*", "", lta[i]))).strip() - ) - i += 1 - elif ( - re.match( - "-*[0-9]\.\S+\W+-*[0-9]\.\S+\W+-*[0-9]\.\S+\W+-*[0-9]\.\S+\W+", lta[i] - ) - is not None - ): - d["lta"] = np.array( - [ - [ - float(x) - for x in re.split( - " +", - re.match( - "-*[0-9]\.\S+\W+-*[0-9]\.\S+\W+-*[0-9]\.\S+\W+-*[0-9]\.\S+\W+", - lta[i], - ).string.strip(), - ) - ], - [ - float(x) - for x in re.split( - " +", - re.match( - "-*[0-9]\.\S+\W+-*[0-9]\.\S+\W+-*[0-9]\.\S+\W+-*[0-9]\.\S+\W+", - lta[i + 1], - ).string.strip(), - ) - ], - [ - float(x) - for x in re.split( - " +", - re.match( - "-*[0-9]\.\S+\W+-*[0-9]\.\S+\W+-*[0-9]\.\S+\W+-*[0-9]\.\S+\W+", - lta[i + 2], - ).string.strip(), - ) - ], - [ - float(x) - for x in re.split( - " +", - re.match( - "-*[0-9]\.\S+\W+-*[0-9]\.\S+\W+-*[0-9]\.\S+\W+-*[0-9]\.\S+\W+", - lta[i + 3], - ).string.strip(), - ) - ], - ] - ) - i += 4 - elif re.match("src volume info", lta[i]) is not None: - while i < len(lta) and re.match("dst volume info", lta[i]) is None: - if re.match("valid", lta[i]) is not None: - d["src_valid"] = int( - re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - elif re.match("filename", lta[i]) is not None: - d["src_filename"] = re.split( - " +", re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - elif re.match("volume", lta[i]) is not None: - d["src_volume"] = [ - int(x) - for x in re.split( - " +", re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - ] - elif re.match("voxelsize", lta[i]) is not None: - d["src_voxelsize"] = [ - float(x) - for x in re.split( - " +", re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - ] - elif re.match("xras", lta[i]) is not None: - d["src_xras"] = [ - float(x) - for x in re.split( - " +", re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - ] - elif re.match("yras", lta[i]) is not None: - d["src_yras"] = [ - float(x) - for x in re.split( - " +", re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - ] - elif re.match("zras", lta[i]) is not None: - d["src_zras"] = [ - float(x) - for x in re.split( - " +", re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - ] - elif re.match("cras", lta[i]) is not None: - d["src_cras"] = [ - float(x) - for x in re.split( - " +", re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - ] - i += 1 - elif re.match("dst volume info", lta[i]) is not None: - while i < len(lta) and re.match("src volume info", lta[i]) is None: - if re.match("valid", lta[i]) is not None: - d["dst_valid"] = int( - re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - elif re.match("filename", lta[i]) is not None: - d["dst_filename"] = re.split( - " +", re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - elif re.match("volume", lta[i]) is not None: - d["dst_volume"] = [ - int(x) - for x in re.split( - " +", re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - ] - elif re.match("voxelsize", lta[i]) is not None: - d["dst_voxelsize"] = [ - float(x) - for x in re.split( - " +", re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - ] - elif re.match("xras", lta[i]) is not None: - d["dst_xras"] = [ - float(x) - for x in re.split( - " +", re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - ] - elif re.match("yras", lta[i]) is not None: - d["dst_yras"] = [ - float(x) - for x in re.split( - " +", re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - ] - elif re.match("zras", lta[i]) is not None: - d["dst_zras"] = [ - float(x) - for x in re.split( - " +", re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - ] - elif re.match("cras", lta[i]) is not None: - d["dst_cras"] = [ - float(x) - for x in re.split( - " +", re.sub(".*=", "", re.sub("#.*", "", lta[i])).strip() - ) - ] - i += 1 - else: - i += 1 - # create full transformation matrices - d["src"] = np.concatenate( - ( - np.concatenate( - ( - np.c_[d["src_xras"]], - np.c_[d["src_yras"]], - np.c_[d["src_zras"]], - np.c_[d["src_cras"]], - ), - axis=1, - ), - np.array([0.0, 0.0, 0.0, 1.0], ndmin=2), - ), - axis=0, - ) - d["dst"] = np.concatenate( - ( - np.concatenate( - ( - np.c_[d["dst_xras"]], - np.c_[d["dst_yras"]], - np.c_[d["dst_zras"]], - np.c_[d["dst_cras"]], - ), - axis=1, - ), - np.array([0.0, 0.0, 0.0, 1.0], ndmin=2), - ), - axis=0, - ) - # return - return d + lines = f.readlines() + + items = [] + shape_lines = [] + matrix_lines = [] + section = "" + for i, line in enumerate(lines): + if line.strip() == "": + continue + if hits := parameter_pattern.match(line): + name = hits.group(1) + if section and name in vol_info_par: + items.append((f"{section}_{name}", vol_info_par[name](hits.group(2)))) + elif name in parameters: + section = "" + items.append((name, parameters[name](hits.group(2)))) + else: + raise NotImplementedError(f"Unrecognized type string in lta-file " + f"{file}:{i+1}: '{name}'") + elif hits := vol_info_pattern.match(line): + section = hits.group(1) + # not a parameter line + elif shape_pattern.search(line): + shape_lines.append(np.fromstring(line, dtype=int, count=-1, sep=" ")) + elif matrix_pattern.search(line): + matrix_lines.append(np.fromstring(line, dtype=float, count=-1, sep=" ")) + + shape_lines = list(map(tuple, shape_lines)) + lta = dict(items) + if lta["nxforms"] != len(shape_lines): + raise IOError("Inconsistent lta format: nxforms inconsistent with shapes.") + if len(shape_lines) > 1 and np.any(np.not_equal([shape_lines[0]], shape_lines[1:])): + raise IOError(f"Inconsistent lta format: shapes inconsistent {shape_lines}") + lta_matrix = np.asarray(matrix_lines).reshape((-1,) + shape_lines[0].shape) + lta["lta"] = lta_matrix + return lta def load_talairach_coordinates(tala_path, img_shape, vox2ras): - tala_lta = readLTA(tala_path) + tala_lta = read_lta(tala_path) # create image grid p x, y, z = np.meshgrid( np.arange(img_shape[0]), @@ -567,7 +436,7 @@ def load_talairach_coordinates(tala_path, img_shape, vox2ras): p1 = np.concatenate((p, np.ones((p.shape[0], 1))), axis=1) assert tala_lta["type"] == 1, "talairach not in ras2ras" # ras2ras - m = np.matmul(tala_lta["lta"], vox2ras) + m = np.matmul(tala_lta["lta"][0, 0], vox2ras) tala_coordinates = np.matmul(m, p1.transpose()).transpose() tala_coordinates = tala_coordinates[:, :-1] diff --git a/CerebNet/inference.py b/CerebNet/inference.py index 9dcc9c7b..c9767911 100644 --- a/CerebNet/inference.py +++ b/CerebNet/inference.py @@ -291,6 +291,7 @@ def _get_ids_startswith(_label_map: Dict[int, str], prefix: str) -> List[int]: table = pv_calc( seg_data, norm_data, + norm_data, list(filter(lambda l: l != 0, label_map.keys())), vox_vol=vox_vol, threads=self.threads, diff --git a/FastSurferCNN/data_loader/data_utils.py b/FastSurferCNN/data_loader/data_utils.py index b5e9abd0..9f98aa53 100644 --- a/FastSurferCNN/data_loader/data_utils.py +++ b/FastSurferCNN/data_loader/data_utils.py @@ -584,29 +584,60 @@ def deep_sulci_and_wm_strand_mask( # Label mapping functions (to aparc (eval) and to label (train)) -def read_classes_from_lut(lut_file: Path | str) -> pd.DataFrame: +def read_classes_from_lut(lut_file: str | Path): """ - Read in FreeSurfer-like LUT table. + Modify from datautils to allow support for FreeSurfer-distributed ColorLUTs. + + Read in **FreeSurfer-like** LUT table. Parameters ---------- lut_file : Path, str - Path and name of FreeSurfer-style LUT file with classes of interest. + The path and name of FreeSurfer-style LUT file with classes of interest. Example entry: ID LabelName R G B A 0 Unknown 0 0 0 0 - 1 Left-Cerebral-Exterior 70 130 180 0. + 1 Left-Cerebral-Exterior 70 130 180 0 + ... Returns ------- - pd.Dataframe + pandas.DataFrame DataFrame with ids present, name of ids, color for plotting. """ - if isinstance(lut_file, str): + if not isinstance(lut_file, Path): lut_file = Path(lut_file) + if lut_file.suffix == ".tsv": + return pd.read_csv(lut_file, sep="\t") + # Read in file - separator = {".tsv": "\t", ".csv": ",", ".txt": " "} - return pd.read_csv(lut_file, sep=separator[lut_file.suffix]) + names = { + "ID": "int", + "LabelName": "str", + "Red": "int", + "Green": "int", + "Blue": "int", + "Alpha": "int", + } + kwargs = {} + if lut_file.suffix == ".csv": + kwargs["sep"] = "," + elif lut_file.suffix == ".txt": + kwargs["delim_whitespace"] = True + else: + raise RuntimeError( + f"Unknown LUT file extension {lut_file}, must be csv, txt or tsv." + ) + return pd.read_csv( + lut_file, + index_col=False, + skip_blank_lines=True, + comment="#", + header=None, + names=list(names.keys()), + dtype=names, + **kwargs, + ) def map_label2aparc_aseg( diff --git a/FastSurferCNN/mri_brainvol_stats.py b/FastSurferCNN/mri_brainvol_stats.py new file mode 100644 index 00000000..e90b3081 --- /dev/null +++ b/FastSurferCNN/mri_brainvol_stats.py @@ -0,0 +1,145 @@ +#!/bin/python + +# Copyright 2024 Image Analysis Lab, German Center for Neurodegenerative Diseases +# (DZNE), Bonn +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# IMPORTS +import argparse +from os import environ as env +from pathlib import Path + +from FastSurferCNN.segstats import HelpFormatter, main, VERSION +from FastSurferCNN.mri_segstats import print_and_exit + +DEFAULT_MEASURES_STRINGS = [ + (False, "BrainSeg"), + (False, "BrainSegNotVent"), + (False, "SupraTentorial"), + (False, "SupraTentorialNotVent"), + (False, "SubCortGray"), + (False, "lhCortex"), + (False, "rhCortex"), + (False, "Cortex"), + (False, "TotalGray"), + (False, "lhCerebralWhiteMatter"), + (False, "rhCerebralWhiteMatter"), + (False, "CerebralWhiteMatter"), + (False, "Mask"), + (False, "SupraTentorialNotVentVox"), + (False, "BrainSegNotVentSurf"), + (False, "VentricleChoroidVol"), +] +DEFAULT_MEASURES = list((False, m) for m in DEFAULT_MEASURES_STRINGS) + +USAGE = "python mri_brainvol_stats.py -s " +HELPTEXT = f""" +Dependencies: + + Python 3.10 + + Numpy + http://www.numpy.org + + Nibabel to read images + http://nipy.org/nibabel/ + + Pandas to read/write stats files etc. + https://pandas.pydata.org/ + +Original Author: David Kügler +Date: Jan-23-2024 + +Revision: {VERSION} +""" +DESCRIPTION = """ +Translates mri_brainvol_stats options for segstats.py. Options not listed here have no +equivalent representation in segstats.py. """ + + +def make_arguments() -> argparse.ArgumentParser: + """Make the argument parser.""" + parser = argparse.ArgumentParser( + usage=USAGE, + epilog=HELPTEXT.replace("\n", "
"), + description=DESCRIPTION, + formatter_class=HelpFormatter, + ) + parser.add_argument( + "--print", + action="append_const", + dest="parse_actions", + default=[], + const=print_and_exit, + help="Print the equivalent native segstats.py options and exit.", + ) + default_sd = Path(env["SUBJECTS_DIR"]) if "SUBJECTS_DIR" in env else None + parser.add_argument( + "--sd", + dest="out_dir", metavar="subjects_dir", type=Path, + default=default_sd, + required=not bool(default_sd), + help="set SUBJECTS_DIR, defaults to environment SUBJECTS_DIR, required to find " + "several files used by measures, e.g. surfaces.") + parser.add_argument( + "-s", + "--subject", + "--sid", + dest="sid", metavar="subject_id", + help="set subject_id, required to find several files used by measures, e.g. " + "surfaces.") + parser.add_argument( + "-o", + "--segstatsfile", + dest="segstatsfile", + default=Path("stats/brainvol.stats"), + help="Where to save the brainvol.stats, if relative path, this will be " + "relative to the subject directory." + ) + fs_home = "FREESURFER_HOME" + default_lut = Path(env[fs_home]) / "ASegStatsLUT.txt" if fs_home in env else None + parser.set_defaults( + segfile=Path("mri/aseg.mgz"), + measures=DEFAULT_MEASURES, + lut=default_lut, + measure_only=True, + ) + advanced = parser.add_argument_group( + "FastSurfer options (no equivalence with FreeSurfer's mri_brainvol_stats)", + ) + advanced.add_argument( + "--no_legacy", + action="store_false", + dest="legacy_freesurfer", + help="use FastSurfer algorithms instead of FastSurfer.", + ) + advanced.add_argument( + "--pvfile", + "-pv", + type=Path, + dest="pvfile", + help="Path to image used to compute the partial volume effects. This file is " + "only used in the FastSurfer algoritms (--no_legacy).", + ) + return parser + + +if __name__ == "__main__": + import sys + + args = make_arguments().parse_args() + parse_actions = getattr(args, "parse_actions", []) + for parse_action in parse_actions: + parse_action(args) + sys.exit(main(args)) diff --git a/FastSurferCNN/mri_segstats.py b/FastSurferCNN/mri_segstats.py new file mode 100644 index 00000000..3224064d --- /dev/null +++ b/FastSurferCNN/mri_segstats.py @@ -0,0 +1,540 @@ +#!/bin/python + +# Copyright 2024 Image Analysis Lab, German Center for Neurodegenerative Diseases +# (DZNE), Bonn +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# IMPORTS +import argparse +from itertools import pairwise, chain +from pathlib import Path +from typing import TypeVar, Sequence, Any, Iterable + +from FastSurferCNN.segstats import ( + main, + HelpFormatter, + add_two_help_messages, + VERSION, + empty, +) + +_T = TypeVar("_T") + + +USAGE = "python mri_segstats.py --seg segvol [optional arguments]" +HELPTEXT = f""" +Dependencies: + + Python 3.10 + + Numpy + http://www.numpy.org + + Nibabel to read images + http://nipy.org/nibabel/ + + Pandas to read/write stats files etc. + https://pandas.pydata.org/ + +Original Author: David Kügler +Date: Jan-04-2024 + +Revision: {VERSION} +""" +DESCRIPTION = """ +Translates mri_segstats options for segstats.py. Options not listed here have no +equivalent representation in segstats.py.
+IMPORTANT NOTES +mri_segstats uses a legacy version for the computation of measures (from FreeSurfer 6). +But mri_segstats.py implements the behavior if first mri_brainvol_stats +and then mri_segstats is run (which uses the stats/brainvol.stats generated by +mri_brainvol_stats). This reflects the output of stats files as created by FreeSurfer's +recon-all. +""" +ETIV_RATIO_KEY = "eTIV-ratios" +ETIV_RATIOS = {"BrainSegVol-to-eTIV": "BrainSeg", "MaskVol-to-eTIV": "Mask"} +ETIV_FROM_TAL = "EstimatedTotalIntraCranialVol" + + +class _ExtendConstAction(argparse.Action): + """Helper class to allow action='extend_const' by action=_ExtendConstAction.""" + def __init__( + self, + option_strings: Sequence[str], + dest: str, + const: _T | None = None, + default: _T | str | None = None, + required: bool = False, + help: str | None = None, + metavar: str | tuple[str, ...] | None = None, + ) -> None: + super(_ExtendConstAction, self).__init__( + option_strings=option_strings, + dest=dest, + nargs=0, + const=const, + default=default, + required=required, + help=help, + metavar=metavar, + ) + + def __call__( + self, + parser: argparse.ArgumentParser, + namespace: argparse.Namespace, + values: str | Sequence[Any], + option_string: str | None = None, + ) -> None: + """ + Extend attribute `self.dest` of `namespace` with the values in `self.const`. + """ + items = getattr(namespace, self.dest, None) + if items is None: + items = [] + elif type(items) is list: + items = items[:] + else: + import copy + items = copy.copy(items) + items.extend(self.const) + setattr(namespace, self.dest, items) + + +def make_arguments() -> argparse.ArgumentParser: + """Create an argument parser object with all parameters of the script.""" + parser = argparse.ArgumentParser( + usage=USAGE, + epilog=HELPTEXT.replace("\n", "
"), + description=DESCRIPTION, + formatter_class=HelpFormatter, + add_help=False, + ) + + def add_etiv_measures(args: argparse.Namespace) -> None: + measures: list[tuple[bool, str]] = getattr(args, "measures", []) + measure_strings = list(map(lambda x: x[1], measures)) + if all(m in measure_strings for m in (ETIV_RATIO_KEY, ETIV_FROM_TAL)): + + measures = [m for m in measures if m[1] == ETIV_RATIO_KEY] + for k, v in ETIV_RATIOS.items(): + for is_imported, m in measures: + if m == v or m.startswith(v + "("): + measures.append((False, k)) + continue + setattr(args, "measures", measures) + + def _update_what_to_import(args: argparse.Namespace) -> argparse.Namespace: + """ + Update the Namespace object based on the existence of the brainvol.stats file. + """ + cachefile = Path(args.measurefile) + if not cachefile.is_absolute(): + cachefile = args.out_dir / args.sid / cachefile + if not args.explicit_no_cached and cachefile.is_file(): + from FastSurferCNN.utils.brainvolstats import read_measure_file + + measure_data = read_measure_file(cachefile) + + def update_key(measure: tuple[bool, str]) -> tuple[bool, str]: + is_imported, measure_string = measure + measure_key, _, _ = measure_string.partition("(") + return measure_key in measure_data.keys(), measure_string + + # for each measure to be computed, replace it with the value in + # brainvol.stats, if available + args.measures = list(map(update_key, getattr(args, "measures", []))) + return args + + parser.set_defaults( + measurefile="stats/brainvol.stats", + parse_actions=[(1, add_etiv_measures), (10, _update_what_to_import)], + ) + + if "--help" in sys.argv: + from FastSurferCNN.utils.brainvolstats import Manager + manager = Manager([]) + + def help_text(keys: Iterable[str]) -> Iterable[str]: + return (manager[k].help() for k in keys) + else: + help_text = None + + def help_add_measures(message: str, keys: list[str]) -> str: + if help_text: + _keys = (k.split(' ')[0] for k in keys) + keys = [f"{k}: {text}" for k, text in zip(keys, help_text(_keys))] + return "
- ".join([message] + list(keys)) + + add_two_help_messages(parser) + parser.add_argument( + "--print", + action="append_const", + dest="parse_actions", + const=(0, print_and_exit), + help="Print the equivalent native segstats.py options and exit.", + ) + parser.add_argument( + "--version", + action="version", + version=f"%(prog)s {VERSION}", + help="Print the version of the mri_segstats.py script", + ) + parser.add_argument( + "--seg", + type=Path, + metavar="segvol", + dest="segfile", + help="Specify the segmentation file.", + ) + # --annot subject hemi parc + # --surf whitesurfname -- used with annot + # --slabel subject hemi full_path_to_label + # --label-thresh threshold + # --seg-from-input + parser.add_argument( + "--o", + "--sum", + type=Path, + metavar="file", + dest="segstatsfile", + help="Specifiy the output summary statistics file.", + ) + parser.add_argument( + "--pv", + type=Path, + metavar="pvvol", + dest="pvfile", + help="file to compensate for partial volume effects.", + ) + parser.add_argument( + "--i", + "--in", + type=Path, + metavar="invol", + dest="normfile", + help="file to compute intensity values.", + ) + + # --seg-erode Nerodes + # --frame frame + def _percent(__value) -> float: + return float(__value) / 50 + + parser.add_argument( + "--robust", + type=_percent, + metavar="percent", + dest="robust", + help="Compute stats after excluding percent from high and and low values, e.g. " + "with --robust 2, min and max are the 2nd and the 98th percentiles.", + ) + + def _add_invol_op(*flags: str, op: str, metavar: str | None = None) -> None: + if metavar: + def _optype(_a) -> str: + # test the argtype for float as well + return f"{flags[0].lstrip('-')}={float(_a)}" + kwargs = { + "action": "append", + "type": _optype, + "dest": "pvfile_preproc", + "help": f"Apply the {op} with `{metavar}` to `invol` (--in)", + } + else: + kwargs = { + "action": "append_const", + "const": flags[0].lstrip("-"), + "dest": "pvfile_preproc", + "help": f"Apply {op} to `invol` (--in)", + } + parser.add_argument(*flags, **kwargs) + + def _no_import(*args: str) -> list[tuple[bool, str]]: + return list((False, a) for a in args) + + _add_invol_op("--sqr", op="squaring") + _add_invol_op("--sqrt", op="the square root") + _add_invol_op("--mul", op="multiplication", metavar="val") + _add_invol_op("--div", op="division", metavar="val") + # --snr + _add_invol_op("--abs", op="absolute value") + # --accumulate + parser.add_argument( + "--ctab", + type=Path, + metavar="ctabfile", + dest="lut", + help="load the Color Lookup Table.", + ) + import os + env = os.environ + if "FREESURFER_HOME" in env: + default_lut = Path(env["FREESURFER_HOME"]) / "FreeSurferColorLUT.txt" + elif "FASTSURFER_HOME" in env: + default_lut = ( + Path(env["FASTSURFER_HOME"]) / "FastSurferCNN/config/FreeSurferColorLUT.txt" + ) + else: + default_lut = None + parser.add_argument( + "--ctab-default", + metavar="ctabfile", + dest="lut", + const=default_lut, + action="store_const", + help="load default Color Lookup Table (from FREESURFER_HOME or " + "FASTSURFER_HOME).", + ) + # --ctab-gca gcafile + parser.add_argument( + "--id", + type=int, + nargs="+", + metavar="segid", + action="extend", + dest="ids", + default=[], + help="Specify segmentation Exclude segmentation ids from report.", + ) + parser.add_argument( + "--excludeid", + type=int, + nargs="+", + metavar="segid", + dest="excludeid", + help="Exclude segmentation ids from report.", + ) + parser.add_argument( + "--no-cached", + action="store_true", + dest="explicit_no_cached", + help="Do not try to load stats/brainvol.stats.", + ) + parser.add_argument( + "--excl-ctxgmwm", + dest="excludeid", + action=_ExtendConstAction, + const=[2, 3, 41, 42], + help="Exclude cortical gray and white matter regions from volume stats.", + ) + surf_wm = ["rhCerebralWhiteMatter", "lhCerebralWhiteMatter", "CerebralWhiteMatter"] + parser.add_argument( + "--surf-wm-vol", + action=_ExtendConstAction, + dest="measures", + const=_no_import(*surf_wm), + help=help_add_measures( + "Compute cortical white matter based on the surface:", + surf_wm, + ), + ) + surf_ctx = ["rhCortex", "lhCortex", "Cortex"] + parser.add_argument( + "--surf-ctx-vol", + action=_ExtendConstAction, + dest="measures", + const=_no_import(*surf_ctx), + help=help_add_measures( + "compute cortical gray matter based on the surface:", + surf_ctx, + ), + ) + parser.add_argument( + "--no_global_stats", + action="store_const", + dest="measures", + const=[], + help="Resets the computed global stats.", + ) + parser.add_argument( + "--empty", + action="store_true", + dest="empty", + help="Report all segmentation labels in ctab, even if they are not in seg.", + ) + # --ctab-out ctaboutput + # --mask maskvol + # --maskthresh thresh + # --masksign sign + # --maskframe frame + # --maskinvert + # --maskerode nerode + brainseg = ["BrainSeg", "BrainSegNotVent"] + parser.add_argument( + "--brain-vol-from-seg", + action=_ExtendConstAction, + dest="measures", + const=_no_import(*brainseg), + help=help_add_measures("Compute measures BrainSeg measures:", brainseg), + ) + + def _mask(__value): + return False, "Mask(" + str(__value) + ")" + + parser.add_argument( + "--brainmask", + type=_mask, + metavar="brainmask", + action="append", + dest="measures", + help="Report the Volume of the brainmask", + ) + supratent = ["SupraTentorial", "SupraTentorialNotVent"] + parser.add_argument( + "--supratent", + action=_ExtendConstAction, + dest="measures", + const=_no_import(*supratent), + help=help_add_measures("Compute supratentorial measures:", supratent), + ) + parser.add_argument( + "--subcortgray", + action="append_const", + dest="measures", + const=(False, "SubCortGray"), + help=help_add_measures("Compute measure SubCortGray:", ["SubCortGray"]), + ) + parser.add_argument( + "--totalgray", + action="append_const", + dest="measures", + const=(False, "TotalGray"), + help=help_add_measures("Compute measure TotalGray:", ["TotalGray"]), + ) + etiv_measures = [f"{k} (if also --brain-vol-from-seg)" for k in ETIV_RATIOS] + parser.add_argument( + "--etiv", + action=_ExtendConstAction, + dest="measures", + const=_no_import(ETIV_FROM_TAL, ETIV_RATIO_KEY), + help=help_add_measures("Compute eTIV:", [ETIV_FROM_TAL] + etiv_measures), + ) + # --etiv-only + # --old-etiv-only + # --xfm2etiv xfm outfile + surf_holes = ["rhSurfaceHoles", "lhSurfaceHoles", "SurfaceHoles"] + parser.add_argument( + "--euler", + action=_ExtendConstAction, + dest="measures", + const=_no_import(*surf_holes), + help=help_add_measures("Compute surface holes measures:", surf_holes), + ) + # --avgwf textfile + # --sumwf testfile + # --avgwfvol mrivol + # --avgwf-remove-mean + # --sfavg textfile + # --vox C R S + # --replace ID1 ID2 + # --replace-file file + # --gtm-default-seg-merge + # --gtm-default-seg-merge-choroid + # --ga-stats subject statsfile + default_sd = Path(env["SUBJECTS_DIR"]) if "SUBJECTS_DIR" in env else None + parser.add_argument( + "--sd", + dest="out_dir", metavar="subjects_dir", type=Path, + default=default_sd, + help="set SUBJECTS_DIR, defaults to environment SUBJECTS_DIR, required to find " + "several files used by measures, e.g. surfaces.") + parser.add_argument( + "--subject", + dest="sid", metavar="subject_id", + help="set subject_id, required to find several files used by measures, e.g. " + "surfaces.") + parser.add_argument( + "--seed", + nargs=1, metavar="N", help="The seed has no effect") + parser.add_argument( + "--in-intensity-name", + type=str, + dest="norm_name", + default="", + help="name of the intensity image" + ) + parser.add_argument( + "--in-intensity-units", + type=str, + dest="norm_unit", + default="", + help="unit of the intensity image" + ) + parser.add_argument( + "--no_legacy", + action="store_false", + dest="legacy_freesurfer", + help="use fastsurfer algorithms instead of fastsurfer." + ) + return parser + + +def print_and_exit(args: object): + """Print the commandline arguments of the segstats script to stdout and exit.""" + print(" ".join(format_cmdline_args(args))) + import sys + sys.exit(0) + + +def format_cmdline_args(args: object) -> list[str]: + """Format the commandline arguments of the segstats script.""" + arglist = ["python", str(Path(__file__).parent / "segstats.py")] + if getattr(args, "allow_root", False): + arglist.append("--allow_root") + if getattr(args, "legacy_freesurfer", False): + arglist.append("--legacy_freesurfer") + if (segfile := getattr(args, "segfile", None)) is not None: + arglist.extend(["--segfile", str(segfile)]) + if (normfile := getattr(args, "normfile", None)) is not None: + arglist.extend(["--normfile", str(normfile)]) + if (pvfile := getattr(args, "pvfile", None)) is not None: + arglist.extend(["--pvfile", str(pvfile)]) + if (segstatsfile := getattr(args, "segstatsfile", None)) is not None: + arglist.extend(["--segstatsfile", str(segstatsfile)]) + if (subjects_dir := getattr(args, "out_dir", None)) is not None: + arglist.extend(["--sd", str(subjects_dir)]) + if (subject_id := getattr(args, "sid", None)) is not None: + arglist.extend(["--sid", str(subject_id)]) + if (threads := getattr(args, "threads", 0)) > 0: + arglist.extend(["--threads", str(threads)]) + if (lut := getattr(args, "lut", None)) is not None: + arglist.extend(["--lut", str(lut)]) + if not empty(__ids := getattr(args, "ids", [])): + arglist.extend(["--id"] + list(map(str, __ids))) + + measures: list[tuple[bool, str]] = getattr(args, "measures", []) + if not empty(measures): + arglist.append("measures") + if (measurefile := getattr(args, "measurefile", None)) is not None: + arglist.extend(("--file", str(measurefile))) + _flag = {True: "--import", False: "--compute"} + blank_measure = (not measures[0][0], "") + flag_measure_iter = ((_flag[i], m) for i, m in [blank_measure, *measures]) + arglist.extend(chain( + (*((flag,) if flag != last_flag else ()), str(measure)) + for (last_flag, _), (flag, measure) in pairwise(flag_measure_iter) + )) + + return arglist + + +if __name__ == "__main__": + import sys + + args = make_arguments().parse_args() + parse_actions = getattr(args, "parse_actions", []) + for i, parse_action in sorted(parse_actions, key=lambda x: x[0], reverse=True): + parse_action(args) + sys.exit(main(args)) diff --git a/FastSurferCNN/run_prediction.py b/FastSurferCNN/run_prediction.py index 5a5eba22..e6799ec6 100644 --- a/FastSurferCNN/run_prediction.py +++ b/FastSurferCNN/run_prediction.py @@ -20,7 +20,7 @@ See Also -------- -:ref:`/scripts/fastsurfercnn.rst` +:doc:`/scripts/fastsurfercnn` `run_prediction.py --help` """ diff --git a/FastSurferCNN/segstats.py b/FastSurferCNN/segstats.py index ba467de2..861d70f5 100644 --- a/FastSurferCNN/segstats.py +++ b/FastSurferCNN/segstats.py @@ -1,3 +1,5 @@ +#!/bin/python + # Copyright 2022 Image Analysis Lab, German Center for Neurodegenerative Diseases(DZNE), Bonn # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -21,37 +23,45 @@ from numbers import Number from pathlib import Path from typing import ( + Any, Callable, - Dict, + cast, Iterable, - List, + IO, + Literal, Optional, + overload, Sequence, - Tuple, + Sized, + Type, + TypedDict, TypeVar, - Union, - cast, - overload, + Container, + Iterator, ) +from concurrent.futures import Executor, ThreadPoolExecutor + -import nibabel as nib import numpy as np import pandas as pd from numpy import typing as npt from FastSurferCNN.utils.arg_types import float_gt_zero_and_le_one as robust_threshold from FastSurferCNN.utils.arg_types import int_ge_zero as id_type -from FastSurferCNN.utils.arg_types import int_gt_zero as patch_size +from FastSurferCNN.utils.arg_types import int_gt_zero as patch_size_type from FastSurferCNN.utils.parser_defaults import add_arguments from FastSurferCNN.utils.threads import get_num_threads -USAGE = "python seg_stats.py -norm -i -o [optional arguments]" -DESCRIPTION = "Script to calculate partial volumes and other segmentation statistics of a segmentation file." - -HELPTEXT = """ +# Constants +USAGE = ("python segstats.py (-norm|-pv) -i " + "-o [optional arguments] [{measures,mri_segstats} ...]") +DESCRIPTION = ("Script to calculate partial volumes and other segmentation statistics " + "of a segmentation file.") +VERSION = "1.1" +HELPTEXT = f""" Dependencies: - Python 3.8+ + Python 3.10 Numpy http://www.numpy.org @@ -64,50 +74,45 @@ Original Author: David Kügler Date: Dec-30-2022 -Modified: May-08-2023 +Modified: Dec-07-2023 + +Revision: {VERSION} """ +FILTER_SIZES = (3, 15) +COLUMNS = ["Index", "SegId", "NVoxels", "Volume_mm3", "StructName", "Mean", "StdDev", + "Min", "Max", "Range"] +# Type definitions _NumberType = TypeVar("_NumberType", bound=Number) _IntType = TypeVar("_IntType", bound=np.integer) _DType = TypeVar("_DType", bound=np.dtype) _ArrayType = TypeVar("_ArrayType", bound=np.ndarray) -PVStats = Dict[str, Union[int, float]] -VirtualLabel = Dict[int, Sequence[int]] +SlicingTuple = tuple[slice, ...] +SlicingSequence = Sequence[slice] +VirtualLabel = dict[int, Sequence[int]] +_GlobalStats = tuple[int, int, Optional[_NumberType], Optional[_NumberType], + Optional[float], Optional[float], float, npt.NDArray[bool]] +SubparserCallback = Type[argparse.ArgumentParser.add_subparsers] -FILTER_SIZES = (3, 15) -UNITS = { - "Volume_mm3": "mm^3", - "normMean": "MR", - "normStdDev": "MR", - "normMin": "MR", - "normMax": "MR", - "normRange": "MR", -} -FIELDS = { - "Index": "Index", - "SegId": "Segmentation Id", - "NVoxels": "Number of Voxels", - "Volume_mm3": "Volume", - "StructName": "Structure Name", - "normMean": "Intensity normMean", - "normStdDev": "Intensity normStdDev", - "normMin": "Intensity normMin", - "normMax": "Intensity normMax", - "normRange": "Intensity normRange", -} -FORMATS = { - "Index": "d", - "SegId": "d", - "NVoxels": "d", - "Volume_mm3": ".3f", - "StructName": "s", - "normMean": ".4f", - "normStdDev": ".4f", - "normMin": ".4f", - "normMax": ".4f", - "normRange": ".4f", -} +class _RequiredPVStats(TypedDict): + SegId: int + NVoxels: int + Volume_mm3: float + + +class _OptionalPVStats(TypedDict, total=False): + StructName: str + Mean: float + StdDev: float + Min: float + Max: float + Range: float + + +class PVStats(_RequiredPVStats, _OptionalPVStats): + """Dictionary of volume statistics for partial volume evaluation and global stats""" + pass class HelpFormatter(argparse.HelpFormatter): @@ -119,58 +124,84 @@ def _linebreak_sub(self): """ Get the linebreak substitution string. - Returns: - str: The linebreak substitution string ("
"). + Returns + ------- + str + The linebreak substitution string ("
"). """ return getattr(self, "linebreak_sub", "
") - def _fill_text(self, text, width, indent): + def _item_symbol(self): + return getattr(self, "item_symbol", "- ") + + def _fill_text(self, text: str, width: int, indent: str) -> str: """ Fill text with line breaks based on the linebreak substitution string. - Args: - text (str): The input text. - width (int): The width for filling the text. - indent (int): The indentation level. + Parameters + ---------- + text : str + The input text. + width : int + The width for filling the text. + indent : int + The indentation level. - Returns: - str: The formatted text with line breaks. + Returns + ------- + str + The formatted text with line breaks. """ + cond_len, texts = self._itemized_lines(text) + lines = (super(HelpFormatter, self)._fill_text(t[p:], width, indent + " " * p) + for t, (c, p) in zip(texts, cond_len)) + return "\n".join("- " + t[p:] if c else t for t, (c, p) in zip(lines, cond_len)) + + def _itemized_lines(self, text): texts = text.split(self._linebreak_sub()) - return "\n".join( - [super(HelpFormatter, self)._fill_text(tex, width, indent) for tex in texts] - ) + item = self._item_symbol() + il = len(item) + cond_len = [(c, il if c else 0) for c in map(lambda t: t[:il] == item, texts)] + texts = [t[p:] for t, (c, p) in zip(texts, cond_len)] + return cond_len, texts - def _split_lines(self, text: str, width: int): + def _split_lines(self, text: str, width: int) -> list[str]: """ Split lines in the text based on the linebreak substitution string. - Args: - text (str): The input text. - width (int): The width for splitting lines. + Parameters + ---------- + text : str + The input text. + width : int + The width for splitting lines. - Returns: - list: The list of lines. + Returns + ------- + list[str] + The list of lines. """ - texts = text.split(self._linebreak_sub()) - from itertools import chain + def indent_list(items: list[str]) -> list[str]: + return ["- " + items[0]] + [" " + l for l in items[1:]] - return list( - chain.from_iterable( - super(HelpFormatter, self)._split_lines(tex, width) for tex in texts - ) - ) + cond_len, texts = self._itemized_lines(text) + from itertools import chain + lines = (super(HelpFormatter, self)._split_lines(tex, width - p) + for tex, (c, p) in zip(texts, cond_len)) + lines = ((indent_list(lst) if c[0] else lst) for lst, c in zip(lines, cond_len)) + return list(chain.from_iterable(lines)) def make_arguments(helpformatter: bool = False) -> argparse.ArgumentParser: """ - Create and configure the argparse.ArgumentParser. + Create an argument parser object with all parameters of the script. Returns ------- argparse.ArgumentParser The configured argument parser. """ + import sys if helpformatter: kwargs = { "epilog": HELPTEXT.replace("\n", "
"), @@ -181,20 +212,33 @@ def make_arguments(helpformatter: bool = False) -> argparse.ArgumentParser: parser = argparse.ArgumentParser( usage=USAGE, description=DESCRIPTION, - **kwargs + add_help=False, + **kwargs, + ) + add_two_help_messages(parser) + parser.add_argument( + "--pvfile", + "-pv", + type=Path, + dest="pvfile", + help="Path to image used to compute the partial volume effects (default: the " + "file passed as normfile). This file is required, either directly or " + "indirectly via normfile.", ) parser.add_argument( "-norm", "--normfile", - type=str, - required=True, + type=Path, dest="normfile", - help="Biasfield-corrected image in the same image space as segmentation (required).", + help="Path to biasfield-corrected image (the same image space as " + "segmentation). This file is used to calculate intensity values. Also, if " + "no pvfile is defined, it is used as pvfile. One of normfile or pvfile is " + "required.", ) parser.add_argument( "-i", "--segfile", - type=str, + type=Path, dest="segfile", required=True, help="Segmentation file to read and use for evaluation (required).", @@ -202,7 +246,7 @@ def make_arguments(helpformatter: bool = False) -> argparse.ArgumentParser: parser.add_argument( "-o", "--segstatsfile", - type=str, + type=Path, required=True, dest="segstatsfile", help="Path to output segstats file.", @@ -212,16 +256,16 @@ def make_arguments(helpformatter: bool = False) -> argparse.ArgumentParser: "--excludeid", type=id_type, nargs="*", - default=[0], + default=[], help="List of segmentation ids (integers) to exclude in analysis, " - "e.g. `--excludeid 0 1 10` (default: 0).", + "e.g. `--excludeid 0 1 10` (default: None).", ) parser.add_argument( "--ids", type=id_type, nargs="*", help="List of exclusive segmentation ids (integers) to use " - "(default: all ids in --lut or all ids in image).", + "(default: all ids in --lut or all ids in image).", ) parser.add_argument( "--merged_label", @@ -230,9 +274,9 @@ def make_arguments(helpformatter: bool = False) -> argparse.ArgumentParser: dest="merged_labels", default=[], action="append", - help="Add a 'virtual' label (first value) that is the combination of all following values, " - "e.g. `--merged_label 100 3 4 8` will compute the statistics for label 100 by aggregating " - "labels 3, 4 and 8.", + help="Add a 'virtual' label (first value) that is the combination of all " + "following values, e.g. `--merged_label 100 3 4 8` will compute the " + "statistics for label 100 by aggregating labels 3, 4 and 8.", ) parser.add_argument( "--robust", @@ -240,23 +284,33 @@ def make_arguments(helpformatter: bool = False) -> argparse.ArgumentParser: dest="robust", default=None, help="Whether to calculate robust segmentation metrics. This parameter " - "expects the fraction of values to keep, e.g. `--robust 0.95` will " - "ignore the 2.5%% smallest and the 2.5%% largest values in the " - "segmentation when calculating the statistics (default: no robust " - "statistics == `--robust 1.0`).", + "expects the fraction of values to keep, e.g. `--robust 0.95` will " + "ignore the 2.5%% smallest and the 2.5%% largest values in the " + "segmentation when calculating the statistics (default: no robust " + "statistics == `--robust 1.0`).", + ) + parser.add_argument( + "--measure_only", + action="store_true", + dest="measure_only", + help="Only calculate the Measures in the header, no PV table." ) - advanced = parser.add_argument_group(title="Advanced options") + subparsers = parser.add_subparsers(title="Suboptions", dest="subparser") + add_measure_parser(subparsers.add_parser) + advanced = parser.add_argument_group(title="Advanced options (not shown in -h)") + if "-h" in sys.argv: + return parser advanced.add_argument( "--threads", dest="threads", default=get_num_threads(), type=int, help=f"Number of threads to use (defaults to number of hardware threads: " - f"{get_num_threads()})", + f"{get_num_threads()})", ) advanced.add_argument( "--patch_size", - type=patch_size, + type=patch_size_type, dest="patch_size", default=32, help="Patch size to use in calculating the partial volumes (default: 32).", @@ -265,60 +319,71 @@ def make_arguments(helpformatter: bool = False) -> argparse.ArgumentParser: "--empty", action="store_true", dest="empty", - help="Keep ids for the table that do not exist in the segmentation (default: drop).", + help="Keep ids for the table that do not exist in the segmentation " + "(default: drop).", + ) + advanced = add_arguments(advanced, ["device", "sid", "sd", "allow_root"]) + advanced.add_argument( + "--lut", + type=Path, + metavar="lut", + dest="lut", + help="Path and name of LUT to use.", ) - advanced = add_arguments(advanced, ["device", "lut", "sid", "in_dir", "allow_root"]) advanced.add_argument( "--legacy_freesurfer", action="store_true", dest="legacy_freesurfer", help="Reproduce FreeSurfer mri_segstats numbers (default: off). \n" - "Please note, that exact agreement of numbers cannot be guaranteed, because the " - "condition number of FreeSurfers algorithm (mri_segstats) combined with the fact that " - "mri_segstats uses 'float' to measure the partial volume corrected volume. This yields " - "differences of more than 60mm3 or 0.1%% in large structures. This uniquely impacts " - "highres images with more voxels (on the boundry) and smaller voxel sizes (volume per " - "voxel).", + "Please note, that exact agreement of numbers cannot be guaranteed, " + "because the condition number of FreeSurfers algorithm (mri_segstats) " + "combined with the fact that mri_segstats uses 'float' to measure the " + "partial volume corrected volume. This yields differences of more than " + "60mm3 or 0.1%% in large structures. This uniquely impacts highres images " + "with more voxels (on the boundary) and smaller voxel sizes (volume per " + "voxel).", ) # Additional info: - # Changing the data type in mri_segstats to double can reduce this difference to nearly zero. + # Changing the data type in mri_segstats to double can reduce this difference to + # nearly zero. # mri_segstats has two operations affecting a bad condition number: # 1. pv = (val - mean_nbr) / (mean_label - mean_nbr) # 2. volume += vox_vol * pv - # This is further affected by the small vox_vol (volume per voxel) of highres images (0.7iso -> 0.343) - # Their effects stack and can result in differences of more than 60mm3 or 0.1% in a comparison between double and - # single-precision evaluations. + # This is further affected by the small vox_vol (volume per voxel) of highres + # images (0.7iso -> 0.343) + # Their effects stack and can result in differences of more than 60mm3 or 0.1% in + # a comparison between double and single-precision evaluations. advanced.add_argument( "--mixing_coeff", - type=str, + type=Path, dest="mix_coeff", default="", help="Save the mixing coefficients (default: off).", ) advanced.add_argument( "--alternate_labels", - type=str, + type=Path, dest="nbr", default="", help="Save the alternate labels (default: off).", ) advanced.add_argument( "--alternate_mixing_coeff", - type=str, + type=Path, dest="nbr_mix_coeff", default="", help="Save the alternate labels' mixing coefficients (default: off).", ) advanced.add_argument( "--seg_means", - type=str, + type=Path, dest="seg_means", default="", help="Save the segmentation labels' means (default: off).", ) advanced.add_argument( "--alternate_means", - type=str, + type=Path, dest="nbr_means", default="", help="Save the alternate labels' means (default: off).", @@ -327,312 +392,825 @@ def make_arguments(helpformatter: bool = False) -> argparse.ArgumentParser: "--volume_precision", type=id_type, dest="volume_precision", - default=None, + default="3", help="Number of digits after dot in summary stats file (default: 3). Note, " - "--legacy_freesurfer sets this to 1.", + "--legacy_freesurfer sets this to 1.", + ) + advanced.add_argument( + "--norm_name", + type=str, + dest="norm_name", + default="norm", + help="Option to change the name of the in volume (default: norm)." + ) + advanced.add_argument( + "--norm_unit", + type=str, + dest="norm_unit", + default="MR", + help="Option to change the unit of the in volume (default: MR)." ) return parser -def loadfile_full(file: str, name: str) -> Tuple[nib.analyze.SpatialImage, np.ndarray]: +def empty(__arg: Any) -> bool: + """ + Checks if the argument is an empty list (or None). + """ + return __arg is None or (isinstance(__arg, Sized) and len(__arg) == 0) + + +def add_measure_parser(subparser_callback: SubparserCallback) -> None: + """ + Add a parser that supports adding measures to the parameters. + """ + measure_parser = subparser_callback( + "measures", + usage="python segstats.py (...) measures [optional arguments]", + argument_default="measures", + help="Configures options to measures", + description="Options to configure measures", + formatter_class=HelpFormatter, + add_help=False, + ) + add_two_help_messages(measure_parser) + + def __add_computed_measure(x: str) -> tuple[bool, str]: + return False, x + measure_parser.add_argument( + "--compute", + type=__add_computed_measure, + nargs="+", + action="extend", + default=[], + dest="measures", + help="Additional Measures to compute based on imported/computed measures:
" + "Cortex, CerebralWhiteMatter, SubCortGray, TotalGray, " + "BrainSegVol-to-eTIV, MaskVol-to-eTIV, SurfaceHoles, " + "EstimatedTotalIntraCranialVol", + ) + + def __add_imported_measure(x: str) -> tuple[bool, str]: + return True, x + measure_parser.add_argument( + '--import', + type=__add_imported_measure, + nargs="+", + action="extend", + default=[], + dest="measures", + help="Additional Measures to import from the measurefile.
" + "Example measures ('all' to import all measures in the measurefile):
" + "BrainSeg, BrainSegNotVent, SupraTentorial, SupraTentorialNotVent, " + "SubCortGray, lhCortex, rhCortex, Cortex, TotalGray, " + "lhCerebralWhiteMatter, rhCerebralWhiteMatter, CerebralWhiteMatter, Mask, " + "SupraTentorialNotVentVox, BrainSegNotVentSurf, VentricleChoroidVol, " + "BrainSegVol-to-eTIV, MaskVol-to-eTIV, lhSurfaceHoles, rhSurfaceHoles, " + "SurfaceHoles, EstimatedTotalIntraCranialVol
" + "Note, 'all' will always be overwritten by any explicitly mentioned " + "measures.", + ) + measure_parser.add_argument( + "--file", + type=Path, + dest="measurefile", + default="brainvol.stats", + help="Default file to read measures (--import ...) from. If the path is " + "relative, it is interpreted as relative to subjects_dir/subject_id from" + "--sd and --subject_id.", + ) + measure_parser.add_argument( + "--from_seg", + type=Path, + dest="aseg_replace", + default=None, + help="Replace the default segfile to compute measures from by -i/--segfile. " + "This will default to 'mri/aseg.mgz' for --legacy_freesurfer and to the " + "value of -i/--segfile otherwise." + ) + + +def add_two_help_messages(parser: argparse.ArgumentParser) -> None: """ - Load full image and data. + Adds separate help flags -h and --help to the parser for simple and detailed help. + Both trigger the help action. Parameters ---------- - file : str - Filename. - name : str - Subject name. + parser : argparse.ArgumentParser + Parser to add the flags to. + """ + def this_msg(msg: str, flag: str) -> str: + import sys + return f"{msg} (this message)" if flag in sys.argv else msg + parser.add_argument( + "-h", action="help", + help=this_msg("show a short help message and exit", "-h")) + parser.add_argument( + "--help", action="help", + help=this_msg("show a long, detailed help message and exit", "--help")) + + +def _check_arg_path( + __args: argparse.Namespace, + __attr: str, + subjects_dir: Path | None, + subject_id: str | None, + allow_subject_dir: bool = True, + require_exist: bool = True, +) -> Path: + """ + Check an argument that is supposed to be a Path object and finding the absolute + path, which can be derived from the subject_dir. + + Parameters + ---------- + __args : argparse.Namespace + The arguments object. + __attr: str + The name of the attribute in the Namespace object. + allow_subject_dir : bool, optional + Whether relative paths are supposed to be understood with respect to + subjects_dir / subject_id (default: True). + require_exist : bool, optional + Raise a ValueError, if the indicated file does not exist (default: True). Returns ------- - Tuple[nib.analyze.SpatialImage, np.ndarray] - A tuple containing the loaded image and its corresponding data. + Path + The resulting Path object. + + Raises + ------ + ValueError + If attribute does not exist, is not a Path (or convertible to a Path), or if + the file does not exist, but reuire_exist is True. """ - try: - img = nib.load(file) - except (IOError, FileNotFoundError) as e: - raise IOError( - f"Failed loading the {name} '{file}' with error: {e.args[0]}" - ) from e - data = np.asarray(img.dataobj) - return img, data + if (_attr_val := getattr(__args, __attr), None) is None: + raise ValueError(f"No {__attr} passed.") + if isinstance(_attr_val, str): + _attr_val = Path(_attr_val) + elif not isinstance(_attr_val, Path): + raise ValueError(f"{_attr_val} is not a Path object.") + if allow_subject_dir and not _attr_val.is_absolute(): + if isinstance(subjects_dir, Path) and subject_id is not None: + _attr_val = subjects_dir / subject_id / _attr_val + if require_exist and not _attr_val.exists(): + raise ValueError(f"Path {_attr_val} did not exist for {__attr}.") + return _attr_val + + +def _check_arg_defined(attr: str, /, args: argparse.Namespace) -> bool: + """ + Check whether the attribute attr is defined in args. + + Parameters + ---------- + attr: str + The name of the attribute. + args: argparse.Namespace + The argument container object. + + Returns + ------- + bool + Whether the argument is defined (not None, not an empty container/str). + """ + value = getattr(args, attr, None) + return not (value is None or empty(value)) + + +def check_shape_affine( + img1: "nib.analyze.SpatialImage", + img2: "nib.analyze.SpatialImage", + name1: str, + name2: str, +) -> None: + """ + Check whether the shape and affine of + + Parameters + ---------- + img1 : nibabel.SpatialImage + Image 1. + img2 : nibabel.SpatialImage + Image 2. + name1 : str + Name of image 1. + name2 : str + Name of image 2. + + Raises + ------ + RuntimeError + If shapes or affines are not the same. + """ + if img1.shape != img2.shape or not np.allclose(img1.affine, img2.affine): + raise RuntimeError( + f"The shapes or affines of the {name1} and the {name2} image are not " + f"similar, both must be the same!" + ) -def main(args): +def parse_files( + args: argparse.Namespace, + subjects_dir: Path | str | None = None, + subject_id: str | None = None, + require_measurefile: bool = False, + require_pvfile: bool = True, +) -> tuple[Path, Path | None, Path | None, Path, Path | None]: """ - Main function. + Parse and read paths of files. Parameters ---------- args : argparse.Namespace - Command-line arguments parsed using argparse. + Parameters object from make_arguments. + subjects_dir : Path, str, optional + Path to SUBJECTS_DIR, where subject directories are. + subject_id : str, optional + The subject_id string. + require_measurefile : bool, default=False + Require the measurefile to exist. + require_pvfile : bool, default=True + Require a pvfile or normfile to exist. Returns ------- - int - Exit code. Returns 0 upon successful execution. + segfile : Path + Path to the segmentation file, most likely an absolute path. + pvfile : Path, None + Path to the pvfile file, most likely an absolute path. + normfile : Path, None + Path to the norm file, most likely an absolute path, or None if not passed. + segstatsfile : Path + Path to the output segstats file, most likely an absolute path. + measurefile : Path, None + Path to the measure file, most likely an absolute path, not None is not passed. + + Raises + ------ + ValueError + If there is a necessary parameter missing or invalid. """ - import os - import time + if subjects_dir is not None: + subjects_dir = Path(subjects_dir) + check_arg_path = partial( + _check_arg_path, subjects_dir=subjects_dir, subject_id=subject_id + ) + segfile = check_arg_path(args, "segfile") + not_has_arg = partial(_check_arg_defined, args=args) + if not any(map(not_has_arg, ("normfile", "pvfile"))): + if require_pvfile: + raise ValueError("Either pvfile or normfile are required.") + pvfile = None + normfile = None + elif getattr(args, "normfile", None) is None: + pvfile = check_arg_path(args, "pvfile") + normfile = None + else: + normfile = check_arg_path(args, "normfile") + if getattr(args, "pvfile", None) is None: + pvfile = normfile + else: + pvfile = check_arg_path(args, "pvfile") - start = time.perf_counter_ns() - from FastSurferCNN.utils.common import assert_no_root + segstatsfile = check_arg_path(args, "segstatsfile", require_exist=False) + if not segstatsfile.is_absolute(): + raise ValueError("segstatsfile must be an absolute path!") - getattr(args, "allow_root", False) or assert_no_root() + if (measurefile := getattr(args, "measurefile", None)) is not None: + measurefile = check_arg_path( + args, + "measurefile", + require_exist=require_measurefile, + ) - if not hasattr(args, "segfile") or not os.path.exists(args.segfile): - return "No segfile was passed or it does not exist." - if not hasattr(args, "normfile") or not os.path.exists(args.normfile): - return "No normfile was passed or it does not exist." - if not hasattr(args, "segstatsfile"): - return "No segstats file was passed" + return segfile, pvfile, normfile, segstatsfile, measurefile - threads = args.threads - if threads <= 0: - threads = get_num_threads() - - from concurrent.futures import ThreadPoolExecutor - with ThreadPoolExecutor(threads) as tpe: - # load these files in different threads to avoid waiting on IO (not parallel due to GIL though) - seg_future = tpe.submit(loadfile_full, args.segfile, "segfile") - norm_future = tpe.submit(loadfile_full, args.normfile, "normfile") +def infer_labels_excludeid( + args: argparse.Namespace, + lut: "pd.DataFrame", + data: "npt.NDArray[int]", +) -> tuple["npt.NDArray[int]", list[int]]: + """ + Infer the labels and excluded ids from command line arguments, the lookup table, or + the segmentation image. - if hasattr(args, "lut") and args.lut is not None: - try: - lut = read_classes_from_lut(args.lut) - except FileNotFoundError as e: - return f"Could not find the ColorLUT in {args.lut}, please make sure the --lut argument is valid." - else: - lut = None - try: - ( - seg, - seg_data, - ) = ( - seg_future.result() - ) # type: nib.analyze.SpatialImage, Union[np.ndarray, torch.IntTensor] - ( - norm, - norm_data, - ) = ( - norm_future.result() - ) # type: nib.analyze.SpatialImage, Union[np.ndarray, torch.Tensor] - - if seg_data.shape != norm_data.shape or not np.allclose( - seg.affine, norm.affine - ): - return ( - "The shapes or affines of the segmentation and the norm image are not similar, both must be " - "the same!" - ) + Parameters + ---------- + args : argparse.Namespace + The commandline arguments object. + lut : pd.DataFrame + The ColorLUT lookup table object, e.g. FreeSurferColorLUT. + data : npt.NDArray[int] + The segmentation array. - except IOError as e: - return e.args[0] + Returns + ------- + labels : npt.NDArray[int] + The array of all labels to calculate partial volumes for. + exclude_id : list[int] + A list of labels exlicitly excluded from the output table. + """ explicit_ids = False - if hasattr(args, "ids") and args.ids is not None and len(args.ids) > 0: - labels = np.asarray(args.ids) + if __ids := getattr(args, "ids", None): + labels = np.asarray(__ids) explicit_ids = True elif lut is not None: labels = lut["ID"] # the column ID contains all ids else: - labels = np.unique(seg_data) - - if ( - hasattr(args, "excludeid") - and args.excludeid is not None - and len(args.excludeid) > 0 - ): - exclude_id = list(args.excludeid) + labels = np.unique(data) + + # filter for excludeid entries + exclude_id = [] + if _excl_id := getattr(args, "excludeid", None): + exclude_id = list(_excl_id) + # check whether if explicit_ids: - excluded_expl_ids = np.asarray( - list(filter(lambda x: x in exclude_id, labels)) - ) + _exclude = list(filter(lambda x: x in exclude_id, labels)) + excluded_expl_ids = np.asarray(_exclude) if excluded_expl_ids.size > 0: - return "Some IDs explicitly passed via --ids are also in the list of ids to exclude (--excludeid)" - labels = np.asarray(list(filter(lambda x: x not in exclude_id, labels))) - else: - exclude_id = [] - - kwargs = { - "vox_vol": np.prod(seg.header.get_zooms()).item(), - "robust_percentage": getattr(args, "robust", None), - "threads": threads, - "legacy_freesurfer": bool(getattr(args, "legacy_freesurfer", False)), - "patch_size": args.patch_size, - } + raise ValueError( + "Some IDs explicitly passed via --ids are also in the list of " + "ids to exclude (--excludeid)." + ) + labels = np.asarray([x for x in labels if x not in exclude_id], dtype=int) + return labels, exclude_id - if getattr(args, "volume_precision", None) is not None: - FORMATS["Volume_mm3"] = f'.{getattr(args, "volume_precision"):d}f' - elif kwargs["legacy_freesurfer"]: - FORMATS["Volume_mm3"] = f".1f" - if args.merged_labels is not None and len(args.merged_labels) > 0: - kwargs["merged_labels"] = {lab: vals for lab, *vals in args.merged_labels} +def main(args: argparse.Namespace) -> Literal[0] | str: + """ + Main segstats function, based on mri_segstats. - names = ["nbr", "nbr_means", "seg_means", "mix_coeff", "nbr_mix_coeff"] - var_names = ["nbr", "nbrmean", "segmean", "pv", "ipv"] - dtypes = [np.int16] + [np.float32] * 4 - if any(getattr(args, n, "") != "" for n in names): - table, maps = pv_calc(seg_data, norm_data, labels, return_maps=True, **kwargs) - - for n, v, dtype in zip(names, var_names, dtypes): - file = getattr(args, n, "") - if file == "": - continue + Parameters + ---------- + args : object + Parameter object as defined by `make_arguments().parse_args()`. + + Returns + ------- + Literal[0], str + Either as a successful return code or a string with an error message. + """ + from time import perf_counter_ns + from FastSurferCNN.utils.common import assert_no_root + from FastSurferCNN.utils.brainvolstats import Manager, read_volume_file, ImageTuple + from FastSurferCNN.data_loader.data_utils import read_classes_from_lut + + start = perf_counter_ns() + getattr(args, "allow_root", False) or assert_no_root() + + subjects_dir = getattr(args, "out_dir", None) + if subjects_dir is not None: + subjects_dir = Path(subjects_dir) + subject_id = str(getattr(args, "sid", None)) + legacy_freesurfer = bool(getattr(args, "legacy_freesurfer", False)) + measure_only = bool(getattr(args, "measure_only", False)) + manager_kwargs = {} + + # Check filename parameters segfile, pvfile, normfile, segstatsfile, and measurefile + try: + # individual entries are: (is_this_imported, the_name_and_parameters) + measures: list[tuple[bool, str]] = getattr(args, "measures", []) + any_imported_measure = any(filter(lambda x: x[0], measures)) + segfile, pvfile, normfile, segstatsfile, measurefile = parse_files( + args, + subjects_dir, + subject_id, + require_measurefile=any_imported_measure, + require_pvfile=not legacy_freesurfer, + ) + if legacy_freesurfer and not measure_only and pvfile is None: + return (f"No files are defined via -pv/--pvfile or -norm/--normfile: " + f"This is only supported for header only in legacy mode.") + if measurefile: + manager_kwargs["measurefile"] = measurefile + except ValueError as e: + return e.args[0] + + threads = getattr(args, "threads", 0) + if threads <= 0: + threads = get_num_threads() + + compute_threads = ThreadPoolExecutor(threads) + + # the manager object supports preloading of files (see below) for io parallelization + # and calculates the measure + manager = Manager(measures, segfile=segfile, **manager_kwargs) + read_lut = manager.make_read_hook(read_classes_from_lut) + if lut_file := getattr(args, "lut", None): + read_lut(lut_file, blocking=False) + # load these files in different threads to avoid waiting on IO + # (not parallel due to GIL though) + load_image = manager.make_read_hook(read_volume_file) + preload_image = partial(load_image, blocking=False) + preload_image(segfile) + if normfile is not None: + preload_image(normfile) + needs_pv_calc = manager.needs_pv_calculation() or not measure_only + if needs_pv_calc: + preload_image(pvfile) + + with manager.with_subject(subjects_dir, subject_id): + try: + _seg: ImageTuple = load_image(segfile, blocking=True) + seg, seg_data = _seg + pv_img, pv_data = None, None + norm, norm_data = None, None + + # trigger preprocessing operations on the pvfile like --mul + pv_preproc_future = None + if needs_pv_calc: + _pv: ImageTuple = load_image(pvfile, blocking=True) + pv_img, pv_data = _pv + + if not empty(pvfile_preproc := getattr(args, "pvfile_preproc", None)): + pv_preproc_future = compute_threads.submit( + preproc_image, pvfile_preproc, pv_data, + ) + + check_shape_affine(seg, pv_img, "segmentation", "pv_guide") + if normfile is not None: + _norm: ImageTuple = load_image(normfile, blocking=True) + norm, norm_data = _norm + check_shape_affine(seg, norm, "segmentation", "norm") + + except (IOError, RuntimeError, FileNotFoundError) as e: + return e.args[0] + + lut: Optional[pd.DataFrame] = None + if lut_file: + try: + lut = read_lut(lut_file) + # manager.lut = lut + except FileNotFoundError: + return ( + f"Could not find the ColorLUT in {lut_file}, make sure the --lut " + f"argument is valid." + ) + except Exception as exception: + return exception.args[0] + + if measure_only: + # in this mode, we do not output a data tabel anyways, so no need to compute + # all these PV values. + labels, exclude_id = np.zeros((0,), dtype=int), [] + else: try: - print(f"Saving {n} to {file}") - from FastSurferCNN.data_loader.data_utils import save_image + # construct the list of labels to calculate PV for + labels, exclude_id = infer_labels_excludeid(args, lut, seg_data) + except ValueError as e: + return e.args[0] + + if (_merged_labels := getattr(args, "merged_labels", None)) is None: + _merged_labels: Sequence[Sequence[int]] = () + merged_labels, measure_labels = infer_merged_labels( + manager, + labels, + merged_labels=_merged_labels, + merge_labels_start=10000, + ) + vox_vol = np.prod(seg.header.get_zooms()).item() + # more args to pass to pv_calc + kwargs = { + "vox_vol": vox_vol, + "legacy_freesurfer": legacy_freesurfer, + "threads": compute_threads, + "robust_percentage": getattr(args, "robust", None), + "patch_size": getattr(args, "patch_size", 16), + "merged_labels": merged_labels, + } + # more args to pass to write_segstatsfile + write_kwargs = { + "vox_vol": vox_vol, + "legacy_freesurfer": legacy_freesurfer, + "exclude": exclude_id, + "segfile": segfile, + "normfile": normfile, + "lut": lut_file, + "volume_precision": getattr(args, "volume_precision", "1"), + } + # ------ + # finished manager io here + # ------ + manager.compute_non_derived_pv(compute_threads) - _header = seg.header.copy() - _header.set_data_dtype(dtype) - save_image(_header, seg.affine, maps[v], file, dtype) - except Exception: - import traceback + names = ["nbr", "nbr_means", "seg_means", "mix_coeff", "nbr_mix_coeff"] + save_maps_paths = (getattr(args, n, "") for n in names) + save_maps = any(bool(path) and path != Path() for path in save_maps_paths) + save_maps = save_maps and not measure_only + + if needs_pv_calc: + if pv_preproc_future is not None: + # wait for preprocessing options on pvfile + pv_data = pv_preproc_future.result() + out = pv_calc(seg_data, pv_data, norm_data, labels, return_maps=save_maps, **kwargs) + else: + out = None - traceback.print_exc() + if measure_only: + # if we are not computing partial volume effects, do not perform pv_calc + try: + if needs_pv_calc: + # make sure required PV measures get computed + dataframe = table_to_dataframe( + out, + bool(getattr(args, "empty", False)), + must_keep_ids=merged_labels.keys(), + ) + manager.update_pv_from_table(dataframe, measure_labels) + manager.wait_write_brainvolstats(segstatsfile) + except RuntimeError as e: + return e.args[0] + print(f"Brain volume stats written to {segstatsfile}.") + duration = (perf_counter_ns() - start) / 1e9 + print(f"Calculation took {duration:.2f} seconds using up to {threads} threads.") + return 0 + + _io_futures = [] + if save_maps: + table, maps = out + dtypes = [np.int16] + [np.float32] * 4 + for name, dtype in zip(names, dtypes): + if not bool(file := getattr(args, name, "")) or file == Path(): + # skip "fullview"-files that are not defined + continue + print(f"Saving {name} to {file}...") + from FastSurferCNN.data_loader.data_utils import save_image + + _header = seg.header.copy() + _header.set_data_dtype(dtype) + _io_futures.append( + manager.executor.submit( + save_image, + _header, + seg.affine, + maps[name], + file, + dtype, + ), + ) + print("Done.") else: - table: List[PVStats] = pv_calc(seg_data, norm_data, labels, **kwargs) + table: list[PVStats] = out if lut is not None: - for i in range(len(table)): - lut_idx = lut["ID"] == table[i]["SegId"] - if lut_idx.any(): - table[i]["StructName"] = lut[lut_idx]["LabelName"].item() - elif ( - "merged_labels" in kwargs - and table[i]["SegId"] in kwargs["merged_labels"].keys() - ): - # noinspection PyTypeChecker - table[i]["StructName"] = "Merged-Label-" + str(table[i]["SegId"]) - else: - # make the label unknown - table[i]["StructName"] = "Unknown-Label" - lut_idx = {i: lut["ID"] == i for i in exclude_id} - exclude = { - i: lut[lut_idx[i]]["LabelName"].item() if lut_idx[i].any() else "" - for i in exclude_id - } - else: - exclude = {i: "" for i in exclude_id} - dataframe = pd.DataFrame(table, index=np.arange(len(table))) - if not bool(getattr(args, "empty", False)): - dataframe = dataframe[dataframe["NVoxels"] != 0] - dataframe = dataframe.sort_values("SegId") - dataframe.index = np.arange(1, len(dataframe) + 1) - lines = [] - if getattr(args, "in_dir", None): - lines.append(f'SUBJECTS_DIR {getattr(args, "in_dir")}') - if getattr(args, "sid", None): - lines.append(f'subjectname {getattr(args, "sid")}') - lines.append( - "compatibility with freesurfer's mri_segstats: " - + ("legacy" if kwargs["legacy_freesurfer"] else "fixed") + update_structnames(table, lut, merged_labels) + + dataframe = table_to_dataframe( + table, + bool(getattr(args, "empty", False)), + must_keep_ids=merged_labels.keys(), ) + lines = format_parameters(SUBJECT_DIR=subjects_dir, subjectname=subject_id) + + # wait for computation of measures and return an error message if errors occur + errors = list(manager.wait_compute()) + if not empty(errors): + error_messages = ["Some errors occurred during measure computation:"] + error_messages.extend(map(lambda e: f"{type(e).__name__}: {e.args[0]}", errors)) + return "\n - ".join(error_messages) + dataframe = manager.update_pv_from_table(dataframe, measure_labels) + lines.extend(manager.format_measures()) write_statsfile( - args.segstatsfile, + segstatsfile, dataframe, - exclude=exclude, - vox_vol=kwargs["vox_vol"], - segfile=args.segfile, - normfile=args.normfile, - lut=getattr(args, "lut", None), extra_header=lines, + **write_kwargs, ) - print( - f"Partial volume stats for {dataframe.shape[0]} labels written to {args.segstatsfile}." - ) - duration = (time.perf_counter_ns() - start) / 1e9 + print(f"Partial volume stats for {dataframe.shape[0]} labels written to " + f"{segstatsfile}.") + duration = (perf_counter_ns() - start) / 1e9 print(f"Calculation took {duration:.2f} seconds using up to {threads} threads.") + + for _io_fut in _io_futures: + if (e := _io_fut.exception()) is not None: + logging.getLogger(__name__).exception(e) + return 0 +def infer_merged_labels( + manager: "Manager", + used_labels: Iterable[int], + merged_labels: Sequence[Sequence[int]] = (), + merge_labels_start: int = 0, +) -> tuple[dict[int, Sequence[int]], dict[int, Sequence[int]]]: + """ + + Parameters + ---------- + manager : Manager + The brainvolstats Manager object to get virtual labels. + used_labels : Iterable[int] + A list of labels at that are already in use. + merged_labels : Sequence[Sequence[int]], default=() + The list of merge labels (first value is SegId, then SegIds it sums across). + merge_labels_start : int, default=0 + Start index to start at for finding multi-class merged label groups. + + Returns + ------- + all_merged_labels : dict[int, Sequence[int]] + The dictionary of all merged labels (via :class:`PVMeasure`s as well as + `merged_labels`). + """ + _merged_labels = {} + if not empty(merged_labels): + _merged_labels = {lab: vals for lab, *vals in merged_labels} + all_labels = list(_merged_labels.keys()) + list(used_labels) + _pv_merged_labels = manager.get_virtual_labels( + i for i in range(merge_labels_start, np.iinfo(int).max) if i not in all_labels + ) + + all_merged_labels = _merged_labels.copy() + all_merged_labels.update(_pv_merged_labels) + return all_merged_labels, _pv_merged_labels + + +def table_to_dataframe( + table: list[PVStats], + report_empty: bool = True, + must_keep_ids: Optional[Container[int]] = None, +) -> pd.DataFrame: + """ + Convert the list of PVStats dictionaries into a dataframe. + + Parameters + ---------- + table : list[PVStats] + List of partial volume stats dictionaries. + report_empty : bool, default=True + Whether empty regions should be part of the dataframe. + must_keep_ids : Container[int], optional + Specifies a list of segids to never remove from the table. + + Returns + ------- + pandas.DataFrame + The DataFrame object of all columns and rows in table. + """ + df = pd.DataFrame(table, index=np.arange(len(table))) + if not report_empty: + df_mask = df["NVoxels"] != 0 + if must_keep_ids and isinstance(must_keep_ids, Container): + df_mask |= df["SegId"].map(lambda x: x in must_keep_ids) + df = df[df_mask] + df = df.sort_values("SegId") + df.index = np.arange(1, len(df) + 1) + return df + + +def update_structnames( + table: list[PVStats], + lut: pd.DataFrame, + merged_labels: Optional[dict[_IntType, Sequence[_IntType]]] = None +) -> None: + """ + Update StructNames from `lut` and `merged_labels` in `table`. + + Parameters + ---------- + table : list[PVStats] + List of partial volume stats dictionaries. + lut : pandas.DataFrame + A pandas DataFrame object containing columns 'ID' and 'LabelName', which serves + as a lookup table for the structure names. + merged_labels : dict[int, Sequence[int]], optional + The dictionary with merged labels. + """ + # table is a list of dicts, so we can add the StructName to the dict + for i in range(len(table)): + lut_idx = lut["ID"] == table[i]["SegId"] + if lut_idx.any(): + # get the label name from the lut, if it is in there + table[i]["StructName"] = lut[lut_idx]["LabelName"].item() + elif merged_labels is not None and table[i]["SegId"] in merged_labels.keys(): + # auto-generate a name for merged labels + table[i]["StructName"] = "Merged-Label-" + str(table[i]["SegId"]) + else: + # make the label unknown + table[i]["StructName"] = "Unknown-Label" + # lut_idx = {i: lut["ID"] == i for i in exclude_id} + # _ids = [(i, lut_idx[i]) for i in exclude_id] + + +def format_parameters(**kwargs) -> list[str]: + """ + Formats each keyword argument passed as a pair of key and value. + + Returns + ------- + list[str] + A list of one string per keyword arg formatted as a string. + """ + return [f"{k} {v}" for k, v in kwargs.items() if v] + + def write_statsfile( - segstatsfile: str, + segstatsfile: Path | str, dataframe: pd.DataFrame, vox_vol: float, - exclude: Optional[Dict[int, str]] = None, - segfile: str = None, - normfile: str = None, - lut: str = None, + exclude: Optional[Sequence[int | str]] = None, + segfile: Optional[Path | str] = None, + normfile: Optional[Path | str] = None, + pvfile: Optional[Path | str] = None, + lut: Optional[Path | str] = None, + report_empty: bool = False, extra_header: Sequence[str] = (), -): + norm_name: str = "norm", + norm_unit: str = "MR", + volume_precision: str = "1", + legacy_freesurfer: bool = False, +) -> None: """ Write a segstatsfile very similar and compatible with mri_segstats output. Parameters ---------- - segstatsfile : str + segstatsfile : Path, str Path to the output file. dataframe : pd.DataFrame Data to write into the file. vox_vol : float Voxel volume for the header. - exclude : Optional[Dict[int, str]] - Dictionary of ids and class names that were excluded from the pv analysis (default: None). - segfile : str + exclude : Sequence[Union[int, str]], optional + Sequence of ids and class names that were excluded from the pv analysis + (default: None). + segfile : Path, str, optional Path to the segmentation file (default: empty). - normfile : str + normfile : Path, str, optional Path to the bias-field corrected image (default: empty). - lut : str + pvfile : Path, str, optional + Path to file used to compute the PV effects (default: empty). + lut : Path, str, optional Path to the lookup table to find class names for label ids (default: empty). - extra_header : Sequence[str] - Sequence of additional lines to add to the header. The initial # and newline characters will be - added. Should not include newline characters (expect at the end of strings). (default: empty sequence). + report_empty : bool, default=False + Do not skip non-empty regions in the lut. + extra_header : Sequence[str], default=() + Sequence of additional lines to add to the header. The initial # and newline + characters will be added. Should not include newline characters (expect at the + end of strings). + norm_name : str, default="norm" + Name of the intensity image. + norm_unit : str, default="MR" + Unit of the intensity image. + volume_precision : str, default="1" + Number of digits after the comma for volume. Forced to 1 for legacy_freesurfer. + legacy_freesurfer : bool, default=False + Whether the script ran with the legacy freesurfer option. """ import datetime - import os - import sys - def file_annotation(_fp, name: str, file: Optional[str]) -> None: - if file is not None: - _fp.write(f"# {name} {file}\n") - stat = os.stat(file) - if stat.st_mtime: - mtime = datetime.datetime.fromtimestamp(stat.st_mtime) - _fp.write(f"# {name}Timestamp {mtime:%Y/%m/%d %H:%M:%S}\n") + volume_precision = "1" if legacy_freesurfer else volume_precision - os.makedirs(os.path.dirname(segstatsfile), exist_ok=True) - with open(segstatsfile, "w") as fp: - fp.write( - "# Title Segmentation Statistics\n#\n" + def _title(file: IO) -> None: + """ + Write the file title to a file. + """ + file.write("# Title Segmentation Statistics\n#\n") + + def _system_info(file: IO) -> None: + """ + Write the call and system information comments of the header to a file. + """ + import os + import sys + from FastSurferCNN.version import read_and_close_version + file.write( "# generating_program segstats.py\n" + "# FastSurfer_version " + read_and_close_version() + "\n" "# cmdline " + " ".join(sys.argv) + "\n" ) - if os.name == "posix": - fp.write( + if os.name == 'posix': + file.write( f"# sysname {os.uname().sysname}\n" f"# hostname {os.uname().nodename}\n" f"# machine {os.uname().machine}\n" ) else: from socket import gethostname - - fp.write(f"# platform {sys.platform}\n" f"# hostname {gethostname()}\n") + file.write( + f"# platform {sys.platform}\n" + f"# hostname {gethostname()}\n" + ) from getpass import getuser try: - fp.write(f"# user {getuser()}\n") + file.write(f"# user {getuser()}\n") except KeyError: - fp.write(f"# user UNKNOWN\n") + file.write(f"# user UNKNOWN\n") - fp.write(f"# anatomy_type volume\n#\n") - - file_annotation(fp, "SegVolFile", segfile) - file_annotation(fp, "ColorTable", lut) - file_annotation(fp, "PVVolFile", normfile) - if exclude is not None and len(exclude) > 0: - if any(len(e) > 0 for e in exclude.values()): - fp.write( - f"# Excluding {', '.join(filter(lambda x: len(x) > 0, exclude.values()))}\n" - ) - fp.write("".join([f"# ExcludeSegId {id}\n" for id in exclude.keys()])) + def _extra_header(file: IO, lines_extra_header: Iterable[str]) -> None: + """ + Write the extra_header (including measures) to a file. + """ warn_msg_sent = False - for i, line in enumerate(extra_header): + for i, line in enumerate(lines_extra_header): if line.endswith("\n"): line = line[:-1] if line.startswith("# "): @@ -648,180 +1226,263 @@ def file_annotation(_fp, name: str, file: Optional[str]) -> None: "Replacing all newline characters with ." ) warn_msg_sent = True - fp.write(f"# {line}\n") - fp.write(f"#\n") - if lut is not None: - fp.write("# Only reporting non-empty segmentations\n") - fp.write(f"# VoxelVolume_mm3 {vox_vol}\n") - # add the Index column, if it is not in dataframe - if "Index" not in dataframe.columns: - index_df = pd.DataFrame.from_dict({"Index": dataframe.index}) - index_df.index = dataframe.index - dataframe = index_df.join(dataframe) + file.write(f"# {line}\n") - for i, col in enumerate(dataframe.columns): - for v, name in zip( - (col, FIELDS.get(col, "Unknown Column"), UNITS.get(col, "NA")), - ("ColHeader", "FieldName", "Units "), - ): - fp.write(f"# TableCol {i+1: 2d} {name} {v}\n") - fp.write( - f"# NRows {len(dataframe)}\n" f"# NTableCols {len(dataframe.columns)}\n" - ) - fp.write("# ColHeaders " + " ".join(dataframe.columns) + "\n") - max_index = int(np.ceil(np.log10(np.max(dataframe.index)))) - - def fmt_field(code: str, data) -> str: + def _file_annotation(file: IO, name: str, path_to_annotate: Optional[Path]) -> None: + """ + Write the annotation to file/path to a file. + """ + if path_to_annotate is not None: + file.write(f"# {name} {path_to_annotate}\n") + stat = path_to_annotate.stat() + if stat.st_mtime: + mtime = datetime.datetime.fromtimestamp(stat.st_mtime) + file.write(f"# {name}Timestamp {mtime:%Y/%m/%d %H:%M:%S}\n") + + def _extra_parameters( + file: IO, + _voxvol: float, + _exclude: Sequence[int | str], + _report_empty: bool = False, + _lut: Optional[Path] = None, + _leg_freesurfer: bool = False, + ) -> None: + """ + Write the comments of the table header to a file. + """ + if _exclude is not None and len(_exclude) > 0: + exclude_str = list(filter(lambda x: isinstance(x, str), _exclude)) + exclude_int = list(filter(lambda x: isinstance(x, int), _exclude)) + if len(exclude_str) > 0: + excl_names = ', '.join(exclude_str) + file.write(f"# Excluding {excl_names}\n") + if len(exclude_int) > 0: + file.write(f"# ExcludeSegId {' '.join(map(str, exclude_int))}\n") + if _lut is not None and not _report_empty: + file.write("# Only reporting non-empty segmentations\n") + file.write("# compatibility with freesurfer's mri_segstats: " + + ("legacy" if _leg_freesurfer else "fixed") + "\n") + file.write(f"# VoxelVolume_mm3 {_voxvol}\n") + + def _is_norm_column(name: str) -> bool: + """Check whether the column `name` is a norm-column.""" + return name in ("Mean", "StdDev", "Min", "Max", "Range") + + def _column_name(name: str) -> str: + """Convert the column name""" + return norm_name + name if _is_norm_column(name) else name + + def _column_unit(name: str) -> str: + if _is_norm_column(name): + return norm_unit + elif name == "Volume_mm3": + return "mm^3" + elif name == "NVoxels": + return "unitless" + return "NA" + + def _column_description(name: str) -> str: + if _is_norm_column(name): + return f"Intensity {_column_name(name)}" + return { + "Index": "Index", "SegId": "Segmentation Id", "NVoxels": "Number of Voxels", + "Volume_mm3": "Volume", "StructName": "Structure Name" + }.get(name, "Unknown Column") + + def _column_format(name: str) -> str: + if _is_norm_column(name): + return ".4f" + elif name == "Volume_mm3": + return f".{volume_precision}f" + elif name in ("Index", "SegId", "NVoxels"): + return "d" + return "s" + + def _table_header(file: IO, _dataframe: pd.DataFrame) -> None: + """Write the comments of the table header to a file.""" + columns = [col for col in COLUMNS if col in _dataframe.columns] + for i, col in enumerate(columns): + file.write(f"# TableCol {i + 1: 2d} ColHeader {_column_name(col)}\n" + f"# TableCol {i + 1: 2d} FieldName {_column_description(col)}\n" + f"# TableCol {i + 1: 2d} Units {_column_unit(col)}\n") + file.write(f"# NRows {len(_dataframe)}\n" + f"# NTableCols {len(columns)}\n") + file.write("# ColHeaders " + " ".join(map(_column_name, columns)) + "\n") + + def _table_body(file: IO, _dataframe: pd.DataFrame) -> None: + """Write the volume stats from _dataframe to a file.""" + + def fmt_field(code: str, data: pd.DataFrame) -> str: is_s, is_f, is_d = code[-1] == "s", code[-1] == "f", code[-1] == "d" filler = "<" if is_s else " >" - prec = int( - data.dropna().map(len).max() if is_s else np.ceil(np.log10(data.max())) - ) + if is_s: + prec = int(data.dropna().map(len).max()) + else: + prec = int(np.ceil(np.log10(data.max()))) if is_f: prec += int(code[-2]) + 1 return filler + str(prec) + code - fmts = ( - "{:" + fmt_field(FORMATS[k], dataframe[k]) + "}" for k in dataframe.columns - ) - fmt = " ".join(fmts) + "\n" - for index, row in dataframe.iterrows(): - data = [row[k] for k in dataframe.columns] - fp.write(fmt.format(*data)) + columns = [col for col in COLUMNS if col in _dataframe.columns] + fmt = " ".join( + ("{:" + fmt_field(_column_format(k), _dataframe[k]) + "}" + for k in columns)) + for index, row in _dataframe.iterrows(): + data = [row[k] for k in columns] + file.write(fmt.format(*data) + "\n") + + if not isinstance(segstatsfile, Path): + segstatsfile = Path(segstatsfile) + if normfile is not None and not isinstance(normfile, Path): + normfile = Path(normfile) + if segfile is not None and not isinstance(segfile, Path): + segfile = Path(segfile) + + segstatsfile.parent.mkdir(exist_ok=True) + with open(segstatsfile, "w") as fp: + _title(fp) + _system_info(fp) + fp.write(f"# anatomy_type volume\n#\n") + _extra_header(fp, extra_header) + + _file_annotation(fp, "SegVolFile", segfile) + # Annot subject hemi annot + # Label subject hemi LabelFile + _file_annotation(fp, "ColorTable", lut) + # ColorTableFromGCA + # GCATimeStamp + # masking applies to PV, not to the Measure Mask + # MaskVolFile MaskThresh MaskSign MaskFrame MaskInvert + _file_annotation(fp, "InVolFile", normfile) + _file_annotation(fp, "PVVolFile", pvfile) + _extra_parameters(fp, vox_vol, exclude, report_empty, lut, legacy_freesurfer) + # add the Index column, if it is not in dataframe + if "Index" not in dataframe.columns: + index_df = pd.DataFrame.from_dict({"Index": dataframe.index}) + index_df.index = dataframe.index + dataframe = index_df.join(dataframe) + _table_header(fp, dataframe) + _table_body(fp, dataframe) -# Label mapping functions (to aparc (eval) and to label (train)) -def read_classes_from_lut(lut_file: str | Path): +def preproc_image( + ops: Sequence[str], + data: npt.NDArray[_NumberType] +) -> npt.NDArray[_NumberType]: """ - Modify from datautils to allow support for FreeSurfer-distributed ColorLUTs. - - Read in **FreeSurfer-like** LUT table. + Apply preprocessing operations to data. Performs, --mul, --abs, --sqr, --sqrt + operations in that order. Parameters ---------- - lut_file : Path, str - Path and name of FreeSurfer-style LUT file with classes of interest. - Example entry: - ID LabelName R G B A - 0 Unknown 0 0 0 0 - 1 Left-Cerebral-Exterior 70 130 180 0. + ops : Sequence[str] + Sequence of operations to perform from 'mul=', 'div=', 'sqr', + 'abs', and 'sqrt'. + data : np.ndarray + Data to perform operations on. Returns ------- - pd.DataFrame - DataFrame with ids present, name of ids, color for plotting. + np.ndarray + Data after ops are performed on it. """ - if Path(lut_file).suffix == ".tsv": - return pd.read_csv(lut_file, sep="\t") - - # Read in file - names = { - "ID": "int", - "LabelName": "str", - "Red": "int", - "Green": "int", - "Blue": "int", - "Alpha": "int", - } - return pd.read_csv( - lut_file, - sep='\s+', - index_col=False, - skip_blank_lines=True, - comment="#", - header=None, - names=names.keys(), - dtype=names, - ) + mul_ops = np.asarray([o.startswith("mul=") or o.startswith("div=") for o in ops]) + if np.any(mul_ops): + mul_op = ops[mul_ops.nonzero()[0][-1].item()] + factor = float(mul_op[4:]) + data = (np.multiply if mul_op.startswith("mul=") else np.divide)(data, factor) + if "abs" in ops: + data = np.abs(data) + if "sqr" in ops: + data = data * data + if "sqrt" in ops: + data = np.sqrt(data) + return data def seg_borders( _array: _ArrayType, - label: Union[np.integer, bool], - out: Optional[_ArrayType] = None, + label: np.integer | bool, + out: Optional[npt.NDArray[bool]] = None, cmp_dtype: npt.DTypeLike = "int8", -) -> _ArrayType: +) -> npt.NDArray[bool]: """ Handle to fast 6-connected border computation. Parameters ---------- - _array : _ArrayType - The input binary image or labeled array. - label : Union[np.int, bool] - The label of the region for which borders will be computed. - out : Optional[_ArrayType], optional - Output array to store the computed borders (Optional). - cmp_dtype : npt.DTypeLike, optional - The data type for the Laplace computation. Default is "int8" (Optional). + _array : numpy.ndarray + Image to compute borders from, typically either a label image or a binary mask. + label : int, bool + Which classes to consider for border computation (True/False for binary mask). + out : nt.NDArray[bool], optional + The array for inplace computation. + cmp_dtype : npt.DTypeLike, default=int8 + The data type to use for border laplace computation. Returns ------- - _ArrayType - A binary image where borders are marked as True. + npt.NDArray[bool] + A binary mask with border voxels as True. """ # binarize - bin_array = _array if np.issubdtype(_array.dtype, bool) else _array == label + bin_array: npt.NDArray[bool] + bin_array = _array if np.issubdtype(_array.dtype, bool) else np.equal(_array, label) # scipy laplace is about 20% faster than skimage laplace on cpu from scipy.ndimage import laplace - def _laplace(data): - """ - Helper function to compute the Laplacian of the data, and return a - boolean array where the Laplacian is not zero. - - Parameters - ---------- - data : np.ndarray - Input data. - - Returns - ------- - npt.NDArray[bool] - Boolean array where Laplacian is not zero. - """ - return laplace(data.astype(cmp_dtype)) != np.asarray(0.0, dtype=cmp_dtype) - - # laplace - if out is not None: - out[:] = _laplace(bin_array) - return out + if np.issubdtype(cmp_dtype, bool): + laplace_data = laplace(bin_array).astype(bool) + if out is not None: + out[:] = laplace_data + laplace_data = out + return laplace_data else: - return _laplace(bin_array) + zeros = np.asarray(0., dtype=cmp_dtype) + # laplace + laplace_data = laplace(bin_array.astype(cmp_dtype)) + return np.not_equal(laplace_data, zeros, out=out) def borders( _array: _ArrayType, - labels: Union[Iterable[np.integer], bool], + labels: Iterable[np.integer] | bool, max_label: Optional[np.integer] = None, six_connected: bool = True, - out: Optional[_ArrayType] = None, -) -> _ArrayType: + out: Optional[npt.NDArray[bool]] = None, +) -> npt.NDArray[bool]: """ Handle to fast border computation. + This is an efficient implementation, for multiple/many classes between which borders + should be computed. + Parameters ---------- _array : _ArrayType - Input labeled array or binary image. - labels : Iterable[np.int], bool + Input labeled image or binary image. + labels : Iterable[int], bool List of labels for which borders will be computed. If labels is True, _array is treated as a binary mask. - max_label : np.int, optional + max_label : int, optional The maximum label ot consider. If None, the maximum label in the array is used. six_connected : bool, default=True - If True, 6-connected borders are computed, - otherwise 26-connected borders are computed. - out : _ArrayType, optional - Output array to store the computed borders (Optional). + If True, 6-connected borders (must share a face) are computed, + otherwise 26-connected borders (must share a vertex) are computed. + out : npt.NDArray[bool], optional + Output array to store the computed borders. Returns ------- - _ArrayType - A binary image where borders are marked as True. + npt.NDArray[bool] + Binary mask of border voxels. + + Raises + ------ + ValueError + If labels does not fit to _array (binary mask and integer and vice-versa). """ dim = _array.ndim - array_alloc = partial(np.full, dtype=_array.dtype) _shape_plus2 = [s + 2 for s in _array.shape] if labels is True: # already binarized @@ -839,7 +1500,7 @@ def cmp(a, b): if max_label is None: max_label = _array.max().item() - lookup = array_alloc((max_label + 1,), fill_value=0) + lookup = np.zeros((max_label + 1,), dtype=_array.dtype) # filter labels from labels that are bigger than max_label labels = list(filter(lambda x: x <= max_label, labels)) if 0 not in labels: @@ -847,105 +1508,96 @@ def cmp(a, b): lookup[labels] = np.arange(len(labels), dtype=lookup.dtype) _array = lookup[_array] logical_or = np.logical_or - __array = array_alloc(_shape_plus2, fill_value=0) - __array[(slice(1, -1),) * dim] = _array + # pad array by 1 voxel of zeros all around + padded = np.pad(_array, 1) - mid = (slice(1, -1),) * dim if six_connected: - - def ii(axis: int, off: int, is_mid: bool) -> Tuple[slice, ...]: - other_slices = mid[:1] if is_mid else (slice(None),) - return ( - other_slices * axis - + (slice(off, -1 if off == 0 else None),) - + other_slices * (dim - axis - 1) - ) - - nbr_same = [ - cmp(__array[ii(i, 0, True)], __array[ii(i, 1, True)]) for i in range(dim) - ] - nbr_same = [ - logical_or(_ns[ii(i, 0, False)], _ns[ii(i, 1, False)]) - for i, _ns in enumerate(nbr_same) - ] + def indexer(axis: int, is_mid: bool) -> tuple[SlicingTuple, SlicingTuple]: + full_slice = (slice(1, -1),) if is_mid else (slice(None),) + more_axes = dim - axis - 1 + return ((full_slice * axis + (slice(0, -1),) + full_slice * more_axes), + (full_slice * axis + (slice(1, None),) + full_slice * more_axes)) + + # compare the [padded] image/array in all directions, x, y, z... + # ([0], 0, 2, 2, 2, [0]) ==> (False, True, False, False, True) for each dim + # is_mid=True: drops padded values in unaffected axes + indexes = (indexer(i, is_mid=True) for i in range(dim)) + nbr_same = [cmp(padded[i], padded[j]) for i, j in indexes] + # merge neighbors so each border is 2 thick (left and right of change) + # (False, True, False, False, True) ==> + # ((False, True), (True, False), (False, False), (False, True)) for each dim + # is_mid=False: padded values already dropped + indexes = (indexer(i, is_mid=False) for i in range(dim)) + nbr_same = [(nbr_[i], nbr_[j]) for (i, j), nbr_ in zip(indexes, nbr_same)] + from itertools import chain + nbr_same = list(chain.from_iterable(nbr_same)) else: - - def ii(off: Iterable[int]) -> Tuple[slice, ...]: - return tuple(slice(o, None if o == 2 else o - 3) for o in off) - - nbr_same = [ - cmp(__array[mid], __array[ii(i - 1)]) - for i in np.ndindex((3,) * dim) - if np.all(i != 1) - ] + # all indexes of the neighbors: ((0, 0, 0), (0, 0, 1) ... (2, 2, 2)) + ndindexes = tuple(np.ndindex((3,) * dim)) + + def nbr_i(__array: _ArrayType, neighbor_index: int) -> _ArrayType: + """Assuming a padded array __array, returns just the neighbor_index-th + neighbors throughout the array.""" + # sample from 1d neighbor index to ndindex + nbr_ndid = ndindexes[neighbor_index] # e.g. (1, 0, 2) + slice_ndindex = tuple(slice(o, None if o == 2 else o - 3) for o in nbr_ndid) + return __array[slice_ndindex] + + # compare the array (center point) with all neighboring voxels + # neighbor samples the neighboring voxel in the padded array + nbr_same = [cmp(_array, nbr_i(padded, i)) for i in range(3**dim) if i != 2**dim] + + # reduce the per-direction/per-neighbor binary arrays into one array return np.logical_or.reduce(nbr_same, out=out) -def unsqueeze(matrix, axis: Union[int, Sequence[int]] = -1): +def pad_slicer( + slicer: Sequence[slice], + whalf: int, + img_size: np.ndarray | Sequence[float], +) -> tuple[SlicingTuple, SlicingTuple]: """ - Unsqueeze the matrix. - - Allows insertions of axis into the data/tensor, see numpy.expand_dims. This expands the torch.unsqueeze - syntax to allow unsqueezing multiple axis at the same time. + Create two slicing tuples for indexing ndarrays/tensors that 'grow' and + re-'ungrow' the patch `patch` by `whalf` (also considering the image shape). Parameters ---------- - matrix : np.ndarray - Matrix to unsqueeze. - axis : Union[int, Sequence[int]] - Axis for unsqueezing. - - Returns - ------- - np.ndarray - The unsqueezed matrix. - """ - if isinstance(matrix, np.ndarray): - return np.expand_dims(matrix, axis=axis) - - -def grow_patch( - patch: Sequence[slice], whalf: int, img_size: Union[np.ndarray, Sequence[float]] -) -> Tuple[Tuple[slice, ...], Tuple[slice, ...]]: - """ - Create two slicing tuples for indexing ndarrays/tensors that 'grow' and re-'ungrow' the patch `patch` by `whalf` (also considering the image shape). - - Parameters - ---------- - patch : Sequence[slice] - A sequence of slices. + slicer : Sequence[slice] + Input slicing tuple. whalf : int - Integer that specifies the amount to grow/ungrow the patch. + How much to pad/grow the slicing tuple all around. img_size : np.ndarray, Sequence[float] - Size of the image. + Shape of the image. Returns ------- - tuple[tuple[slice, ...], tuple[slice, ...]] - A tuple containing the grown patch and the ungrown patch. + SlicingTuple + Tuple of slice-objects to go from image to padded patch. + SlicingTuple + Tuple of slice-objects to go from padded patch to patch. + """ # patch start/stop - _patch = np.asarray([(s.start, s.stop) for s in patch]) + _patch = np.asarray([(s.start, s.stop) for s in slicer]) start, stop = _patch.T # grown patch start/stop _start, _stop = np.maximum(0, start - whalf), np.minimum(stop + whalf, img_size) + def _slice(start_end: npt.NDArray[int]) -> slice: + _start, _end = start_end + return slice(_start.item(), None if _end.item() == 0 else _end.item()) # make grown patch and grown patch to patch - grown_patch = tuple(slice(s.item(), e.item()) for s, e in zip(_start, _stop)) - ungrow_patch = tuple( - slice(s.item(), None if e.item() == 0 else e.item()) - for s, e in zip(start - _start, stop - _stop) - ) - return grown_patch, ungrow_patch + padded_slicer = tuple(slice(s.item(), e.item()) for s, e in zip(_start, _stop)) + unpadded_slicer = tuple(map(_slice, zip(start - _start, stop - _stop))) + return padded_slicer, unpadded_slicer def uniform_filter( - arr: _ArrayType, + data: _ArrayType, filter_size: int, - fillval: float, - patch: Optional[Tuple[slice, ...]] = None, - out: Optional[_ArrayType] = None, + fillval: float = 0., + slicer_patch: Optional[SlicingTuple] = None, ) -> _ArrayType: """ Apply a uniform filter (with kernel size `filter_size`) to `input`. @@ -954,405 +1606,457 @@ def uniform_filter( Parameters ---------- - arr : _ArrayType - Input array. + data : _ArrayType + Data to perform uniform filter on. filter_size : int - Size of the uniform filter. - fillval : float - Fill value when the filter is outside the array. - patch : tuple[slice, ...], optional - Sub-region of the array to apply filter to (Default: None). - out : _ArrayType, optional - Output array to store the result (Default: None). + Size of the filter. + fillval : float, default=0 + Value to fill around the image. + slicer_patch : SlicingTuple, optional + Sub_region of data to crop to (e.g. to undo the padding (default: full image). Returns ------- _ArrayType - The filtered array. + The filtered data. + """ - _patch = (slice(None),) if patch is None else patch - arr = arr.astype(float) + _patch = (slice(None),) if slicer_patch is None else slicer_patch + data = data.astype(float) from scipy.ndimage import uniform_filter def _uniform_filter(_arr, out=None): - return uniform_filter( - _arr, size=filter_size, mode="constant", cval=fillval, output=out - )[_patch] + uni_filt = uniform_filter( + _arr, + size=filter_size, + mode="constant", + cval=fillval, + output=out, + ) + return uni_filt[_patch] - if out is not None: - _uniform_filter(arr, out) - return out - return _uniform_filter(arr) + return _uniform_filter(data) @overload def pv_calc( seg: npt.NDArray[_IntType], + pv_guide: np.ndarray, norm: np.ndarray, - labels: Sequence[_IntType], + labels: npt.ArrayLike, patch_size: int = 32, vox_vol: float = 1.0, eps: float = 1e-6, robust_percentage: Optional[float] = None, merged_labels: Optional[VirtualLabel] = None, - threads: int = -1, + threads: int | Executor = -1, return_maps: False = False, legacy_freesurfer: bool = False, -) -> List[PVStats]: - """ - [MISSING]. - """ +) -> list[PVStats]: ... @overload def pv_calc( seg: npt.NDArray[_IntType], + pv_guide: np.ndarray, norm: np.ndarray, - labels: Sequence[_IntType], + labels: npt.ArrayLike, patch_size: int = 32, vox_vol: float = 1.0, eps: float = 1e-6, robust_percentage: Optional[float] = None, merged_labels: Optional[VirtualLabel] = None, - threads: int = -1, + threads: int | Executor = -1, return_maps: True = True, legacy_freesurfer: bool = False, -) -> Tuple[List[PVStats], Dict[str, Dict[int, np.ndarray]]]: - """ - [MISSING]. - """ +) -> tuple[list[PVStats], dict[str, dict[int, np.ndarray]]]: ... def pv_calc( seg: npt.NDArray[_IntType], - norm: np.ndarray, - labels: Sequence[_IntType], + pv_guide: np.ndarray, + norm: Optional[np.ndarray], + labels: npt.ArrayLike, patch_size: int = 32, vox_vol: float = 1.0, eps: float = 1e-6, - robust_percentage: Optional[float] = None, - merged_labels: Optional[VirtualLabel] = None, - threads: int = -1, + robust_percentage: float | None = None, + merged_labels: VirtualLabel | None = None, + threads: int | Executor = -1, return_maps: bool = False, legacy_freesurfer: bool = False, -) -> Union[List[PVStats], Tuple[List[PVStats], Dict[str, np.ndarray]]]: +) -> list[PVStats] | tuple[list[PVStats], dict[str, np.ndarray]]: """ Compute volume effects. Parameters ---------- - seg : npt.NDArray[_IntType] + seg : np.ndarray Segmentation array with segmentation labels. + pv_guide : np.ndarray + Image to use to calculate partial volume effects from. norm : np.ndarray - Bias. - labels : Sequence[_IntType] + Intensity image to use to calculate image statistics from. + labels : array_like Which labels are of interest. - patch_size : int - Size of patches (Default value = 32). - vox_vol : float - Volume per voxel (Default value = 1.0). - eps : float - Threshold for computation of equality (Default value = 1e-6). - robust_percentage : Optional[float] - Fraction for robust calculation of statistics (Default value = None). - merged_labels : Optional[VirtualLabel] - Defines labels to compute statistics for that are (Default value = None). - threads : int - Number of parallel threads to use in calculation (Default value = -1). - return_maps : bool - Returns a dictionary containing the computed maps (Default value = False). - legacy_freesurfer : bool - Whether to use a freesurfer legacy compatibility mode to exactly replicate freesurfer (Default value = False). + patch_size : int, default=32 + Size of patches. + vox_vol : float, default=1.0 + Volume per voxel. + eps : float, default=1e-6 + Threshold for computation of equality. + robust_percentage : float, optional + Fraction for robust calculation of statistics. + merged_labels : VirtualLabel, optional + Defines labels to compute statistics for that are. + threads : int, concurrent.futures.Executor, default=-1 + Number of parallel threads to use in calculation, alternatively an executor + object. + return_maps : bool, default=False + Returns a dictionary containing the computed maps. + legacy_freesurfer : bool, default=False + Whether to use a freesurfer legacy compatibility mode to exactly replicate + freesurfer. Returns ------- - Union[List[PVStats],Tuple[List[PVStats],Dict[str,np.ndarray]]] - Table (list of dicts) with keys SegId, NVoxels, Volume_mm3, StructName, normMean, normStdDev, - normMin, normMax, and normRange. (Note: StructName is unfilled) - if return_maps: a dictionary with the 5 meta-information pv-maps: - nbr: An image of alternative labels that were considered instead of the voxel's label - nbrmean: The local mean intensity of the label nbr at the specific voxel - segmean: The local mean intensity of the primary label at the specific voxel - pv: The partial volume of the primary label at the location - ipv: The partial volume of the alternative (nbr) label at the location - ipv: The partial volume of the alternative (nbr) label at the location. + pv_stats : list[PVStats] + Table (list of dicts) with keys SegId, NVoxels, Volume_mm3, Mean, StdDev, Min, + Max, and Range. + maps : dict[str, np.ndarray], optional + Only returned, if return_maps is True: + A dictionary with the 5 meta-information pv-maps: + nbr: The alternative labels that were considered instead of the voxel's label. + nbr_means: The local mean intensity of the label nbr at the specific voxel. + seg_means: The local mean intensity of the primary label at the specific voxel. + mixing_coeff: The partial volume of the primary label at the location. + nbr_mixing_coeff: The partial volume of the alternative (nbr) label. """ - if not isinstance(seg, np.ndarray) and np.issubdtype(seg.dtype, np.integer): - raise TypeError("The seg object is not a numpy.ndarray of int type.") - if not isinstance(norm, np.ndarray) and np.issubdtype(seg.dtype, np.numeric): - raise TypeError("The norm object is not a numpy.ndarray of numeric type.") - if not isinstance(labels, Sequence) and all(isinstance(lab, int) for lab in labels): - raise TypeError("The labels list is not a sequence of ints.") - - if seg.shape != norm.shape: + from math import ceil + + input_checker = { + "seg": (seg, np.integer), + "pv_guide": (pv_guide, np.number), + "norm": (norm, np.number), + } + for name, (img, _type) in input_checker.items(): + if (img is not None and + not (isinstance(img, np.ndarray) and np.issubdtype(img.dtype, _type))): + raise TypeError(f"The {name} object is not a numpy.ndarray of {_type}.") + _labels = np.asarray(labels) + if not isinstance(labels, Sequence): + labels = _labels.tolist() + if not np.issubdtype(_labels.dtype, np.integer): + raise TypeError("The labels list is not an arraylike of ints.") + + if seg.shape != pv_guide.shape: + raise RuntimeError( + f"The shapes of the segmentation and the pv_guide must be identical, but " + f"shapes are {seg.shape} and {pv_guide.shape}!" + ) + + has_norm = isinstance(norm, np.ndarray) + if has_norm and seg.shape != norm.shape: raise RuntimeError( - f"The shape of the segmentation and the norm must be identical, but shapes are {seg.shape} " - f"and {norm.shape}!" + f"The shape of the segmentation and the norm must be identical, but shapes " + f"are {seg.shape} and {norm.shape}!" ) - mins, maxes, voxel_counts, __voxel_counts, sums, sums_2, volumes = [ - {} for _ in range(7) - ] - loc_border = {} + mins, maxes, voxel_counts, robust_voxel_counts = [{} for _ in range(4)] + borders, sums, sums_2, volumes = [{} for _ in range(4)] - if merged_labels is not None: + if isinstance(merged_labels, dict) and len(merged_labels) > 0: + _more_labels = list(merged_labels.values()) all_labels = set(labels) - all_labels = all_labels | reduce( - lambda i, j: i | j, (set(s) for s in merged_labels.values()) - ) + all_labels |= reduce(set.union, _more_labels[1:], set(_more_labels[0])) else: all_labels = labels # initialize global_crop with the full image - global_crop: Tuple[slice, ...] = tuple(slice(0, _shape) for _shape in seg.shape) + global_crop: SlicingTuple = tuple(slice(0, _shape) for _shape in seg.shape) # ignore all regions of the image that are background only if 0 not in all_labels: # crop global_crop to the data (plus one extra voxel) - any_in_global, global_crop = crop_patch_to_mask(seg != 0, sub_patch=global_crop) + not_background = cast(npt.NDArray[bool], seg != 0) + any_in_global, global_crop = crop_patch_to_mask( + not_background, + sub_patch=global_crop + ) # grow global_crop by one, so all border voxels are included - global_crop = grow_patch(global_crop, 1, seg.shape)[0] + global_crop = pad_slicer(global_crop, 1, seg.shape)[0] if not any_in_global: raise RuntimeError("Segmentation map only consists of background") global_stats_filled = partial( global_stats, - norm=norm[global_crop], + norm=norm[global_crop] if has_norm else None, seg=seg[global_crop], robust_percentage=robust_percentage, ) - if threads < 0: - threads = get_num_threads() - elif threads == 0: - raise ValueError("Zero is not a valid number of threads.") - map_kwargs = {"chunksize": np.ceil(len(labels) / threads)} - from concurrent.futures import ThreadPoolExecutor + if threads == 0: + raise ValueError("Zero is not a valid number of threads.") + elif isinstance(threads, int) and threads > 0: + nthreads = threads + elif isinstance(threads, (Executor, int)): + nthreads: int = get_num_threads() + else: + raise TypeError("threads must be int or concurrent.futures.Executor object.") + executor = ThreadPoolExecutor(nthreads) if isinstance(threads, int) else threads + map_kwargs = {"chunksize": 1 if nthreads < 0 else ceil(len(labels) / nthreads)} - with ThreadPoolExecutor(threads) as pool: - global_stats_future = pool.map(global_stats_filled, all_labels, **map_kwargs) + global_stats_future = executor.map(global_stats_filled, all_labels, **map_kwargs) - if return_maps: - _ndarray_alloc = np.full - full_nbr_label = _ndarray_alloc(seg.shape, fill_value=0, dtype=seg.dtype) - full_nbr_mean = _ndarray_alloc( - norm.shape, fill_value=0, dtype=np.dtype(float) + if return_maps: + from concurrent.futures import ProcessPoolExecutor + if isinstance(executor, ProcessPoolExecutor): + raise NotImplementedError( + "The ProcessPoolExecutor is not compatible with return_maps=True!" ) - full_seg_mean = _ndarray_alloc( - norm.shape, fill_value=0, dtype=np.dtype(float) + full_nbr_label = np.zeros(seg.shape, dtype=seg.dtype) + full_nbr_mean = np.zeros(pv_guide.shape, dtype=float) + full_seg_mean = np.zeros(pv_guide.shape, dtype=float) + full_pv = np.ones(pv_guide.shape, dtype=float) + full_ipv = np.zeros(pv_guide.shape, dtype=float) + else: + full_nbr_label, full_seg_mean, full_nbr_mean, full_pv, full_ipv = [None] * 5 + + for lab, data in global_stats_future: + if data[0] != 0: + voxel_counts[lab], robust_voxel_counts[lab] = data[:2] + mins[lab], maxes[lab], sums[lab], sums_2[lab] = data[2:-2] + volumes[lab], borders[lab] = data[-2] * vox_vol, data[-1] + + # un_global_crop border here + any_border = np.any(list(borders.values()), axis=0) + pad_width = np.asarray( + [(slc.start, shp - slc.stop) for slc, shp in zip(global_crop, seg.shape)], + dtype=int, + ) + any_border = np.pad(any_border, pad_width) + if not np.array_equal(any_border.shape, seg.shape): + raise RuntimeError("border and seg_array do not have same shape.") + + # iterate through patches of the image + patch_iters = [range(slc.start, slc.stop, patch_size) for slc in global_crop] + # 4 chunks per core + num_valid_labels = len(voxel_counts) + map_kwargs["chunksize"] = np.ceil(num_valid_labels / nthreads / 4).item() + patch_filter_func = partial(patch_filter, mask=any_border, + global_crop=global_crop, patch_size=patch_size) + _patches = executor.map(patch_filter_func, product(*patch_iters), **map_kwargs) + patches = (patch for has_pv_vox, patch in _patches if has_pv_vox) + + patchwise_pv_calc_func = partial( + pv_calc_patch, + global_crop=global_crop, + borders=borders, + border=any_border, + seg=seg, + pv_guide=pv_guide, + full_nbr_label=full_nbr_label, + full_seg_mean=full_seg_mean, + full_pv=full_pv, + full_ipv=full_ipv, + full_nbr_mean=full_nbr_mean, + eps=eps, + legacy_freesurfer=legacy_freesurfer, + ) + for vols in executor.map(patchwise_pv_calc_func, patches, **map_kwargs): + for lab in volumes.keys(): + volumes[lab] += vols.get(lab, 0.0) * vox_vol + + # ColHeaders: Index SegId NVoxels Volume_mm3 StructName Mean StdDev Min Max Range + def prep_dict(lab: int): + nvox = voxel_counts.get(lab, 0) + vol = volumes.get(lab, 0.) + return {"SegId": lab, "NVoxels": nvox, "Volume_mm3": vol} + + table = list(map(prep_dict, labels)) + if has_norm: + robust_vc_it = robust_voxel_counts.items() + means = {lab: sums.get(lab, 0.) / cnt for lab, cnt in robust_vc_it if cnt > eps} + + def get_std(lab: _IntType, nvox: int) -> float: + # *std = sqrt((sum * (*mean) - 2 * (*mean) * sum + sum2) / (nvoxels - 1)); + return np.sqrt((sums_2[lab] - means[lab] * sums[lab]) / (nvox - 1)) + + stds = {lab: get_std(lab, nvox) for lab, nvox in robust_vc_it if nvox > eps} + + for lab, this in zip(labels, table): + this.update( + Mean=means.get(lab, 0.0), + StdDev=stds.get(lab, 0.0), + Min=mins.get(lab, 0.0), + Max=maxes.get(lab, 0.0), + Range=maxes.get(lab, 0.0) - mins.get(lab, 0.0), ) - full_pv = _ndarray_alloc(norm.shape, fill_value=1, dtype=np.dtype(float)) - full_ipv = _ndarray_alloc(norm.shape, fill_value=0, dtype=np.dtype(float)) - else: - full_nbr_label, full_seg_mean, full_nbr_mean, full_pv, full_ipv = [None] * 5 - - for lab, *data in global_stats_future: - if data[0] != 0: - voxel_counts[lab], __voxel_counts[lab] = data[:2] - mins[lab], maxes[lab], sums[lab], sums_2[lab] = data[2:-2] - volumes[lab], loc_border[lab] = data[-2] * vox_vol, data[-1] - - # un_global_crop border here - _border = np.any(list(loc_border.values()), axis=0) - border = np.pad( - _border, - tuple( - (slc.start, shp - slc.stop) for slc, shp in zip(global_crop, seg.shape) - ), - ) - if not np.array_equal(border.shape, seg.shape): - raise RuntimeError("border and seg_array do not have same shape.") - - # iterate through patches of the image - patch_iters = [ - range(slice_.start, slice_.stop, patch_size) for slice_ in global_crop - ] # for 3D - - map_kwargs["chunksize"] = int( - np.ceil(len(voxel_counts) / get_num_threads() / 4) - ) # 4 chunks per core - _patches = pool.map( - partial( - patch_filter, - mask=border, - global_crop=global_crop, - patch_size=patch_size, - ), - product(*patch_iters), - **map_kwargs, - ) - patches = (patch for has_pv_vox, patch in _patches if has_pv_vox) - - for vols in pool.map( - partial( - pv_calc_patch, - global_crop=global_crop, - loc_border=loc_border, - border=border, - seg=seg, - norm=norm, - full_nbr_label=full_nbr_label, - full_seg_mean=full_seg_mean, - full_pv=full_pv, - full_ipv=full_ipv, - full_nbr_mean=full_nbr_mean, - eps=eps, - legacy_freesurfer=legacy_freesurfer, - ), - patches, - **map_kwargs, - ): - for lab in volumes.keys(): - volumes[lab] += vols.get(lab, 0.0) * vox_vol - - means = { - lab: s / __voxel_counts[lab] - for lab, s in sums.items() - if __voxel_counts.get(lab, 0) > eps - } - # *std = sqrt((sum * (*mean) - 2 * (*mean) * sum + sum2) / (nvoxels - 1)); - stds = { - lab: np.sqrt((sums_2[lab] - means[lab] * sums[lab]) / (nvox - 1)) - for lab, nvox in __voxel_counts.items() - if nvox > 1 - } - # ColHeaders Index SegId NVoxels Volume_mm3 StructName normMean normStdDev normMin normMax normRange - table = [ - { - "SegId": lab, - "NVoxels": voxel_counts.get(lab, 0), - "Volume_mm3": volumes.get(lab, 0.0), - "StructName": "", - "normMean": means.get(lab, 0.0), - "normStdDev": stds.get(lab, 0.0), - "normMin": mins.get(lab, 0.0), - "normMax": maxes.get(lab, 0.0), - "normRange": maxes.get(lab, 0.0) - mins.get(lab, 0.0), - } - for lab in labels - ] if merged_labels is not None: - - def agg( - f: Callable[..., np.ndarray], - source: Dict[int, _NumberType], - merge_labels: Iterable[int], - ) -> _NumberType: - return f( - [ - source.get(l, 0) - for l in merge_labels - if __voxel_counts.get(l) is not None - ] - ).item() - - for lab, merge in merged_labels.items(): - if all(__voxel_counts.get(l) is None for l in merge): - logging.getLogger(__name__).warning( - f"None of the labels {merge} for merged label {lab} exist in the " - f"segmentation." - ) - continue - - nvoxels, _min, _max = ( - agg(np.sum, voxel_counts, merge), - agg(np.min, mins, merge), - agg(np.max, maxes, merge), - ) - _sums = [(l, sums.get(l, 0)) for l in merge] - _std_tmp = np.sum( - [ - s * s / __voxel_counts.get(l, 0) - for l, s in _sums - if __voxel_counts.get(l, 0) > 0 - ] - ) - _std = np.sqrt( - (agg(np.sum, sums_2, merge) - _std_tmp) / (nvoxels - 1) - ).item() - merge_row = { - "SegId": lab, - "NVoxels": nvoxels, - "Volume_mm3": agg(np.sum, volumes, merge), - "StructName": "", - "normMean": agg(np.sum, sums, merge) / nvoxels, - "normStdDev": _std, - "normMin": _min, - "normMax": _max, - "normRange": _max - _min, - } - table.append(merge_row) + labs_vol_args = (merged_labels, voxel_counts, robust_voxel_counts, volumes) + intensity_args = (mins, maxes, sums, sums_2) if has_norm else () + table.extend(calculate_merged_labels(*labs_vol_args, *intensity_args, eps=eps)) if return_maps: - return table, { + maps = { "nbr": full_nbr_label, - "segmean": full_seg_mean, - "nbrmean": full_nbr_mean, - "pv": full_pv, - "ipv": full_ipv, + "seg_means": full_seg_mean, + "nbr_means": full_nbr_mean, + "mixing_coeff": full_pv, + "nbr_mixing_coeff": full_ipv, } + return table, maps return table +def calculate_merged_labels( + merged_labels: VirtualLabel, + voxel_counts: dict[_IntType, int], + robust_voxel_counts: dict[_IntType, int], + volumes: dict[_IntType, float], + mins: Optional[dict[_IntType, float]] = None, + maxes: Optional[dict[_IntType, float]] = None, + sums: Optional[dict[_IntType, float]] = None, + sums_of_squares: Optional[dict[_IntType, float]] = None, + eps: float = 1e-6, +) -> Iterator[PVStats]: + """ + Calculate the statistics for meta-labels, i.e. labels based on other labels + (`merge_labels`). Add respective items to `table`. + + Parameters + ---------- + merged_labels : VirtualLabel + A dictionary of key 'merged id' to value list of ids it references. + voxel_counts : dict[int, int] + A dict of voxel counts for labels in the image/referenced in `merged_labels`. + robust_voxel_counts : dict[int, int] + A dict of the robust number of voxels referenced in `merged_labels`. + volumes : dict[int, float] + A dict of the volumes associated with each label. + mins : dict[int, float], optional + A dict of the minimum intensity associated with each label. + maxes : dict[int, float], optional + A dict of the minimum intensity associated with each label. + sums : dict[int, float], optional + A dict of the sums of voxel intensities associated with each label. + sums_of_squares : dict[int, float], optional + A dict of the sums of squares of voxel intensities associated with each label. + eps : float, default=1e-6 + An epsilon value for numeric stability. + + Yields + ------ + PVStats + A dictionary per entry in `merged_labels`. + """ + def num_robust_voxels(lab): + return robust_voxel_counts.get(lab, 0) + + def aggregate(source, merge_labels, f: Callable[..., np.ndarray] = np.sum): + """aggregate labels `merge_labels` from `source` with function `f`""" + _data = [source.get(l, 0) for l in merge_labels if num_robust_voxels(l) > eps] + return f(_data).item() + + def aggregate_std(sums, sums2, merge_labels, nvox): + """aggregate std of labels `merge_labels` from `source`""" + s2 = [(s := sums.get(l, 0)) * s / r for l in group + if (r := num_robust_voxels(l)) > eps] + return np.sqrt((aggregate(sums2, merge_labels) - np.sum(s2)) / nvox).item() + + for lab, group in merged_labels.items(): + stats = {"SegId": lab} + if all(l not in robust_voxel_counts for l in group): + logging.getLogger(__name__).warning( + f"None of the labels {group} for merged label {lab} exist in the " + f"segmentation." + ) + stats.update(NVoxels=0, Volume_mm3=0.0) + for k, v in {"Min": mins, "Max": maxes, "Mean": sums}.items(): + if v is not None: + stats[k] = 0. + if all(v is not None for v in (mins, maxes)): + stats["Range"] = 0. + if all(v is not None for v in (sums, sums_of_squares)): + stats["StdDev"] = 0. + else: + num_voxels = aggregate(voxel_counts, group) + stats.update(NVoxels=num_voxels, Volume_mm3=aggregate(volumes, group)) + if mins is not None: + stats["Min"] = aggregate(mins, group, np.min) + if maxes is not None: + stats["Max"] = aggregate(maxes, group, np.max) + if "Min" in stats: + stats["Range"] = stats["Max"] - stats["Min"] + if sums is not None: + stats["Mean"] = aggregate(sums, group) / num_voxels + if sums_of_squares is not None: + stats["StdDev"] = aggregate_std( + sums, + sums_of_squares, + group, + num_voxels - 1, + ) + yield stats + + def global_stats( lab: _IntType, - norm: npt.NDArray[_NumberType], + norm: npt.NDArray[_NumberType] | None, seg: npt.NDArray[_IntType], out: Optional[npt.NDArray[bool]] = None, robust_percentage: Optional[float] = None, -) -> Union[ - Tuple[_IntType, int], - Tuple[ - _IntType, - int, - int, - _NumberType, - _NumberType, - float, - float, - float, - npt.NDArray[bool], - ], -]: +) -> tuple[_IntType, _GlobalStats]: """ - Compute Label, Number of voxels, 'robust' number of voxels, norm minimum, maximum, sum, - sum of squares and 6-connected border of label lab (out references the border). + Compute Label, Number of voxels, 'robust' number of voxels, norm minimum, maximum, + sum, sum of squares and 6-connected border of label lab (out references the border). Parameters ---------- lab : _IntType Label to compute statistics for. - norm : pt.NDArray[_NumberType] - Normalized image. + norm : npt.NDArray[_NumberType], optional + The intensity image (default: None, do not compute intensity stats such as + normMin, normMax, etc.). seg : npt.NDArray[_IntType] - Segmentation image. - out : npt.NDArray[bool], Optional - Output array to store the computed borders (Optional). - robust_percentage : float, Optional - Percentage of values to keep for robust statistics (Default: None). + The segmentation image. + out : npt.NDArray[bool], optional + Output array to store the computed borders. + robust_percentage : float, optional + A robustness percentile to compute the statistics with (default: None/off = 1). Returns ------- - _IntType and int - Label and number of voxels. - or _IntType, int, int, _NumberType, _NumberType, float, float, float and npt.NDArray[bool] - Label, number of voxels, 'robust' number of voxels, norm minimum, maximum, sum, - sum of squares, volume and border. + label : int + The label the stats belong to (input). + stats : _GlobalStats + A tuple of number_of_voxels, number_of_within_robustness_thresholds, + minimum_intensity, maximum_intensity, sum_of_intensities, + sum_of_intensity_squares, and border with respect to the label. + """ - bin_array = cast(npt.NDArray[bool], seg == lab) - data = norm[bin_array].astype( - int if np.issubdtype(norm.dtype, np.integer) else float - ) + def __compute_borders(out: Optional[np.ndarray]) -> np.ndarray: + # compute/update the border + if out is None: + out = seg_borders(label_mask, True, cmp_dtype="int8").astype(bool) + else: + out[:] = seg_borders(label_mask, True, cmp_dtype="int").astype(bool) + return out + + label_mask = cast(npt.NDArray[bool], seg == lab) + if norm is None: + nvoxels = int(label_mask.sum()) + out = __compute_borders(out) + return lab, (nvoxels, nvoxels, None, None, None, None, 0., out) + + data_dtype = int if np.issubdtype(norm.dtype, np.integer) else float + data = norm[label_mask].astype(data_dtype) nvoxels: int = data.shape[0] # if lab is not in the image at all if nvoxels == 0: - return lab, 0 - # compute/update the border - if out is None: - out = seg_borders(bin_array, True, cmp_dtype="int8").astype(bool) - else: - out[:] = seg_borders(bin_array, True, cmp_dtype="int").astype(bool) + return lab, (0, 0, None, None, None, None, 0., out) + out = __compute_borders(out) if robust_percentage is not None: data = np.sort(data) @@ -1368,117 +2072,119 @@ def global_stats( _sum: float = data.sum().item() sum_2: float = (data * data).sum().item() # this is independent of the robustness criterium - volume: float = ( - np.sum(np.logical_and(bin_array, np.logical_not(out))).astype(float).item() - ) - return lab, nvoxels, __voxel_count, _min, _max, _sum, sum_2, volume, out + _volume_mask = np.logical_and(label_mask, np.logical_not(out)) + volume: float = np.sum(_volume_mask).astype(float).item() + return lab, (nvoxels, __voxel_count, _min, _max, _sum, sum_2, volume, out) def patch_filter( - pos: Tuple[int, int, int], + patch_corner: tuple[int, int, int], mask: npt.NDArray[bool], - global_crop: Tuple[slice, ...], + global_crop: SlicingTuple, patch_size: int = 32, -) -> Tuple[bool, Sequence[slice]]: +) -> tuple[bool, SlicingSequence]: """ - Return, whether there are mask-True voxels in the patch starting at pos with size patch_size and the resulting patch shrunk to mask-True regions. + Return, whether there are mask-True voxels in the patch starting at pos with size + patch_size and the resulting patch shrunk to mask-True regions. Parameters ---------- - pos : Tuple[int, int, int] - Starting position of the patch. + patch_corner : tuple[int, int, int] + The top left corner of the patch. mask : npt.NDArray[bool] - Mask to crop to. - global_crop : Tuple[slice, ...] - Global cropping context. + The mask of interest in the patch. + global_crop : SlicingTuple + A image-wide slicing mask to constrain the 'search space'. patch_size : int, default=32 - Size of patch. Defaults to 32. + The size of the patch. Returns ------- bool - Whether there are mask-True voxels in the patch. - Sequence[slice] - Cropped patch. + Whether there is any data in the patch at all. + SlicingSequence + Sequence of slice objects that describe patches with patch_corner and patch_size. """ + + def _slice(patch_start, _patch_size, image_stop): + return slice(patch_start, min(patch_start + _patch_size, image_stop)) + # create slices for current patch context (constrained by the global_crop) - patch = [ - slice(p, min(p + patch_size, slice_.stop)) - for p, slice_ in zip(pos, global_crop) - ] + patch = [_slice(pc, patch_size, s.stop) for pc, s in zip(patch_corner, global_crop)] # crop patch context to the image content return crop_patch_to_mask(mask, sub_patch=patch) def crop_patch_to_mask( - mask: npt.NDArray[_NumberType], sub_patch: Optional[Sequence[slice]] = None -) -> Tuple[bool, Sequence[slice]]: + mask: npt.NDArray[_NumberType], + sub_patch: Optional[SlicingSequence] = None, +) -> tuple[bool, SlicingSequence]: """ Crop the patch to regions of the mask that are non-zero. - Assumes mask is always positive. Returns whether there - is any mask>0 in the patch and a patch shrunk to mask>0 regions. The optional subpatch constrains this operation to - the sub-region defined by a sequence of slicing operations. + Assumes mask is always positive. Returns whether there is any mask>0 in the patch + and a slicer/patch shrunk to mask>0 regions. The optional subpatch constrains this + operation to the sub-region defined by a sequence of slicing operations. Parameters ---------- mask : npt.NDArray[_NumberType] - To crop to. + Mask to crop to. sub_patch : Optional[Sequence[slice]] Subregion of mask to only consider (default: full mask). Returns ------- - bool - Whether there are mask-True voxels in the patch. - Sequence[slice] - Cropped patch. - - Notes - ----- - This function requires device synchronization. + not_empty : bool + Whether there is any voxel in the patch at all. + target_slicer : SlicingSequence + Sequence of slice-objects to extract the subregion of mask that is 'True'. """ - _patch = [] - patch = tuple([slice(0, s) for s in mask.shape] if sub_patch is None else sub_patch) - patch_in_patch_coords = tuple( - [slice(0, slice_.stop - slice_.start) for slice_ in patch] - ) + _target_slicer = [] + if sub_patch is None: + slicer_context = tuple(slice(0, s) for s in mask.shape) + else: + slicer_context = tuple(sub_patch) + slicer_in_patch_coords = tuple([slice(0, s.stop - s.start) for s in slicer_context]) in_mask = True - _mask = mask[patch].sum(axis=2) - for i, pat in enumerate(patch_in_patch_coords): + _mask = mask[slicer_context].sum(axis=2) + for i, pat in enumerate(slicer_in_patch_coords): p = pat.start if in_mask: if i == 2: - _mask = mask[patch][tuple(_patch)].sum(axis=0) + _mask = mask[slicer_context][tuple(_target_slicer)].sum(axis=0) + slicer_ith_axis = tuple(_target_slicer[1:] if i != 2 else []) # can we shrink the patch context in i-th axis? - pat_has_mask_in_axis = ( - _mask[tuple(_patch[1:] if i != 2 else [])].sum(axis=int(i == 0)) > 0 - ) + pat_has_mask_in_axis = _mask[slicer_ith_axis].sum(axis=int(i == 0)) > 0 # modify both the _patch_size and the coordinate p to shrink the patch - _pat_mask = np.argwhere(pat_has_mask_in_axis) - if _pat_mask.shape[0] == 0: + pat_mask_indices = np.argwhere(pat_has_mask_in_axis) + if pat_mask_indices.shape[0] == 0: + # none in here _patch_size = 0 in_mask = False else: - offset = _pat_mask[0].item() + # some in the mask, find first and distance to last + offset = pat_mask_indices[0].item() p += offset - _patch_size = _pat_mask[-1].item() - offset + 1 + _patch_size = pat_mask_indices[-1].item() - offset + 1 else: _patch_size = 0 - _patch.append(slice(p, p + _patch_size)) + _target_slicer.append(slice(p, p + _patch_size)) - out_patch = [ - slice(_p.start + p.start, p.start + _p.stop) for _p, p in zip(_patch, patch) - ] - return _patch[0].start != _patch[0].stop, out_patch + def _move_slice(the_slice: slice, offset: int) -> slice: + return slice(the_slice.start + offset, the_slice.stop + offset) + + target_slicer = [_move_slice(ts, sc.start) for ts, sc in zip(_target_slicer, + slicer_context)] + return _target_slicer[0].start != _target_slicer[0].stop, target_slicer def pv_calc_patch( - patch: Tuple[slice, ...], - global_crop: Tuple[slice, ...], - loc_border: Dict[_IntType, npt.NDArray[bool]], + slicer_patch: SlicingTuple, + global_crop: SlicingTuple, + borders: dict[_IntType, npt.NDArray[bool]], seg: npt.NDArray[_IntType], - norm: np.ndarray, + pv_guide: npt.NDArray, border: npt.NDArray[bool], full_pv: Optional[npt.NDArray[float]] = None, full_ipv: Optional[npt.NDArray[float]] = None, @@ -1487,88 +2193,92 @@ def pv_calc_patch( full_nbr_mean: Optional[npt.NDArray[float]] = None, eps: float = 1e-6, legacy_freesurfer: bool = False, -) -> Dict[_IntType, float]: +) -> dict[_IntType, float]: """ Calculate PV for patch. - If full* keyword arguments are passed, also fills, per voxel results for the respective - voxels in the patch. + If full* keyword arguments are passed, the function also fills in per voxel results + for the respective voxels in the patch. Parameters ---------- - patch : Tuple[slice, ...] - Patch to calculate PV for. - global_crop : Tuple[slice, ...] - Global cropping context. - loc_border : Dict[_IntType, npt.NDArray[bool]] - Dictionary mapping labels to their borders. - seg : npt.NDArray[_IntType] - Segmentation image. - norm : np.ndarray - Normalized image. + slicer_patch : SlicingTuple + Tuple of slice-objects, with indexing origin at the image origin. + global_crop : SlicingTuple + Tuple of slice-objects, a global mask to limit computing to relevant parts of + the image. + borders : dict[int, npt.NDArray[bool]] + Dictionary containing the borders for each label. + seg : numpy.typing.NDArray[int] + The segmentation (full image) defining the labels. + pv_guide : numpy.ndarray + The (full) image with intensities to guide the PV calculation. border : npt.NDArray[bool] - Border of the patch. - full_pv : npt.NDArray[float], Optional - Array to store the partial volume for each voxel in the patch (Optional). - full_ipv : npt.NDArray[float], Optional - Array to store the inverse partial volume for each voxel in the patch (Optional). - full_nbr_label : npt.NDArray[_IntType], Optional - Array to store the label for each neighboring voxel that contributes to the - partial volume calculation. (Optional). - full_seg_mean : npt.NDArray[float], Optional - Array to store the mean intensity of the segmentation label for each voxel in - the patch (Optional). - full_nbr_mean : npt.NDArray[float], Optional - Array to store the mean intensity of the neighboring voxels that contribute to - the partial volume calculation for each voxel in the patch (Optional). + Binary mask, True, where a voxel is considered to be a border voxel. + full_pv : npt.NDArray[float], optional + PV image to fill with values for debugging. + full_ipv : npt.NDArray[float], optional + IPV image to fill with values for debugging. + full_nbr_label : npt.NDArray[_IntType], optional + NBR image to fill with values for debugging. + full_seg_mean : npt.NDArray[float], optional + Mean pv_guide-values for current segmentation label-image to fill with values + for debugging. + full_nbr_mean : npt.NDArray[float], optional + Mean pv_guide-values for nbr label-image to fill with values for debugging. eps : float, default=1e-6 - Epsilon. Defaults to 1e-6. + Epsilon for considering a voxel being in the neighborhood. legacy_freesurfer : bool, default=False - Whether to use a freesurfer legacy compatibility mode to exactly replicate freesurfer. + Whether to use the legacy freesurfer mri_segstats formula or the corrected + formula. Returns ------- - Dict[_IntType, float] - Partial and inverse partial volumes for each label in the patch. + dict[int, float] + Dictionary of per-label PV-corrected volume of affected voxels in the patch. + """ + + # Variable conventions: + # pat_* : *, but sliced to the patch, i.e. a 3D/4D array + # pat1d_* : like pat_*, but only those voxels, that are part of the border and + # flattened + log_eps = -int(np.log10(eps)) - patch = tuple(patch) - patch_grow1, ungrow1_patch = grow_patch( - patch, (FILTER_SIZES[0] - 1) // 2, seg.shape - ) - patch_grow7, ungrow7_patch = grow_patch( - patch, (FILTER_SIZES[1] - 1) // 2, seg.shape - ) - patch_shrink6 = tuple( - slice( - ug7.start - ug1.start, None if ug7.stop == ug1.stop else ug7.stop - ug1.stop - ) - for ug1, ug7 in zip(ungrow1_patch, ungrow7_patch) - ) + slicer_patch = tuple(slicer_patch) + slicer_small_patch, slicer_small_to_patch = pad_slicer(slicer_patch, + (FILTER_SIZES[0] - 1) // 2, + seg.shape) + slicer_large_patch, slicer_large_to_patch = pad_slicer(slicer_patch, + (FILTER_SIZES[1] - 1) // 2, + seg.shape) + slicer_large_to_small = tuple( + slice(l2p.start - s2p.start, + None if l2p.stop == s2p.stop else l2p.stop - s2p.stop) + for s2p, l2p in zip(slicer_small_to_patch, slicer_large_to_patch)) patch_in_gc = tuple( - slice(p.start - gc.start, p.stop - gc.start) - for p, gc in zip(patch, global_crop) - ) + slice(p.start - gc.start, + p.stop - gc.start) + for p, gc in zip(slicer_patch, global_crop)) - label_lookup = np.unique(seg[patch_grow1]) + label_lookup = np.unique(seg[slicer_small_patch]) maxlabels = label_lookup[-1] + 1 if maxlabels > 100_000: raise RuntimeError("Maximum number of labels above 100000!") # create a view for the current patch border - pat_border = border[patch] + pat_border = border[slicer_patch] pat_is_border, pat_is_nbr, pat_label_counts, pat_label_sums = patch_neighbors( label_lookup, - norm, + pv_guide, seg, pat_border, - loc_border, - patch_grow7, + borders, + slicer_large_patch, patch_in_gc, - patch_shrink6, - ungrow1_patch, - ungrow7_patch, - ndarray_alloc=np.full, + slicer_large_to_small, + slicer_small_to_patch, + slicer_large_to_patch, eps=eps, legacy_freesurfer=legacy_freesurfer, ) @@ -1578,37 +2288,33 @@ def pv_calc_patch( label_lookup_fwd[label_lookup] = np.arange(label_lookup.shape[0]) # shrink 3d patch to 1d list of border voxels - pat1d_norm, pat1d_seg = norm[patch][pat_border], seg[patch][pat_border] + pat1d_pv = pv_guide[slicer_patch][pat_border] + pat1d_seg = seg[slicer_patch][pat_border] pat1d_label_counts = pat_label_counts[:, pat_border] - # both sums and counts are normalized by n-hood-size**3, so the output is not anymore - pat1d_label_means = ( - pat_label_sums[:, pat_border] / np.maximum(pat1d_label_counts, eps * 0.0003) - ).round( - log_eps + 4 - ) # float + pat1d_robust_lblcnt = np.maximum(pat1d_label_counts, eps * 3e-4) + # both sums and counts are normalized by neighborhood-size**3, both are float + pat1d_label_means = pat_label_sums[:, pat_border] / pat1d_robust_lblcnt + pat1d_label_means = pat1d_label_means.round(log_eps + 4) # get the mean label intensity of the "local label" - mean_label = np.take_along_axis( - pat1d_label_means, unsqueeze(label_lookup_fwd[pat1d_seg], 0), axis=0 - )[0] + pat1d_seg_reindexed = np.expand_dims(label_lookup_fwd[pat1d_seg], 0) + _mean_label = np.take_along_axis(pat1d_label_means, pat1d_seg_reindexed, axis=0) + mean_label = _mean_label[0] # get the index of the "alternative label" pat1d_is_this_6border = pat_is_border[:, pat_border] # calculate which classes to consider: - is_valid = np.all( - # 1. considered (mean of) alternative label must be on the other side of norm as the (mean of) the segmentation - # label of the current voxel + pat1d_mean_intensity_higher = pat1d_label_means > np.expand_dims(pat1d_pv, 0) + pat1d_mean_intensity_lower = np.expand_dims(mean_label > pat1d_pv, 0) + pat1d_mean_different = np.expand_dims(np.abs(mean_label - pat1d_pv) > eps, 0) + pat1d_is_valid = np.all( [ - np.logical_xor( - pat1d_label_means > unsqueeze(pat1d_norm, 0), - unsqueeze(mean_label > pat1d_norm, 0), - ), - # 2. considered (mean of) alternative label must be different to norm of voxel - pat1d_label_means != unsqueeze(pat1d_norm, 0), - # 3. (mean of) segmentation label must be different to norm of voxel - np.broadcast_to( - unsqueeze(np.abs(mean_label - pat1d_norm) > eps, 0), - pat1d_label_means.shape, - ), + # 1. considered (mean of) alternative label must be on the other side of pv + # as the (mean of) the segmentation label of the current voxel + np.logical_xor(pat1d_mean_intensity_higher, pat1d_mean_intensity_lower), + # 2. considered (mean of) alternative label must be different to pv of voxel + pat1d_label_means != np.expand_dims(pat1d_pv, 0), + # 3. (mean of) segmentation label must be different to pv of voxel + np.broadcast_to(pat1d_mean_different, pat1d_label_means.shape), # 4. label must be a neighbor pat_is_nbr[:, pat_border], # 3. label must not be the segmentation @@ -1617,170 +2323,199 @@ def pv_calc_patch( axis=0, ) - none_valid = ~is_valid.any(axis=0, keepdims=False) - # select the label, that is valid or not valid but also exists and is not the current label - max_counts_index = np.round(pat1d_label_counts * is_valid, log_eps).argmax( - axis=0, keepdims=False - ) + pat1d_none_valid = ~pat1d_is_valid.any(axis=0, keepdims=False) + # select the label, that is valid or not valid but also exists and is not the + # current label + pat1d_label_frequency = np.round(pat1d_label_counts * pat1d_is_valid, log_eps) + pat1d_max_frequency_index = pat1d_label_frequency.argmax(axis=0, keepdims=False) - nbr_label = label_lookup[max_counts_index] # label with max_counts - nbr_label[none_valid] = 0 + pat1d_nbr_label = label_lookup[pat1d_max_frequency_index] # label with max_counts + pat1d_nbr_label[pat1d_none_valid] = 0 # get the mean label intensity of the "alternative label" - mean_nbr = np.take_along_axis( - pat1d_label_means, unsqueeze(label_lookup_fwd[nbr_label], 0), axis=0 - )[0] + pat1d_label_lookup_nbr = np.expand_dims(label_lookup_fwd[pat1d_nbr_label], 0) + mean_nbr = np.take_along_axis(pat1d_label_means, pat1d_label_lookup_nbr, axis=0)[0] # interpolate between the "local" and "alternative label" mean_to_mean_nbr = mean_label - mean_nbr delta_gt_eps = np.abs(mean_to_mean_nbr) > eps - pat1d_pv = (pat1d_norm - mean_nbr) / np.where( - delta_gt_eps, mean_to_mean_nbr, eps - ) # make sure no division by zero - - pat1d_pv[~delta_gt_eps] = 1.0 # set pv fraction to 1 if division by zero - pat1d_pv[ - none_valid - ] = 1.0 # set pv fraction to 1 for voxels that have no 'valid' nbr + # make sure no division by zero + pat1d_pv = (pat1d_pv - mean_nbr) / np.where(delta_gt_eps, mean_to_mean_nbr, eps) + + # set pv fraction to 1 if division by zero + pat1d_pv[~delta_gt_eps] = 1.0 + # set pv fraction to 1 for voxels that have no valid nbr + pat1d_pv[pat1d_none_valid] = 1.0 pat1d_pv[pat1d_pv > 1.0] = 1.0 pat1d_pv[pat1d_pv < 0.0] = 0.0 pat1d_inv_pv = 1.0 - pat1d_pv if legacy_freesurfer: - # re-create the "supposed" freesurfer inconsistency that does not count vertex neighbors, if the voxel label - # is not of question + # re-create the "supposed" freesurfer inconsistency that does not count vertex + # neighbors, if the voxel label is not of question mask_by_6border = np.take_along_axis( - pat1d_is_this_6border, unsqueeze(label_lookup_fwd[nbr_label], 0), axis=0 - )[0] - pat1d_inv_pv = pat1d_inv_pv * mask_by_6border + pat1d_is_this_6border, pat1d_label_lookup_nbr, axis=0 + ) + pat1d_inv_pv = pat1d_inv_pv * mask_by_6border[0] if full_pv is not None: - full_pv[patch][pat_border] = pat1d_pv + full_pv[slicer_patch][pat_border] = pat1d_pv if full_nbr_label is not None: - full_nbr_label[patch][pat_border] = nbr_label + full_nbr_label[slicer_patch][pat_border] = pat1d_nbr_label if full_ipv is not None: - full_ipv[patch][pat_border] = pat1d_inv_pv + full_ipv[slicer_patch][pat_border] = pat1d_inv_pv if full_nbr_mean is not None: - full_nbr_mean[patch][pat_border] = mean_nbr + full_nbr_mean[slicer_patch][pat_border] = mean_nbr if full_seg_mean is not None: - full_seg_mean[patch][pat_border] = mean_label - - return { - lab: ( - pat1d_pv.sum(where=pat1d_seg == lab) - + pat1d_inv_pv.sum(where=nbr_label == lab) - ).item() - for lab in label_lookup - } + full_seg_mean[slicer_patch][pat_border] = mean_label + + def _vox_calc_pv(lab: _IntType) -> float: + """ + Compute the PV of voxels labels lab and voxels not labeled lab, but chosen as + mixing label. + """ + pv_sum = pat1d_pv.sum(where=pat1d_seg == lab).item() + inv_pv_sum = pat1d_inv_pv.sum(where=pat1d_nbr_label == lab).item() + return pv_sum + inv_pv_sum + + return {lab: _vox_calc_pv(lab) for lab in label_lookup} def patch_neighbors( - labels, - norm, - seg, - pat_border, - loc_border, - patch_grow7, - patch_in_gc, - patch_shrink6, - ungrow1_patch, - ungrow7_patch, - ndarray_alloc, - eps, - legacy_freesurfer=False, -): + labels: Sequence[_IntType], + pv_guide: npt.NDArray, + seg: npt.NDArray[_IntType], + border_patch: npt.NDArray[bool], + borders: dict[_IntType, npt.NDArray[bool]], + slicer_large_patch: SlicingTuple, + slicer_patch: SlicingTuple, + slicer_large_to_small: SlicingTuple, + slicer_small_to_patch: SlicingTuple, + slicer_large_to_patch: SlicingTuple, + eps: float = 1e-6, + legacy_freesurfer: bool = False, +) -> tuple[ + "npt.NDArray[bool]", + "npt.NDArray[bool]", + "npt.NDArray[float]", + "npt.NDArray[float]", +]: """ - Calculate the neighbor statistics of labels, etc.. + Calculate the neighbor statistics of labels for a specific patch. + + The patch is defined by `slicer_large_patch`, `slicer_large_to_small`, + `slicer_small_to_patch`, and `slicer_large_to_patch`. Parameters ---------- - labels : List[int] - List of unique labels. - norm : np.ndarray - Array containing normalization values. - seg : np.ndarray - Segmentation array. - pat_border : np.ndarray - Array indicating whether each voxel in the patch is on the border. - loc_border : Dict[int, np.ndarray] - Dictionary mapping labels to arrays indicating whether each voxel is on the global crop border. - patch_grow7 : Tuple[slice, ...] - Grown patch for label detection. - patch_in_gc : Tuple[slice, ...] - Patch within the global crop. - patch_shrink6 : Tuple[slice, ...] - Shrunken patch for neighbor detection. - ungrow1_patch : Tuple[slice, ...] - Ungrown patch for border detection. - ungrow7_patch : Tuple[slice, ...] - Ungrown patch for label statistics. - ndarray_alloc : Callable[..., np.ndarray] - Function for allocating NumPy arrays. - eps : float - Small value for numerical stability. - legacy_freesurfer : bool, optional - Whether to use legacy FreeSurfer mode. Defaults to False. + labels : Sequence[int] + A sequence of all labels that we want to compute the PV for. + pv_guide : numpy.ndarray + The (full) image with intensities to guide the PV calculation. + seg : numpy.typing.NDArray[int] + The segmentation (full image) defining the labels. + border_patch : npt.NDArray[bool] + Binary mask for the current patch, True, where a voxel is considered to be a + border voxel. + borders : dict[_IntType, npt.NDArray[bool]] + Dictionary containing the borders for each label. + slicer_large_patch : SlicingTuple + Slicing tuple to obtain a patch of shape like the patch but padded to the large + filter size. + slicer_patch : SlicingTuple + Tuple of slice-objects to extract the patch from the full image. + slicer_large_to_small : SlicingTuple + Tuple of slice-objects to extract the small patch (patch plus small filter + window) from the large patch (patch plus large filter window). + slicer_small_to_patch : SlicingTuple + Tuple of slice-objects to extract the patch from the patch padded by the small + filter size. + slicer_large_to_patch : SlicingTuple + Tuple of slice-objects to extract the patch from the patch padded by the large + filter size. + eps : float, default=1e-6 + Epsilon for considering a voxel being in the neighborhood. + legacy_freesurfer : bool, default=False + Whether to use the legacy freesurfer mri_segstats formula or the corrected + formula. Returns ------- - Tuple of NumPy arrays: - - pat_is_border : np.ndarray - Array indicating whether each label is on the patch border. - - pat_is_nbr : np.ndarray - Array indicating whether each label is a neighbor in the patch. - - pat_label_counts : np.ndarray - Array containing label counts in the patch. - - pat_label_sums : np.ndarray - Array containing the sum of normalized values for each label in the patch. + pat_is_border : npt.NDArray[bool] + Array indicating whether each label is on the patch border. + pat_is_nbr : npt.NDArray[bool] + Array indicating whether each label is a neighbor in the patch. + pat_label_count : npt.NDArray[float] + Array containing label counts in the patch (divided by the neighborhood size). + pat_label_sums : npt.NDArray[float] + Array containing the sum of normalized values for each label in the patch. """ - loc_shape = (len(labels),) + pat_border.shape + shape_of_patch = (len(labels),) + border_patch.shape - pat_label_counts, pat_label_sums = ndarray_alloc( - (2,) + loc_shape, fill_value=0.0, dtype=float - ) - pat_is_nbr, pat_is_border = ndarray_alloc( - (2,) + loc_shape, fill_value=False, dtype=bool - ) + pat_label_counts, pat_label_sums = np.zeros((2,) + shape_of_patch, dtype=float) + pat_is_nbr, pat_is_border = np.zeros((2,) + shape_of_patch, dtype=bool) # all False for i, lab in enumerate(labels): - # in legacy freesurfer mode, we want to fill the binary labels with True if we are looking at the background - fill_binary_label = float(legacy_freesurfer and lab == 0) + # in legacy freesurfer mode, we want to fill the binary labels with True if we + # are looking at the background + fillvalue_binary_label = float(legacy_freesurfer and lab == 0) - pat7_bin_array = cast(npt.NDArray[bool], seg[patch_grow7] == lab) + same_label_large_patch = cast(npt.NDArray[bool], seg[slicer_large_patch] == lab) + same_label_small_patch = same_label_large_patch[slicer_large_to_small] # implicitly also a border detection: is lab a neighbor of the "current voxel" + # returns 'small patch'-array of float (shape: (patch_size + filter_size)**3) + # for label 'lab' tmp_nbr_label_counts = uniform_filter( - pat7_bin_array[patch_shrink6], FILTER_SIZES[0], fill_binary_label - ) # as float (*filter_size**3) + same_label_small_patch, + FILTER_SIZES[0], + fillvalue_binary_label, + ) if tmp_nbr_label_counts.sum() > eps: # lab is at least once a nbr in the patch (grown by one) - if lab in loc_border: - pat_is_border[i] = loc_border[lab][patch_in_gc] + if lab in borders: + pat_is_border[i] = borders[lab][slicer_patch] else: pat7_is_border = seg_borders( - pat7_bin_array[patch_shrink6], True, cmp_dtype="int8" + same_label_small_patch, + label=True, + cmp_dtype="int8", ) - pat_is_border[i] = pat7_is_border[ungrow1_patch].astype(bool) + pat_is_border[i] = pat7_is_border[slicer_small_to_patch].astype(bool) - pat_is_nbr[i] = tmp_nbr_label_counts[ungrow1_patch] > eps + pat_is_nbr[i] = tmp_nbr_label_counts[slicer_small_to_patch] > eps + # as float (*filter_size**3) pat_label_counts[i] = uniform_filter( - pat7_bin_array, FILTER_SIZES[1], fill_binary_label - )[ - ungrow7_patch - ] # as float (*filter_size**3) - pat7_filtered_norm = norm[patch_grow7] * pat7_bin_array - pat_label_sums[i] = uniform_filter(pat7_filtered_norm, FILTER_SIZES[1], 0)[ - ungrow7_patch - ] + same_label_large_patch, + FILTER_SIZES[1], + fillvalue_binary_label, + slicer_patch=slicer_large_to_patch + ) + pat_large_filter_pv = pv_guide[slicer_large_patch] * same_label_large_patch + pat_label_sums[i] = uniform_filter( + pat_large_filter_pv, + FILTER_SIZES[1], + fillval=0, slicer_patch=slicer_large_to_patch + ) # else: lab is not present in the patch return pat_is_border, pat_is_nbr, pat_label_counts, pat_label_sums +# timeit cmd arg: +# python -m timeit < int: return val -def int_ge_zero(value) -> int: +def int_ge_zero(value: str) -> int: """ Convert to integers greater 0. Parameters ---------- value : str - Integer to convert. + String to convert to int. Returns ------- diff --git a/FastSurferCNN/utils/brainvolstats.py b/FastSurferCNN/utils/brainvolstats.py new file mode 100644 index 00000000..40451bb5 --- /dev/null +++ b/FastSurferCNN/utils/brainvolstats.py @@ -0,0 +1,2442 @@ +import abc +import logging +import re +from concurrent.futures import Executor +from contextlib import contextmanager +from pathlib import Path +from typing import (TYPE_CHECKING, Sequence, cast, Literal, Iterable, Callable, Union, + Optional, overload, TextIO, Protocol, TypeVar, Generic, Type) +from concurrent.futures import Future + +import numpy as np + +if TYPE_CHECKING: + from numpy import typing as npt + import lapy + import nibabel as nib + import pandas as pd + + from CerebNet.datasets.utils import LTADict + +MeasureTuple = tuple[str, str, int | float, str] +ImageTuple = tuple["nib.analyze.SpatialImage", "np.ndarray"] +UnitString = Literal["unitless", "mm^3"] +MeasureString = Union[str, "Measure"] +AnyBufferType = Union[ + dict[str, MeasureTuple], + ImageTuple, + "lapy.TriaMesh", + "npt.NDArray[float]", + "pd.DataFrame", +] +T_BufferType = TypeVar( + "T_BufferType", + bound=Union[ + ImageTuple, + dict[str, MeasureTuple], + "lapy.TriaMesh", + "np.ndarray", + "pd.DataFrame", + ]) +DerivedAggOperation = Literal["sum", "ratio", "by_vox_vol"] +AnyMeasure = Union["AbstractMeasure", str] +PVMode = Literal["vox", "pv"] +ClassesType = Sequence[int] +ClassesOrCondType = ClassesType | Callable[["npt.NDArray[int]"], "npt.NDArray[bool]"] +MaskSign = Literal["abs", "pos", "neg"] +_ToBoolCallback = Callable[["npt.NDArray[int]"], "npt.NDArray[bool]"] + + +class ReadFileHook(Protocol[T_BufferType]): + + @overload + def __call__(self, file: Path, blocking: True = True) -> T_BufferType: ... + + @overload + def __call__(self, file: Path, blocking: False) -> None: ... + + def __call__(self, file: Path, b: bool = True) -> Optional[T_BufferType]: ... + + +class _DefaultFloat(float): + pass + + +def read_measure_file(path: Path) -> dict[str, MeasureTuple]: + """ + Read '# Measure '-entries from stats files. + + Parameters + ---------- + path : Path + The path to the file to read from. + + Returns + ------- + A dictionary of Measure keys to tuple of descriptors like + {'': ('', '', , '')}. + """ + if not path.exists(): + raise IOError(f"Measures could not be imported from {path}, " + f"the file does not exist.") + with open(path, "r") as fp: + lines = list(fp.readlines()) + vox_line = list(filter(lambda l: l.startswith("# VoxelVolume_mm3 "), lines)) + lines = filter(lambda l: l.startswith("# Measure "), lines) + + def to_measure(line: str) -> tuple[str, MeasureTuple]: + data_tup = line.removeprefix("# Measure ").strip() + import re + key, name, desc, sval, unit = re.split("\\s*,\\s*", data_tup) + value = float(sval) if "." in sval else int(sval) + return key, (name, desc, value, unit) + + data = dict(map(to_measure, lines)) + if len(vox_line) > 0: + vox_vol = float(vox_line[-1].split(" ")[2].strip()) + data["vox_vol"] = ("Voxel volume", "The volume of a voxel", vox_vol, "mm^3") + + return data + + +def read_volume_file(path: Path) -> ImageTuple: + """ + Read a volume from disk. + + Parameters + ---------- + path : Path + The path to the file to read from. + + Returns + ------- + A tuple of nibabel image object and the data. + """ + try: + import nibabel as nib + img = cast(nib.analyze.SpatialImage, nib.load(path)) + if not isinstance(img, nib.analyze.SpatialImage): + raise RuntimeError( + f"Loading the file '{path}' for Measure was invalid, no SpatialImage." + ) + except (IOError, FileNotFoundError) as e: + args = e.args[0] + raise IOError(f"Failed loading the file '{path}' with error: {args}") from e + data = np.asarray(img.dataobj) + return img, data + + +def read_mesh_file(path: Path) -> "lapy.TriaMesh": + """ + Read a mesh from disk. + + Parameters + ---------- + path : Path + The path to the file. + + Returns + ------- + lapy.TriaMesh + The mesh object read from the file. + """ + try: + import lapy + mesh = lapy.TriaMesh.read_fssurf(str(path)) + except (IOError, FileNotFoundError) as e: + args = e.args[0] + raise IOError( + f"Failed loading the file '{path}' with error: {args}") from e + return mesh + + +def read_lta_transform_file(path: Path) -> "npt.NDArray[float]": + """ + Read and extract the first lta transform from an LTA file. + + Parameters + ---------- + path : Path + The path of the LTA file. + + Returns + ------- + matrix : npt.NDArray[float] + Matrix of shape (4, 4). + """ + from CerebNet.datasets.utils import read_lta + return read_lta(path)["lta"][0, 0] + + +def read_xfm_transform_file(path: Path) -> "npt.NDArray[float]": + """ + Read XFM talairach transform. + + Parameters + ---------- + path : str | Path + The filename/path of the transform file. + + Returns + ------- + tal + The talairach transform matrix. + + Raises + ------ + ValueError + If the file is of an invalid format. + """ + with open(path) as f: + lines = f.readlines() + + try: + transf_start = [l.lower().startswith("linear_") for l in lines].index(True) + 1 + tal_str = [l.replace(";", " ") for l in lines[transf_start:transf_start + 3]] + tal = np.genfromtxt(tal_str) + tal = np.vstack([tal, [0, 0, 0, 1]]) + + return tal + except Exception as e: + err = ValueError(f"Could not find taiairach transform in {path}.") + raise err from e + + +def read_transform_file(path: Path) -> "npt.NDArray[float]": + """ + Read xfm or lta transform file. + + Parameters + ---------- + path : Path + The path to the file. + + Returns + ------- + tal + The talairach transform matrix. + """ + if path.suffix == ".lta": + return read_lta_transform_file(path) + elif path.suffix == ".xfm": + return read_xfm_transform_file(path) + else: + raise NotImplementedError( + f"The extension {path.suffix} is not '.xfm' or '.lta' and not recognized.") + + +def mask_in_array(arr: "npt.NDArray", items: "npt.ArrayLike") -> "npt.NDArray[bool]": + """ + Efficient function to generate a mask of elements in `arr`, which are also in items. + + Parameters + ---------- + arr : npt.NDArray + An array with data, most likely int. + items : npt.ArrayLike + Which elements of `arr` in arr should yield True. + + Returns + ------- + mask : npt.NDArray[bool] + A binary array, true, where elements in `arr` are in `items`. + + See Also + -------- + mask_not_in_array + """ + _items = np.asarray(items) + if _items.size == 0: + return np.zeros_like(arr, dtype=bool) + elif _items.size == 1: + return np.asarray(arr == _items.flat[0]) + else: + max_index = max(np.max(items), np.max(arr)) + if max_index >= 2 ** 16: + logging.getLogger(__name__).warning( + f"labels in arr are larger than {2 ** 16 - 1}, this is not recommended!" + ) + lookup = np.zeros(max_index + 1, dtype=bool) + lookup[_items] = True + return lookup[arr] + + +def mask_not_in_array( + arr: "npt.NDArray", + items: "npt.ArrayLike", +) -> "npt.NDArray[bool]": + """ + Inverse of mask_in_array. + + Parameters + ---------- + arr : npt.NDArray + An array with data, most likely int. + items : npt.ArrayLike + Which elements of `arr` in arr should yield False. + + Returns + ------- + mask : npt.NDArray[bool] + A binary array, true, where elements in `arr` are not in `items`. + + See Also + -------- + mask_in_array + """ + _items = np.asarray(items) + if _items.size == 0: + return np.ones_like(arr, dtype=bool) + elif _items.size == 1: + return np.asarray(arr != _items.flat[0]) + else: + max_index = max(np.max(items), np.max(arr)) + if max_index >= 2 ** 16: + logging.getLogger(__name__).warning( + f"labels in arr are larger than {2 ** 16 - 1}, this is not recommended!" + ) + lookup = np.ones(max_index + 1, dtype=bool) + lookup[_items] = False + return lookup[arr] + + +class AbstractMeasure(metaclass=abc.ABCMeta): + """ + The base class of all measures, which implements the name, description, and unit + attributes as well as the methods as_tuple(), __call__(), read_subject(), + set_args(), parse_args(), help(), and __str__(). + """ + + __PATTERN = re.compile("^([^\\s=]+)\\s*=\\s*(\\S.*)$") + + def __init__(self, name: str, description: str, unit: str): + self._name: str = name + self._description: str = description + self._unit: str = unit + self._subject_dir: Path | None = None + + def as_tuple(self) -> MeasureTuple: + return self._name, self._description, self(), self.unit + + @property + def name(self) -> str: + return self._name + + @property + def description(self) -> str: + return self._description + + @property + def unit(self) -> str: + return self._unit + + @property + def subject_dir(self) -> Path: + return self._subject_dir + + @abc.abstractmethod + def __call__(self) -> int | float: + ... + + def read_subject(self, subject_dir: Path) -> bool: + """ + Perform IO required to compute/fill the Measure. + + Parameters + ---------- + subject_dir : Path + Path to the directory of the subject_dir (often subject_dir/subject_id). + + Returns + ------- + bool + Whether there was an update. + """ + updated = subject_dir != self.subject_dir + if updated: + self._subject_dir = subject_dir + return updated + + @abc.abstractmethod + def _parsable_args(self) -> list[str]: + ... + + def set_args(self, **kwargs: str) -> None: + """ + Set the arguments of the Measure. + + Raises + ------ + ValueError + If there are unrecognized keyword arguments. + """ + if len(kwargs) > 0: + raise ValueError(f"Invalid args {tuple(kwargs.keys())}") + + def parse_args(self, *args: str) -> None: + """ + Parse additional args defining the behavior of the Measure. + + Parameters + ---------- + *args : str + Each args can be a string of '' (arg-style) and '=' + (keyword-arg-style), arg-style cannot follow keyword-arg-style args. + + Raises + ------ + ValueError + If there are more arguments than registered argument names. + RuntimeError + If an arg-style follows a keyword-arg-style argument, or if a keyword value + is redefined, or a keyword is not valid. + """ + + def kwerror(i, args, msg) -> RuntimeError: + return RuntimeError(f"Error parsing arg {i} in {args}: {msg}") + + _pargs = self._parsable_args() + if len(args) > len(_pargs): + raise ValueError( + f"The measure {self.name} can have up to {len(_pargs)} arguments, but " + f"parsing {len(args)}: {args}." + ) + _kwargs = {} + _kwmode = False + for i, (arg, default_key) in enumerate(zip(args, _pargs)): + if (hit := self.__PATTERN.match(arg)) is None: + # non-keyword mode + if _kwmode: + raise kwerror(i, args, f"non-keyword after keyword") + _kwargs[default_key] = arg + else: + # keyword mode + _kwmode = True + k = hit.group(1) + if k in _kwargs: + raise kwerror(i, args, f"keyword '{k}' already assigned") + if k not in _pargs: + raise kwerror(i, args, f"keyword '{k}' not in {_pargs}") + _kwargs[k] = hit.group(2) + self.set_args(**_kwargs) + + def help(self) -> str: + """ + Compiles a help message for the measure describing the measure's settings. + + Returns + ------- + A help string describing the Measure settings. + """ + return f"{self.name}=" + + @abc.abstractmethod + def __str__(self) -> str: + ... + + +class NullMeasure(AbstractMeasure): + """ + A Measure that supports no operations, always returns a value of zero. + """ + + def _parsable_args(self) -> list[str]: + return [] + + def __call__(self) -> int | float: + return 0 if self.unit == "unitless" else 0.0 + + def help(self) -> str: + return super().help() + "NULL" + + def __str__(self) -> str: + return "NullMeasure()" + + +class Measure(AbstractMeasure, Generic[T_BufferType], metaclass=abc.ABCMeta): + """ + Class to buffer computed values, buffers computed values. Implements a value + buffering interface for computed measure values and implement the read_subject + pattern. + """ + + __buffer: float | int | None + __token: str = "" + __PATTERN = re.compile("^([^\\s=]*file)\\s*=\\s*(\\S.*)$") + + def __call__(self) -> int | float: + token = str(self._subject_dir) + if self.__buffer is None or self.__token != token: + self.__token = token + self.__buffer = self._compute() + return self.__buffer + + @abc.abstractmethod + def _compute(self) -> int | float: + ... + + def __init__( + self, + file: Path, + name: str, + description: str, + unit: str, + read_hook: ReadFileHook[T_BufferType], + ): + self._file = file + self._callback = read_hook + self._data: Optional[T_BufferType] = None + self.__buffer = None + super().__init__(name, description, unit) + + def _load_error(self, name: str = "data") -> RuntimeError: + return RuntimeError( + f"The '{name}' is not available for {self.name} ({type(self).__name__}), " + f"maybe the subject has not been loaded or the cache been invalidated." + ) + + def _filename(self) -> Path: + return self._subject_dir / self._file + + def read_subject(self, subject_dir: Path) -> bool: + """ + Perform IO required to compute/fill the Measure. Delegates file reading to + read_hook (set in __init__). + + Parameters + ---------- + subject_dir : Path + Path to the directory of the subject_dir (often subject_dir/subject_id). + + Returns + ------- + bool + Whether there was an update to the data. + """ + if super().read_subject(subject_dir): + try: + self._data = self._callback(self._filename()) + except Exception as e: + e.args = f"{e.args[0]} ... during reading for measure {self}.", + raise e + return True + return False + + def _parsable_args(self) -> list[str]: + return ["file"] + + def set_args(self, file: str | None = None, **kwargs: str) -> None: + if file is not None: + self._file = Path(file) + return super().set_args(**kwargs) + + def __str__(self) -> str: + return f"{type(self).__name__}(file={self._file})" + + +class ImportedMeasure(Measure[dict[str, MeasureTuple]]): + """ + A Measure that implements reading measure values from a statsfile. + """ + + PREFIX = "__IMPORTEDMEASURE-prefix__" + read_file = staticmethod(read_measure_file) + + def __init__( + self, + key: str, + measurefile: Path, + name: str = "N/A", + description: str = "N/A", + unit: UnitString = "unitless", + read_file: Optional[ReadFileHook[dict[str, MeasureTuple]]] = None, + vox_vol: Optional[float] = None, + ): + self._key: str = key + super().__init__( + measurefile, + name, + description, + unit, + self.read_file if read_file is None else read_file, + ) + self._vox_vol: Optional[float] = vox_vol + + def _compute(self) -> int | float: + """ + Will also update the name, description and unit from the strings in the file. + + Returns + ------- + value : int | float + value of the measure (as read from the file) + """ + try: + self._name, self._description, out, self._unit = self._data[self._key] + except KeyError as e: + raise KeyError(f"Could not find {self._key} in {self._file}.") from e + return out + + def _parsable_args(self) -> list[str]: + return ["key", "measurefile"] + + def set_args( + self, + key: str | None = None, + measurefile: str | None = None, + **kwargs: str, + ) -> None: + if measurefile is not None: + kwargs["file"] = measurefile + if key is not None: + self._key = key + return super().set_args(**kwargs) + + def help(self) -> str: + return super().help() + f" imported from {self._file}" + + def __str__(self) -> str: + return f"ImportedMeasure(key={self._key}, measurefile={self._file})" + + def assert_measurefile_absolute(self): + """ + Assert that the Measure can be imported without a subject and subject_dir. + + Raises + ------ + AssertionError + """ + if not self._file.is_absolute() or not self._file.exists(): + raise AssertionError( + f"The ImportedMeasures {self.name} is defined for import, but the " + f"associated measure file {self._file} is not an absolute path or " + f"does not exist and no subjects dir or subject id are defined." + ) + + def get_vox_vol(self) -> float: + """ + Returns the voxel volume. + + Returns + ------- + float + The voxel volume associated with the imported measure. + + Raises + ------ + RuntimeError + If the voxel volume was not defined. + """ + if self._vox_vol is None: + raise RuntimeError(f"The voxel volume of {self} has never been specified.") + return self._vox_vol + + def set_vox_vol(self, value: float): + self._vox_vol = value + + def read_subject(self, subject_dir: Path) -> bool: + if super().read_subject(subject_dir): + vox_vol_tup = self._data.get("vox_vol", None) + if isinstance(vox_vol_tup, tuple) and len(vox_vol_tup) > 2: + self._vox_vol = vox_vol_tup[2] + return True + return False + + +class SurfaceMeasure(Measure["lapy.TriaMesh"], metaclass=abc.ABCMeta): + """ + Class to implement default Surface io. + """ + + read_file = staticmethod(read_mesh_file) + + def __init__( + self, + surface_file: Path, + name: str, + description: str, + unit: UnitString, + read_mesh: Optional[ReadFileHook["lapy.TriaMesh"]] = None, + ): + super().__init__( + surface_file, + name, + description, + unit, + self.read_file if read_mesh is None else read_mesh, + ) + + def __str__(self) -> str: + return f"{type(self).__name__}(surface_file={self._file})" + + def _parsable_args(self) -> list[str]: + return ["surface_file"] + + def set_args(self, surface_file: str | None = None, **kwargs: str) -> None: + if surface_file is not None: + kwargs["file"] = surface_file + return super().set_args(**kwargs) + + +class SurfaceHoles(SurfaceMeasure): + """Class to compute surfaces holes for surfaces.""" + + def _compute(self) -> int: + return int(1 - self._data.euler() / 2) + + def help(self) -> str: + return super().help() + f"surface holes from {self._file}" + + +class SurfaceVolume(SurfaceMeasure): + """Class to compute surface volume for surfaces.""" + + def _compute(self) -> float: + return self._data.volume() + + def help(self) -> str: + return super().help() + f"volume from {self._file}" + + +class PVMeasure(AbstractMeasure): + """Class to compute volume for segmentations (includes PV-correction).""" + + read_file = None + + def __init__( + self, + classes: ClassesType, + name: str, + description: str, + unit: Literal["mm^3"] = "mm^3", + ): + if unit != "mm^3": + raise ValueError("unit must be mm^3 for PVMeasure!") + self._classes = classes + super().__init__(name, description, unit) + self._pv_value = None + + @property + def vox_vol(self) -> float: + return self._vox_vol + + @vox_vol.setter + def vox_vol(self, v: float): + self._vox_vol = v + + def labels(self) -> list[int]: + return list(self._classes) + + def update_data(self, value: "pd.Series"): + self._pv_value = value + + def __call__(self) -> float: + if self._pv_value is None: + raise RuntimeError( + f"The partial volume of {self._name} has not been updated in the " + f"PVMeasure object yet!" + ) + col = "NVoxels" if self.unit == "unitless" else "Volume_mm3" + return self._pv_value[col].item() + + def _parsable_args(self) -> list[str]: + return ["classes"] + + def set_args(self, classes: str | None = None, **kwargs: str) -> None: + if classes is not None: + self._classes = classes + return super().set_args(**kwargs) + + def __str__(self) -> str: + return f"PVMeasure(classes={list(self._classes)})" + + def help(self) -> str: + help_str = f"partial volume of {format_classes(self._classes)} in seg file" + return super().help() + help_str + + +def format_classes(_classes: Iterable[int]) -> str: + """ + Formats an iterable of classes. This compresses consecutive integers into ranges. + >>> format_classes([1, 2, 3, 6]) # '1-3,6' + + Parameters + ---------- + _classes : Iterable[int] + An iterable of integers. + + Returns + ------- + A string of sorted integers and integer ranges, '()' if iterable is empty, or just + the string conversion of _classes, if _classes is not an iterable. + + Notes + ----- + This function will likely be moved to a different file. + """ + # TODO move this function to a more appropriate module + if not isinstance(_classes, Iterable): + return str(_classes) + from itertools import pairwise + sorted_list = list(sorted(_classes)) + if len(sorted_list) == 0: + return "()" + prev = "" + out = str(sorted_list[0]) + + for a, b in pairwise(sorted_list): + if a != b - 1: + out += f"{prev},{b}" + prev = "" + else: + prev = f"-{b}" + return out + prev + + +class VolumeMeasure(Measure[ImageTuple]): + """ + Counts Voxels belonging to a class or condition. + """ + + read_file = staticmethod(read_volume_file) + + def __init__( + self, + segfile: Path, + classes_or_cond: ClassesOrCondType, + name: str, + description: str, + unit: UnitString = "unitless", + read_file: Optional[ReadFileHook[ImageTuple]] = None, + ): + if callable(classes_or_cond): + self._classes: Optional[ClassesType] = None + self._cond: _ToBoolCallback = classes_or_cond + else: + if len(classes_or_cond) == 0: + raise ValueError(f"No operation passed to {type(self).__name__}.") + self._classes = classes_or_cond + from functools import partial + self._cond = partial(mask_in_array, items=self._classes) + if unit not in ["unitless", "mm^3"]: + raise ValueError("unit must be either 'mm^3' or 'unitless' for " + + type(self).__name__) + super().__init__(segfile, name, description, unit, + self.read_file if read_file is None else read_file) + + def get_vox_vol(self) -> float: + return np.prod(self._data[0].header.get_zooms()).item() + + def _compute(self) -> int | float: + if not isinstance(self._data, tuple) or len(self._data) != 2: + raise self._load_error("data") + vox_vol = 1 if self._unit == "unitless" else self.get_vox_vol() + return np.sum(self._cond(self._data[1]), dtype=int).item() * vox_vol + + def _parsable_args(self) -> list[str]: + return ["segfile", "classes"] + + def _set_classes(self, classes: str | None, attr_name: str, cond_name: str) -> None: + """Helper method for set_args.""" + if classes is not None: + from functools import partial + _classes = re.split("\\s+", classes.lstrip("[ ").rstrip("] ")) + items = list(map(int, _classes)) + setattr(self, attr_name, items) + setattr(self, cond_name, partial(mask_in_array, items=items)) + + def set_args( + self, + segfile: str | None = None, + classes: str | None = None, + **kwargs: str, + ) -> None: + if segfile is not None: + kwargs["file"] = segfile + self._set_classes(classes, "_classes", "_cond") + return super().set_args(**kwargs) + + def __str__(self) -> str: + return f"{type(self).__name__}(segfile={self._file}, {self._param_string()})" + + def help(self) -> str: + return f"{self._name}={self._param_help()} in {self._file}" + + def _param_help(self, prefix: str = ""): + """Helper method for format classes and cond.""" + cond = getattr(self, prefix + "_cond") + classes = getattr(self, prefix + "_classes") + return prefix + (f"cond={cond}" if classes is None else format_classes(classes)) + + def _param_string(self, prefix: str = ""): + """Helper method to convert classes and cond to string.""" + cond = getattr(self, prefix + "_cond") + classes = getattr(self, prefix + "_classes") + return prefix + (f"cond={cond}" if classes is None else f"classes={classes}") + + +class MaskMeasure(VolumeMeasure): + + def __init__( + self, + maskfile: Path, + name: str, + description: str, + unit: UnitString = "unitless", + threshold: float = 0.5, + # sign: MaskSign = "abs", frame: int = 0, + # erode: int = 0, invert: bool = False, + read_file: Optional[ReadFileHook[ImageTuple]] = None, + ): + self._threshold: float = threshold + # self._sign: MaskSign = sign + # self._invert: bool = invert + # self._frame: int = frame + # self._erode: int = erode + super().__init__(maskfile, self.mask, name, description, unit, read_file) + + def mask(self, data: "npt.NDArray[int]") -> "npt.NDArray[bool]": + """Generates a mask from data similar to mri_binarize + erosion.""" + # if self._sign == "abs": + # data = np.abs(data) + # elif self._sign == "neg": + # data = -data + out = np.greater(data, self._threshold) + # if self._invert: + # out = np.logical_not(out) + # if self._erode != 0: + # from scipy.ndimage import binary_erosion + # binary_erosion(out, iterations=self._erode, output=out) + return out + + def set_args( + self, + maskfile: Path | None = None, + threshold: float | None = None, + **kwargs: str, + ) -> None: + if threshold is not None: + self._threshold = float(threshold) + if maskfile is not None: + kwargs["file"] = maskfile + return super().set_args(**kwargs) + + def _parsable_args(self) -> list[str]: + return ["maskfile", "threshold"] + + def __str__(self) -> str: + return ( + f"{type(self).__name__}(maskfile={self._file}, threshold={self._threshold})" + ) + + def _param_help(self, prefix: str = ""): + return f"voxel > {self._threshold}" + + +AnyParentsTuple = tuple[float, AnyMeasure] +ParentsTuple = tuple[float, AnyMeasure] + + +class TransformMeasure(Measure, metaclass=abc.ABCMeta): + read_file = staticmethod(read_transform_file) + + def __init__( + self, + lta_file: Path, + name: str, + description: str, + unit: str, + read_lta: Optional[ReadFileHook["npt.NDArray[float]"]] = None, + ): + super().__init__( + lta_file, + name, + description, + unit, + self.read_file if read_lta is None else read_lta, + ) + + def _parsable_args(self) -> list[str]: + return ["lta_file"] + + def set_args(self, lta_file: str | None = None, **kwargs: str) -> None: + if lta_file is not None: + kwargs["file"] = lta_file + return super().set_args(**kwargs) + + def __str__(self) -> str: + return f"{type(self).__name__}(lta_file={self._file})" + + +class ETIVMeasure(TransformMeasure): + """ + Compute the eTIV based on the freesurfer talairach registration and lta. + + Notes + ----- + Reimplemneted from freesurfer/mri_sclimbic_seg + https://github.com/freesurfer/freesurfer/blob/ + 3296e52f8dcffa740df65168722b6586adecf8cc/mri_sclimbic_seg/mri_sclimbic_seg#L627 + """ + + def __init__( + self, + lta_file: Path, + name: str, + description: str, + unit: str, + read_lta: Optional[ReadFileHook["LTADict"]] = None, + etiv_scale_factor: float | None = None, + ): + if etiv_scale_factor is None: + self._etiv_scale_factor = 1948106. # 1948.106 cm^3 * 1e3 mm^3/cm^3 + else: + self._etiv_scale_factor = etiv_scale_factor + super().__init__(lta_file, name, description, unit, read_lta) + + def _parsable_args(self) -> list[str]: + return super()._parsable_args() + ["etiv_scale_factor"] + + def set_args(self, etiv_scale_factor: str | None = None, **kwargs: str) -> None: + if etiv_scale_factor is not None: + self._etiv_scale_factor = float(etiv_scale_factor) + return super().set_args(**kwargs) + + def _compute(self) -> float: + # this scale factor is a fixed number derived by freesurfer + return self._etiv_scale_factor / np.linalg.det(self._data).item() + + def help(self) -> str: + return super().help() + f"eTIV from {self._file}" + + def __str__(self) -> str: + return f"{super().__str__()[:-1]}, etiv_scale_factor={self._etiv_scale_factor})" + + +class DerivedMeasure(AbstractMeasure): + + def __init__( + self, + parents: Iterable[tuple[float, AnyMeasure] | AnyMeasure], + name: str, + description: str, + unit: str = "from parents", + operation: DerivedAggOperation = "sum", + measure_host: Optional[dict[str, AbstractMeasure]] = None, + ): + """ + Create the Measure, which depends on other measures, called parent measures. + + Parameters + ---------- + parents : Iterable[tuple[float, AbstractMeasure] | AbstractMeasure] + Iterable of either the measures (or a tuple of a float and a measure), the + float is the factor by which the value of the respective measure gets + weighted and defaults to 1. + name : str + Name of the Measure. + description : str + Description text of the measure + unit : str, optional + Unit of the measure, typically 'mm^3' or 'unitless', autogenerated from + parents' unit. + operation : "sum", "ratio", "by_vox_vol", optional + How to aggregate multiple `parents`, default = 'sum' + 'ratio' only supports exactly 2 parents. + 'by_vox_vol' only supports exactly one parent. + measure_host : dict[str, AbstractMeasure], optional + A dict-like to provide AbstractMeasure objects for strings. + """ + + def to_tuple( + value: tuple[float, AnyMeasure] | AnyMeasure, + ) -> tuple[float, AnyMeasure]: + if isinstance(value, Sequence) and not isinstance(value, str): + if len(value) != 2: + raise ValueError("A tuple was not length 2.") + factor, measure = value + else: + factor, measure = 1., value + + if not isinstance(measure, (str, AbstractMeasure)): + raise ValueError(f"Expected a str or AbstractMeasure, not " + f"{type(measure).__name__}!") + if not isinstance(factor, float): + factor = float(factor) + return factor, measure + + self._parents: list[AnyParentsTuple] = [to_tuple(p) for p in parents] + if len(self._parents) == 0: + raise ValueError("No parents defined in DerivedMeasure.") + self._measure_host = measure_host + if operation in ("sum", "ratio", "by_vox_vol"): + self._operation: DerivedAggOperation = operation + else: + raise ValueError("operation must be 'sum', 'ratio' or 'by_vox_vol'.") + super().__init__(name, description, unit) + + @property + def unit(self) -> str: + """ + Property to access the unit attribute, also implements auto-generation of unit, + if the stored unit is 'from parents'. + + Returns + ------- + str + A string that identifies the unit of the Measure. + + Raises + ------ + RuntimeError + If unit is 'from parents' and some parent measures are inconsistent with + each other. + """ + if self._unit == "from parents": + units = list(map(lambda x: x.unit, self.parents)) + if self._operation == "sum": + if len(units) == 0: + raise ValueError("DerivedMeasure has no parent measures.") + elif len(units) == 1 or all(units[0] == u for u in units[1:]): + return units[0] + elif self._operation == "ratio": + if len(units) != 2: + raise self.invalid_len_ratio() + elif units[0] == units[1]: + return "unitless" + elif units[1] == "unitless": + return units[0] + elif self._operation == "by_vox_vol": + if len(units) != 1: + raise self.invalid_len_vox_vol() + elif units[0] == "mm^3": + return "unitless" + else: + raise RuntimeError("Invalid value of parent, must be mm^3, but " + f"was {units[0]}.") + raise RuntimeError( + f"unit is set to auto-generate from parents, but the parents' units " + f"are not consistent: {units}!" + ) + else: + return super().unit + + def invalid_len_ratio(self) -> RuntimeError: + return RuntimeError(f"Invalid number of parents ({len(self._parents)}) for " + f"operation 'ratio'.") + + def invalid_len_vox_vol(self) -> RuntimeError: + return RuntimeError(f"Invalid number of parents ({len(self._parents)}) for " + f"operation 'by_vox_vol'.") + + @property + def parents(self) -> Iterable[AbstractMeasure]: + """Iterable of the measures this measure depends on.""" + return (p for _, p in self.parents_items()) + + def parents_items(self) -> Iterable[tuple[float, AbstractMeasure]]: + """Iterable of the measures this measure depends on.""" + return ((f, self._measure_host[p] if isinstance(p, str) else p) + for f, p in self._parents) + + def __read_subject(self, subject_dir: Path) -> bool: + """Default implementation for the read_subject_on_parents function hook.""" + return any(m.read_subject(subject_dir) for m in self.parents) + + @property + def read_subject_on_parents(self) -> Callable[[Path], bool]: + """read_subject_on_parents function hook property""" + if (self._measure_host is not None and + hasattr(self._measure_host, "read_subject_parents")): + from functools import partial + return partial(self._measure_host.read_subject_parents, self.parents) + else: + return self.__read_subject + + def read_subject(self, subject_dir: Path) -> bool: + """ + Perform IO required to compute/fill the Measure. Will trigger the + read_subject_on_parents function hook to populate the values of parent measures. + + Parameters + ---------- + subject_dir : Path + Path to the directory of the subject_dir (often subject_dir/subject_id). + + Returns + ------- + bool + Whether there was an update. + + Notes + ----- + Might trigger a race condition if the function hook `read_subject_on_parents` + depends on this method finishing first, e.g. because of thread availability. + """ + if super().read_subject(subject_dir): + return self.read_subject_on_parents(self._subject_dir) + return False + + def __call__(self) -> int | float: + """ + Compute dependent measures and accumulate them according to the operation. + """ + factor_value = [(s, m()) for s, m in self.parents_items()] + isint = all(isinstance(v, int) for _, v in factor_value) + isint &= all(np.isclose(s, np.round(s)) for s, _ in factor_value) + values = [s * v for s, v in factor_value] + if self._operation == "sum": + # sum should be an int, if all contributors are int + # and all factors are integers (but not necessarily int) + out = np.sum(values) + target_type = int if isint else float + return target_type(out) + elif self._operation == "by_vox_vol": + if len(self._parents) != 1: + raise self.invalid_len_vox_vol() + vox_vol = self.get_vox_vol() + if isinstance(vox_vol, _DefaultFloat): + logging.getLogger(__name__).warning( + f"The vox_vol in {self} was unexpectedly not initialized; using " + f"{vox_vol}!" + ) + # ratio should always be float / could be partial voxels + return float(values[0]) / vox_vol + else: # operation == "ratio" + if len(self._parents) != 2: + raise self.invalid_len_ratio() + # ratio should always be float + return float(values[0]) / float(values[1]) + + def get_vox_vol(self) -> float | None: + """ + Return the voxel volume of the first parent measure. + + Returns + ------- + float, None + voxel volume of the first parent + """ + _types = (VolumeMeasure, DerivedMeasure) + _type = ImportedMeasure + fallback = None + for p in self.parents: + if isinstance(p, _types) and (_vvol := p.get_vox_vol()) is not None: + return _vvol + if isinstance(p, _type) and (_vvol := p.get_vox_vol()) is not None: + if isinstance(_vvol, _DefaultFloat): + fallback = _vvol + else: + return _vvol + return fallback + + def _parsable_args(self) -> list[str]: + return ["parents", "operation"] + + def set_args( + self, + parents: str | None = None, + operation: str | None = None, + **kwargs: str, + ) -> None: + if parents is not None: + pat = re.compile("^(\\d+\\.?\\d*\\s+)?(\\s.*)") + stripped = parents.lstrip("[ ").rstrip("] ") + + def parse(p: str) -> tuple[float, str]: + hit = pat.match(p) + if hit is None: + return 1., p + return 1. if hit.group(1).strip() else float(hit.group(1)), hit.group(2) + + self._parents = list(map(parse, re.split("\\s+", stripped))) + if operation is not None: + from typing import get_args as args + if operation in args(DerivedAggOperation): + self._operation = operation + else: + raise ValueError(f"operation can only be {args(DerivedAggOperation)}") + return super().set_args(**kwargs) + + def __str__(self) -> str: + return f"DerivedMeasure(parents={self._parents}, operation={self._operation})" + + def help(self) -> str: + sign = {True: "+", False: "-"} + + def format_factor(f: float) -> str: + return f"{sign[f >= 0]} " + ((str(abs(f)) + " ") if abs(f) != 1. else '') + + def format_parent(measure: str | AnyMeasure) -> str: + if isinstance(measure, str): + measure = self._measure_host[measure] + return measure if isinstance(measure, str) else measure.help() + + if self._operation == "sum": + par = "".join(f" {format_factor(f)}({format_parent(p)})" + for f, p in self._parents) + return par.lstrip(' +') + elif self._operation == "by_vox_vol": + f, measure = self._parents[0] + return f"{sign[f >= 0]} {format_factor(f)} [{format_parent(measure)}]" + elif self._operation == "ratio": + f = self._parents[0][0] / self._parents[1][0] + return (f" {sign[f >= 0]} {format_factor(f)} (" + + ") / (".join(format_parent(p[1]) for p in self._parents) + ")") + else: + return f"invalid operation {self._operation}" + + +class VoxelClassGenerator(Protocol): + """ + Generator for voxel-based metric Measures. + """ + + def __call__( + self, + classes: Sequence[int], + name: str, + description: str, + unit: str, + ) -> PVMeasure | VolumeMeasure: + ... + + +def format_measure(key: str, data: MeasureTuple) -> str: + value = data[2] if isinstance(data[2], int) else ("%.6f" % data[2]) + return f"# Measure {key}, {data[0]}, {data[1]}, {value}, {data[3]}" + + +class Manager(dict[str, AbstractMeasure]): + _PATTERN_NO_ARGS = re.compile("^\\s*([^(]+?)\\s*$") + _PATTERN_ARGS = re.compile("^\\s*([^(]+)\\(\\s*([^)]*)\\s*\\)\\s*$") + _PATTERN_DELIM = re.compile("\\s*,\\s*") + + _compute_futures: list[Future] + __DEFAULT_MEASURES = ( + "BrainSeg", + "BrainSegNotVent", + "VentricleChoroidVol", + "lhCortex", + "rhCortex", + "Cortex", + "lhCerebralWhiteMatter", + "rhCerebralWhiteMatter", + "CerebralWhiteMatter", + "SubCortGray", + "TotalGray", + "SupraTentorial", + "SupraTentorialNotVent", + "Mask", + "BrainSegVol-to-eTIV", + "MaskVol-to-eTIV", + "lhSurfaceHoles", + "rhSurfaceHoles", + "SurfaceHoles", + "EstimatedTotalIntraCranialVol", + ) + + def __init__( + self, + measures: Sequence[tuple[bool, str]], + measurefile: Optional[Path] = None, + segfile: Optional[Path] = None, + on_missing: Literal["fail", "skip", "fill"] = "fail", + executor: Optional[Executor] = None, + legacy_freesurfer: bool = False, + aseg_replace: Optional[Path] = None, + ): + """ + + Parameters + ---------- + measures : Sequence[tuple[bool, str]] + The measures to be included as whether it is computed and name/measure str. + measurefile : Path, optional + The path to the file to import measures from (other stats file, absolute or + relative to subject_dir). + segfile : Path, optional + The path to the file to use for segmentation (other stats file, absolute or + relative to subject_dir). + on_missing : Literal["fail", "skip", "fill"], optional + behavior to follow if a requested measure does not exist in path. + executor : concurrent.futures.Executor, optional + thread pool to parallelize io + legacy_freesurfer : bool, default=False + FreeSurfer compatibility mode. + """ + from concurrent.futures import ThreadPoolExecutor, Future + from copy import deepcopy + + def _check_measures(x): + return not (isinstance(x, tuple) and len(x) == 2 or + isinstance(x[0], bool) or isinstance(x[1], str)) + super().__init__() + self._default_measures = deepcopy(self.__DEFAULT_MEASURES) + if not isinstance(measures, Sequence) or any(map(_check_measures, measures)): + raise ValueError("measures must be sequences of str.") + if executor is None: + self._executor = ThreadPoolExecutor(8) + elif isinstance(executor, ThreadPoolExecutor): + self._executor = executor + else: + raise TypeError( + "executor must be a futures.concurrent.ThreadPoolExecutor to ensure " + "proper multitask behavior." + ) + self._io_futures: list[Future] = [] + self.__update_context: list[AbstractMeasure] = [] + self._on_missing = on_missing + self._import_all_measures: list[Path] = [] + self._subject_all_imported: list[Path] = [] + self._exported_measures: list[str] = [] + self._cache: dict[Path, Future[AnyBufferType] | AnyBufferType] = {} + # self._lut: Optional[pd.DataFrame] = None + self._fs_compat: bool = legacy_freesurfer + self._seg_from_file = Path("mri/aseg.mgz") + if aseg_replace: + # explicitly defined a file to reduce the aseg for segmentation mask with + logging.getLogger(__name__).info( + f"Replacing segmentation volume to compute volume measures from with " + f"the explicitly defined {aseg_replace}." + ) + self._seg_from_file = Path(aseg_replace) + elif not self._fs_compat and segfile and Path(segfile) != self._seg_from_file: + # not in freesurfer compatibility mode, so implicitly use segfile + logging.getLogger(__name__).info( + f"Replacing segmentation volume to compute volume measures from with " + f"the segmentation file {segfile}." + ) + self._seg_from_file = Path(segfile) + + import_kwargs = {"vox_vol": _DefaultFloat(1.0)} + if any(filter(lambda x: x[0], measures)): + if measurefile is None: + raise ValueError( + "Measures defined to import, but no measurefile specified. " + "A default must always be defined." + ) + import_kwargs["measurefile"] = Path(measurefile) + import_kwargs["read_file"] = self.make_read_hook(read_measure_file) + import_kwargs["read_file"](Path(measurefile), blocking=False) + for is_imported, measure_string in measures: + if is_imported: + self.add_imported_measure(measure_string, **import_kwargs) + else: + self.add_computed_measure(measure_string) + self.instantiate_measures(self.values()) + + @property + def executor(self) -> Executor: + return self._executor + + # @property + # def lut(self) -> Optional["pd.DataFrame"]: + # return self._lut + # + # @lut.setter + # def lut(self, lut: Optional["pd.DataFrame"]): + # self._lut = lut + + def assert_measure_need_subject(self) -> None: + """ + Assert whether the measure expects a definition of the subject_dir. + + Raises + ------ + AssertionError + """ + any_computed = False + for key, measure in self.items(): + if isinstance(measure, DerivedMeasure): + pass + elif isinstance(measure, ImportedMeasure): + measure.assert_measurefile_absolute() + else: + any_computed = True + if any_computed: + raise AssertionError( + "Computed measures are defined, but no subjects dir or subject id." + ) + + def instantiate_measures(self, measures: Iterable[AbstractMeasure]) -> None: + """ + Make sure all measures that dependent on `measures` are instantiated. + """ + for measure in list(measures): + if isinstance(measure, DerivedMeasure): + self.instantiate_measures(measure.parents) + + def add_imported_measure(self, measure_string: str, **kwargs) -> None: + """ + Add an imported measure from the measure_string definition and default + measurefile. + + Parameters + ---------- + measure_string : str + Definition of the measure. + + Other Parameters + ---------------- + measurefile : Path + Path to the default measurefile to import from (ImportedMeasure argument). + read_file : ReadFileHook[dict[str, MeasureTuple]] + Function handle to read and parse the file (argument to ImportedMeasure). + vox_vol: float, optional + The voxel volume to associate the measure with. + + Raises + ------ + RuntimeError + If trying to replace a computed Measure of the same key. + """ + # currently also extracts args, this maybe should be removed for simpler code + key, args = self.extract_key_args(measure_string) + if key == "all": + _mfile = kwargs["measurefile"] if len(args) == 0 else Path(args[0]) + self._import_all_measures.append(_mfile) + elif key not in self.keys() or isinstance(self[key], ImportedMeasure): + # note: name, description and unit are always updated from the input file + self[key] = ImportedMeasure(key, **kwargs) + # parse the arguments (inplace) + self[key].parse_args(*args) + self._exported_measures.append(key) + else: + raise RuntimeError( + "Illegal operation: Trying to replace the computed measure at " + f"{key} ({self[key]}) with an imported measure." + ) + + def add_computed_measure( + self, + measure_string: str, + ) -> None: + """Add a computed measure from the measure_string definition.""" + # currently also extracts args, this maybe should be removed for simpler code + key, args = self.extract_key_args(measure_string) + # also overwrite prior definition + if key in self._exported_measures: + self[key] = self.default(key) + else: + self._exported_measures.append(key) + # load the default config of the measure and copy, overwriting other measures + # with the same key (only keep computed versions or the last) parse the + # arguments (inplace) + self[key].parse_args(*args) + + def __getitem__(self, key: str) -> AbstractMeasure: + """ + Get the value of the key. + + Parameters + ---------- + key : str + A string naming the Measure, may also include extra parameters as format + '()', e.g. 'Mask(maskfile=/path/to/mask.mgz)'. + + Returns + ------- + AbstractMeasure + The measure associated with the '' + """ + if "(" in key: + key, args = key.split("(", 1) + args = list(map(str.strip, args.rstrip(") ").split(","))) + else: + args = [] + try: + out = super().__getitem__(key) + except KeyError: + out = self.default(key) + if out is not None: + self[key] = out + else: + raise + if len(args) > 0: + out.parse_args(*args) + return out + + def start_read_subject(self, subject_dir: Path) -> None: + """ + Start the threads to read the subject in subject_dir, pairs with + `wait_read_subject`. + + Parameters + ---------- + subject_dir : Path + The path to the directory of the subject (with folders 'mri', 'stats', ...). + """ + if len(self._io_futures) != 0: + raise RuntimeError("Did not process/wait on finishing the processing for " + "the previous start_read_subject run. Needs call to " + "`wait_read_subject`.") + self.__update_context = [] + self._subject_all_imported = [] + read_file = self.make_read_hook(read_measure_file) + for file in self._import_all_measures: + path = file if file.is_absolute() else subject_dir / file + read_file(path, blocking=False) + self._subject_all_imported.append(path) + self.read_subject_parents(self.values(), subject_dir, False) + + @contextmanager + def with_subject(self, subjects_dir: Path | None, subject_id: str | None) -> None: + """ + Contextmanager for the `start_read_subject` and the `wait_read_subject` pair. + + If one value is None, it is assumed the subject_dir and subject_id are not + needed, for example because all file names are given by absolute paths. + + Parameters + ---------- + subjects_dir : Path, None + The path to the directory of the subject (with folders 'mri', 'stats', ...). + subject_id : str, None + The subject_id identifying folder of the subjects_dir. + + Raises + ------ + AssertionError + If subjects_dir and or subject_id are needed. + """ + if subjects_dir is None or subject_id is None: + yield self.assert_measure_need_subject() + # no reading the subject required, we have no measures to include + return + else: + # the subject is defined, we read it. + yield self.start_read_subject(subjects_dir / subject_id) + return self.wait_read_subject() + + def wait_read_subject(self) -> None: + """ + Wait for all threads to finish reading the 'current' subject. + + Raises + ------ + Exception + The first exception encountered during the read operation. + """ + for f in self._io_futures: + exception = f.exception() + if exception is not None: + raise exception + self._io_futures.clear() + vox_vol = None + + def check_needs_init(m: AbstractMeasure) -> bool: + return isinstance(m, ImportedMeasure) and isinstance(m.get_vox_vol(), + _DefaultFloat) + + # and an ImportedMeasure is present, but not initialized + for m in filter(check_needs_init, self.values()): + # lazily load a value for vox_vol + if vox_vol is None: + # if the _seg_from_file file is loaded into the cache (should be) + if self._seg_from_file in self._cache: + read_func = self.make_read_hook(read_volume_file) + img, _ = read_func(self._seg_from_file, blocking=True) + vox_vol = np.prod(img.header.get_zooms()) + if vox_vol is not None: + m.set_vox_vol(vox_vol) + + def read_subject_parents( + self, + measures: Iterable[AbstractMeasure], + subject_dir: Path, + blocking: bool = False, + ) -> True: + """ + Multi-threaded iteration through measures and application of read_subject, also + implementation for the read_subject_on_parents function hook. Guaranteed to + return + independent of state and thread availability to avoid a race condition. + + Parameters + ---------- + measures : Iterable[AbstractMeasure] + iterable of Measures to read + subject_dir : Path + Path to the subject directory (often subjects_dir/subject_id). + blocking : bool, optional + whether the execution should be parallel or not (default: False/parallel). + + Returns + ------- + True + """ + + def _read(measure: AbstractMeasure) -> bool: + """Callback so files for measures are loaded in other threads.""" + return measure.read_subject(subject_dir) + + _update_context = set( + filter(lambda m: m not in self.__update_context, measures) + ) + # __update_context is the structure that holds measures that have read_subject + # already called / submitted to the executor + self.__update_context.extend(_update_context) + for x in _update_context: + # DerivedMeasure.read_subject calls Manager.read_subject_parents (this + # method) to read the data from dependent measures (through the callback + # DerivedMeasure.read_subject_on_parents, and DerivedMeasure.measure_host). + if blocking or isinstance(x, DerivedMeasure): + x.read_subject(subject_dir) + else: + # calls read_subject on all measures, redundant io operations are + # handled/skipped through Manager.make_read_hook and the internal + # caching of files within the _cache attribute of Manager. + self._io_futures.append(self._executor.submit(_read, x)) + return True + + def extract_key_args(self, measure: str) -> tuple[str, list[str]]: + """ + Extract the name and options from a string like '()'. + + The '' is optional and is similar to python parameters. It starts + with numbered parameters, followed by key-value pairs. + Examples are: + - 'Mask(mri/aseg.mgz)' + returns: ('BrainSeg', ['mri/aseg.mgz', 'classes=[2, 4]']) + - 'TotalGray(mri/aseg.mgz, classes=[2, 4])' + returns: ('BrainSeg', ['mri/aseg.mgz', 'classes=[2, 4]']) + - 'BrainSeg(segfile=mri/aseg.mgz, classes=[2, 4])' + returns: ('BrainSeg', ['segfile=mri/aseg.mgz', 'classes=[2, 4]']) + + Parameters + ---------- + measure : str + The measure string of the format '' or '()'. + + Returns + ------- + key : str + the name of the measure + args : list[str] + a list of options + + Raises + ------ + ValueError + If the string `measure` does not conform to the format requirements. + """ + hits_no_args = self._PATTERN_NO_ARGS.match(measure) + if hits_no_args is not None: + key = hits_no_args.group(1) + args = [] + elif (hits_args := self._PATTERN_ARGS.match(measure)) is not None: + key = hits_args.group(1) + args = self._PATTERN_DELIM.split(hits_args.group(2)) + else: + extra = "" + if any(q in measure for q in "\"'"): + extra = ", watch out for quotes" + raise ValueError(f"Invalid Format of Measure \"{measure}\"{extra}!") + return key, args + + def make_read_hook( + self, + read_func: Callable[[Path], T_BufferType], + ) -> ReadFileHook[T_BufferType]: + """ + Wraps an io function to buffer results, multi-thread calls, etc. + + Parameters + ---------- + read_func : Callable[[Path], T_BufferType] + Function to read Measure entries/ images/ surfaces from a file. + + Returns + ------- + wrapped_func : ReadFileHook[T_BufferType] + The returned function takes a path and whether to wait for the io to finish. + file : Path + the path to the read from (path can be used for buffering) + blocking : bool, optional + do not return the data, do not wait for the io to finish, just preload + (default: False) + The function returns None or the output of the wrapped function. + """ + + def read_wrapper(file: Path, blocking: bool = True) -> Optional[T_BufferType]: + out = self._cache.get(file, None) + if out is None: + # not already in cache + if blocking: + out = read_func(file) + else: + out = self._executor.submit(read_func, file) + self._cache[file] = out + if not blocking: + return + elif isinstance(out, Future): + self._cache[file] = out = out.result() + return out + + return read_wrapper + + def clear(self): + """ + Clear the file buffers. + """ + self._cache = {} + + def update_measures(self) -> dict[str, float | int]: + """ + Get the values to all measures (including imported via 'all'). + + Returns + ------- + dict[str, Union[float, int]] + A dictionary of '' (the Measure key) and the associated value. + """ + m = {key: v[2] for key, v in self.get_imported_all_measures().items()} + m.update({key: self[key]() for key in self._exported_measures}) + return m + + def print_measures(self, file: Optional[TextIO] = None) -> None: + """ + Print the measures to stdout or file. + + Parameters + ---------- + file: TextIO, optional + The file object to write to. If None, writes to stdout. + """ + kwargs = {} if file is None else {"file": file} + for line in self.format_measures(): + print(line, **kwargs) + + def get_imported_all_measures(self) -> dict[str, MeasureTuple]: + """ + Get the measures imported through the 'all' keyword. + + Returns + ------- + dict[str, MeasureTuple] + A dictionary of Measure keys and tuples of name, description, value, unit. + """ + if len(self._subject_all_imported) == 0: + return {} + measures = {} + read_file = self.make_read_hook(ImportedMeasure.read_file) + for path in self._subject_all_imported: + measures.update(read_file(path)) + return measures + + def format_measures( + self, /, + fmt_func: Callable[[str, MeasureTuple], str] = format_measure, + ) -> Iterable[str]: + """ + Formats all measures as strings and returns them as an iterable of str. + + In the output, measures are ordered in the order they are added to the Manager + object. Finally, the "all"-imported Measures are appended. + + Parameters + ---------- + fmt_func: callable, default=fmt_measure + Function to format the key and a MeasureTuple object into a string. + + Returns + ------- + Iterable[str] + An iterable of the measure strings. + """ + measures = {key: self[key].as_tuple() for key in self._exported_measures} + for k, v in self.get_imported_all_measures().items(): + measures.setdefault(k, v) + + return map(lambda x: fmt_func(*x), measures.items()) + + @property + def default_measures(self) -> Iterable[str]: + """ + Iterable over measures typically included stats files in correct order. + + Returns + ------- + Iterable[str] + An ordered iterable of the default Measure keys. + """ + return self._default_measures + + @default_measures.setter + def default_measures(self, values: Iterable[str]): + """ + Sets the iterable over measure keys in correct order. + + Parameters + ---------- + values : Iterable[str] + An ordered iterable of the default Measure keys. + """ + self._default_measures = values + + @property + def voxel_class(self) -> VoxelClassGenerator: + """ + A callable initializing a Volume-based Measure object with the legacy mode. + + Returns + ------- + type[AbstractMeasure] + A callable to create an object to perform a Volume-based Measure. + """ + from functools import partial + if self._fs_compat: + return partial( + VolumeMeasure, + self._seg_from_file, + read_file=self.make_read_hook(VolumeMeasure.read_file), + ) + else: # FastSurfer compat == None + return partial(PVMeasure) + + def default(self, key: str) -> AbstractMeasure: + """ + Returns the default Measure object for the measure with key `key`. + + Parameters + ---------- + key : str + The key name of the Measure. + + Returns + ------- + AbstractMeasure + The Measure object initialized with default values. + + Supported keys are: + - `lhSurfaceHoles`, `rhSurfaceHoles`, and `SurfaceHoles` + The number of holes in the surfaces. + - `lhPialTotal`, and `rhPialTotal` + The volume enclosed in the pial surfaces. + - `lhWhiteMatterVol`, and `rhWhiteMatterVol` + The Volume of the white matter in the segmentation (incl. lateralized + WM-hypo). + - `lhWhiteMatterTotal`, and `rhWhiteMatterTotal` + The volume enclosed in the white matter surfaces. + - `lhCortex`, `rhCortex`, and `Cortex` + The volume between the pial and the white matter surfaces. + - `CorpusCallosumVol` + The volume of the corpus callosum in the segmentation. + - `lhWM-hypointensities`, and `rhWM-hypointensities` + The volume of unlateralized the white matter hypointensities in the + segmentation, but lateralized by neigboring voxels + (FreeSurfer uses talairach coordinates to re-lateralize). + - `lhCerebralWhiteMatter`, `rhCerebralWhiteMatter`, and `CerebralWhiteMatter` + The volume of the cerebral white matter in the segmentation (including corpus + callosum split evenly into left and right and white matter and WM-hypo). + - `CerebellarGM` + The volume of the cerbellar gray matter in the segmentation. + - `CerebellarWM` + The volume of the cerbellar white matter in the segmentation. + - `SubCortGray` + The volume of the subcortical gray matter in the segmentation. + - `TotalGray` + The total gray matter volume in the segmentation. + - `TFFC` + The volume of the 3rd-5th ventricles and CSF in the segmentation. + - `VentricleChoroidVol` + The volume of the choroid plexus and inferiar and lateral ventricles and CSF. + - `BrainSeg` + The volume of all brains structres in the segmentation. + - `BrainSegNotVent`, and `BrainSegNotVentSurf` + The brain segmentation volume without ventricles. + - `Cerebellum` + The total cerebellar volume. + - `SupraTentorial`, `SupraTentorialNotVent`, and `SupraTentorialNotVentVox` + The supratentorial brain volume/voxel count (without centricles and CSF). + - `Mask` + The volume of the brain mask. + - `EstimatedTotalIntraCranialVol` + The eTIV estimate (via talairach registration). + - `BrainSegVol-to-eTIV`, and `MaskVol-to-eTIV` + The ratios of the brain segmentation volume and the mask volume with respect + to the eTIV estimate. + """ + + hemi = key[:2] + side = "Left" if hemi != "rh" else "Right" + cc_classes = tuple(range(251, 256)) + if key in ("lhSurfaceHoles", "rhSurfaceHoles"): + # FastSurfer and FS7 are same + # l/rSurfaceHoles: (1-lheno/2) -- Euler number of /surf/l/rh.orig.nofix + return SurfaceHoles( + Path(f"surf/{hemi}.orig.nofix"), + f"{hemi}SurfaceHoles", + f"Number of defect holes in {hemi} surfaces prior to fixing", + "unitless", + ) + elif key == "SurfaceHoles": + # sum of holes in left and right surfaces + return DerivedMeasure( + ["rhSurfaceHoles", "lhSurfaceHoles"], + "SurfaceHoles", + "Total number of defect holes in surfaces prior to fixing", + measure_host=self, + ) + elif key in ("lhPialTotal", "rhPialTotal"): + # FastSurfer and FS7 are same + return SurfaceVolume( + Path(f"surf/{hemi}.pial"), + f"{hemi}PialTotalVol", + f"{side} hemisphere total pial volume", + "mm^3", + ) + elif key in ("lhWhiteMatterVol", "rhWhiteMatterVol"): + # This is volume-based in FS7 (ComputeBrainVolumeStats2) + if key[:1] == "l": + classes = (2, 78) + else: # r + classes = (41, 79) + return self.voxel_class( + classes, + f"{hemi}WhiteMatterVol", + f"{side} hemisphere total white matter volume", + "mm^3", + ) + elif key in ("lhWhiteMatterTotal", "rhWhiteMatterTotal"): + return SurfaceVolume( + Path(f"surf/{hemi}.white"), + f"{hemi}WhiteMatterSurfVol", + f"{side} hemisphere total white matter volume", + "mm^3", + ) + elif key in ("lhCortex", "rhCortex"): + # From https://github.com/freesurfer/freesurfer/blob/ + # 3753f8a1af484ac2507809c0edf0bc224bb6ccc1/utils/cma.cpp#L1190C1-L1192C52 + # CtxGM = everything inside pial surface minus everything in white surface. + parents = [f"{hemi}PialTotal", (-1, f"{hemi}WhiteMatterTotal")] + # With version 7, don't need to do a correction because the pial surface is + # pinned to the white surface in the medial wall + return DerivedMeasure( + parents, + f"{hemi}CortexVol", + f"{side} hemisphere cortical gray matter volume", + measure_host=self, + ) + elif key == "Cortex": + # 7 => lhCtxGM + rhCtxGM: sum of left and right cerebral GM + return DerivedMeasure( + ["lhCortex", "rhCortex"], + "CortexVol", + f"Total cortical gray matter volume", + measure_host=self, + ) + elif key == "CorpusCallosumVol": + # FastSurfer and FS7 are same + # CCVol: + # CC_Posterior CC_Mid_Posterior CC_Central CC_Mid_Anterior CC_Anterior + return self.voxel_class( + cc_classes, + "CorpusCallosumVol", + "Volume of the Corpus Callosum", + "mm^3", + ) + elif key in ("lhWM-hypointensities", "rhWM-hypointensities"): + # lateralized counting of class 77 WM hypo intensities + def mask_77_lat(arr): + """ + This function returns a lateralized mask of hypo-WM (class 77). + + This is achieved by looking at surrounding labels and associating them + with left or right (this is not 100% robust when there is no clear + classes with left aseg labels present, but it is cheap to perform. + """ + mask = arr == 77 + left_aseg = (2, 4, 5, 7, 8, 10, 11, 12, 13, 17, 18, 26, 28, 30, 31) + is_left = mask_in_array(arr, left_aseg) + from scipy.ndimage import uniform_filter + is_left = uniform_filter(is_left.astype(np.float32), size=7) > 0.2 + is_side = np.logical_not(is_left) if hemi == "rh" else is_left + return np.logical_and(mask, is_side) + + return VolumeMeasure( + self._seg_from_file, + mask_77_lat, + f"{side}WhiteMatterHypoIntensities", + f"Volume of {side} White matter hypointensities", + "mm^3" + ) + elif key in ("lhCerebralWhiteMatter", "rhCerebralWhiteMatter"): + # SurfaceVolume + # 9/10 => l/rCerebralWM + parents = [ + f"{hemi}WhiteMatterVol", + f"{hemi}WM-hypointensities", + (0.5, "CorpusCallosumVol"), + ] + return DerivedMeasure( + parents, + f"{hemi}CerebralWhiteMatterVol", + f"{side} hemisphere cerebral white matter volume", + measure_host=self, + ) + elif key == "CerebralWhiteMatter": + # 11 => lhCtxWM + rhCtxWM: sum of left and right cerebral WM + return DerivedMeasure( + ["rhCerebralWhiteMatter", "lhCerebralWhiteMatter"], + "CerebralWhiteMatterVol", + "Total cerebral white matter volume", + measure_host=self, + ) + elif key == "CerebellarGM": + # Left-Cerebellum-Cortex Right-Cerebellum-Cortex Cbm_Left_I_IV + # Cbm_Right_I_IV Cbm_Left_V Cbm_Right_V Cbm_Left_VI Cbm_Vermis_VI + # Cbm_Right_VI Cbm_Left_CrusI Cbm_Vermis_CrusI Cbm_Right_CrusI + # Cbm_Left_CrusII Cbm_Vermis_CrusII Cbm_Right_CrusII Cbm_Left_VIIb + # Cbm_Vermis_VIIb Cbm_Right_VIIb Cbm_Left_VIIIa Cbm_Vermis_VIIIa + # Cbm_Right_VIIIa Cbm_Left_VIIIb Cbm_Vermis_VIIIb Cbm_Right_VIIIb + # Cbm_Left_IX Cbm_Vermis_IX Cbm_Right_IX Cbm_Left_X Cbm_Vermis_X Cbm_Right_X + # Cbm_Vermis_VII Cbm_Vermis_VIII Cbm_Vermis + cerebellum_classes = [8, 47] + cerebellum_classes.extend(range(601, 629)) + cerebellum_classes.extend(range(630, 633)) + return self.voxel_class( + cerebellum_classes, + "CerebellarGMVol", + "Cerebellar gray matter volume", + "mm^3", + ) + elif key == "CerebellarWM": + # Left-Cerebellum-White-Matter Right-Cerebellum-White-Matter + cerebellum_classes = [7, 46] + return self.voxel_class( + cerebellum_classes, + "CerebellarWMVol", + "Cerebellar white matter volume", + "mm^3", + ) + elif key == "SubCortGray": + # 4 => SubCortGray + # Left-Thalamus Right-Thalamus Left-Caudate Right-Caudate Left-Putamen + # Right-Putamen Left-Pallidum Right-Pallidum Left-Hippocampus + # Right-Hippocampus Left-Amygdala Right-Amygdala Left-Accumbens-area + # Right-Accumbens-area Left-VentralDC Right-VentralDC Left-Substantia-Nigra + # Right-Substantia-Nigra + subcortgray_classes = [17, 18, 26, 27, 28, 58, 59, 60] + subcortgray_classes.extend(range(10, 14)) + subcortgray_classes.extend(range(49, 55)) + return self.voxel_class( + subcortgray_classes, + "SubCortGrayVol", + "Subcortical gray matter volume", + "mm^3", + ) + elif key == "TotalGray": + # FastSurfer, FS6 and FS7 are same + # 8 => TotalGMVol: sum of SubCortGray., Cortex and Cerebellar GM + return DerivedMeasure( + ["SubCortGray", "Cortex", "CerebellarGM"], + "TotalGrayVol", + "Total gray matter volume", + measure_host=self, + ) + elif key == "TFFC": + # FastSurfer, FS6 and FS7 are same + # TFFC: + # 3rd-Ventricle 4th-Ventricle 5th-Ventricle CSF + tffc_classes = (14, 15, 72, 24) + return self.voxel_class( + tffc_classes, + "Third-Fourth-Fifth-CSF", + "volume of 3rd, 4th, 5th ventricle and CSF", + "mm^3", + ) + elif key == "VentricleChoroidVol": + # FastSurfer, FS6 and FS7 are same, except FS7 adds a KeepCSF flag, which + # excludes CSF (but not by default) + # 15 => VentChorVol: + # Left-Choroid-Plexus Right-Choroid-Plexus Left-Lateral-Ventricle + # Right-Lateral-Ventricle Left-Inf-Lat-Vent Right-Inf-Lat-Vent + ventchor_classes = (4, 5, 31, 43, 44, 63) + return self.voxel_class( + ventchor_classes, + "VentricleChoroidVol", + "Volume of ventricles and choroid plexus", + "mm^3", + ) + elif key in "BrainSeg": + # 0 => BrainSegVol: + # FS7 (does mot use ribbon any more, just ) + # not background, in aseg ctab, not Brain stem, not optic chiasm, + # aseg undefined in aseg ctab and not cortex or WM (L/R Cerebral + # Ctx/WM) + # ComputeBrainStats2 also removes any regions that are not part of the + # AsegStatsLUT.txt + # background, brainstem, optic chiasm: 0, 16, 85 + brain_seg_classes = [2, 3, 4, 5, 7, 8] + brain_seg_classes.extend(range(10, 16)) + brain_seg_classes.extend([17, 18, 24, 26, 28, 30, 31]) + brain_seg_classes.extend(range(41, 55)) + brain_seg_classes.remove(45) + brain_seg_classes.remove(48) + brain_seg_classes.extend([58, 60, 62, 63, 72]) + brain_seg_classes.extend(range(77, 83)) + brain_seg_classes.extend(cc_classes) + if not self._fs_compat: + # also add asegdkt regions 1002-1035, 2002-2035 + brain_seg_classes.extend(range(1002, 1032)) + brain_seg_classes.remove(1004) + brain_seg_classes.extend((1034, 1035)) + brain_seg_classes.extend(range(2002, 2032)) + brain_seg_classes.remove(2004) + brain_seg_classes.extend((2034, 2035)) + return self.voxel_class( + brain_seg_classes, + "BrainSegVol", + "Brain Segmentation Volume", + "mm^3", + ) + elif key in ("BrainSegNotVent", "BrainSegNotVentSurf"): + # FastSurfer, FS6 and FS7 are same + # 1 => BrainSegNotVent: BrainSegVolNotVent (BrainSegVol-VentChorVol-TFFC) + return DerivedMeasure( + ["BrainSeg", (-1, "VentricleChoroidVol"), (-1, "TFFC")], + key.replace("SegNot", "SegVolNot"), + "Brain Segmentation Volume Without Ventricles", + measure_host=self, + ) + elif key == "Cerebellum": + return DerivedMeasure( + ("CerebellarGM", "CerebellarWM"), + "CerebellumVol", + "Cerebellar volume", + measure_host=self, + ) + elif key == "SupraTentorial": + parents = ["BrainSeg", (-1.0, "Cerebellum")] + return DerivedMeasure( + parents, + "SupraTentorialVol", + "Supratentorial volume", + measure_host=self, + ) + elif key == "SupraTentorialNotVent": + # 3 => SupraTentVolNotVent: SupraTentorial w/o Ventricles & Choroid Plexus + parents = ["SupraTentorial", (-1, "VentricleChoroidVol"), (-1, "TFFC")] + return DerivedMeasure( + parents, + "SupraTentorialVolNotVent", + "Supratentorial volume", + measure_host=self, + ) + elif key == "SupraTentorialNotVentVox": + # 3 => SupraTentVolNotVent: SupraTentorial w/o Ventricles & Choroid Plexus + return DerivedMeasure( + ["SupraTentorialNotVent"], + "SupraTentorialVolNotVentVox", + "Supratentorial volume voxel count", + operation="by_vox_vol", + measure_host=self, + ) + elif key == "Mask": + # 12 => MaskVol: Any voxel in mask > 0 + return MaskMeasure( + Path("mri/brainmask.mgz"), + "MaskVol", + "Mask Volume", + "mm^3", + ) + elif key == "EstimatedTotalIntraCranialVol": + # atlas_icv: eTIV from talairach transform determinate + return ETIVMeasure( + Path("mri/transforms/talairach.xfm"), + "eTIV", + "Estimated Total Intracranial Volume", + "mm^3", + ) + elif key == "BrainSegVol-to-eTIV": + # 0/atlas_icv: ratio BrainSegVol to eTIV + return DerivedMeasure( + ["BrainSeg", "EstimatedTotalIntraCranialVol"], + "BrainSegVol-to-eTIV", + "Ratio of BrainSegVol to eTIV", + measure_host=self, + operation="ratio", + ) + elif key == "MaskVol-to-eTIV": + # 12/atlas_icv: ratio Mask to eTIV + return DerivedMeasure( + ["Mask", "EstimatedTotalIntraCranialVol"], + "MaskVol-to-eTIV", + "Ratio of MaskVol to eTIV", + measure_host=self, + operation="ratio", + ) + + def __iter__(self) -> list[AbstractMeasure]: + """ + Iterate through all measures that are exported directly or indirectly. + """ + + out = [self[name] for name in self._exported_measures] + i = 0 + while i < len(out): + this = out[i] + if isinstance(this, DerivedMeasure): + out.extend(filter(lambda x: x not in out, this.parents_items())) + i += 1 + return out + + def compute_non_derived_pv( + self, + compute_threads: Executor | None = None + ) -> "list[Future[int | float]]": + """ + Trigger computation of all non-derived, non-pv measures that are required. + + Parameters + ---------- + compute_threads : concurrent.futures.Executor, optional + An Executor object to perform the computation of measures, if an Executor + object is passed, the computation of measures is submitted to the Executor + object. If not, measures are computed in the main thread. + + Returns + ------- + list[Future[int | float]] + For each non-derived and non-PV measure, a future object that is associated + with the call to the measure. + """ + + def run(f: Callable[[], int | float]) -> Future[int | float]: + out = Future() + out.set_result(f()) + return out + + if isinstance(compute_threads, Executor): + run = compute_threads.submit + + invalid_types = (DerivedMeasure, PVMeasure) + self._compute_futures = [ + run(this) for this in self.values() if not isinstance(this, invalid_types) + ] + return self._compute_futures + + def needs_pv_calculation(self) -> bool: + """ + Returns whether the manager has PV-dependent measures. + + Returns + ------- + bool + Whether the manager has PVMeasure children. + """ + return any(isinstance(this, PVMeasure) for this in self.values()) + + def get_virtual_labels(self, label_pool: Iterable[int]) -> dict[int, list[int]]: + """ + Get the virtual substitute labels that are required. + + Parameters + ---------- + label_pool : Iterable[int] + An iterable over available labels. + + Returns + ------- + dict[int, list[int]] + A dictionary of key-value pairs of new label and a list of labels this + represents. + """ + lbls = (this.labels() for this in self.values() if isinstance(this, PVMeasure)) + no_duplicate_dict = {self.__to_lookup(labs): labs for labs in lbls} + return dict(zip(label_pool, no_duplicate_dict.values())) + + @staticmethod + def __to_lookup(labels: Sequence[int]) -> str: + return str(list(sorted(set(map(int, labels))))) + + def update_pv_from_table( + self, + dataframe: "pd.DataFrame", + merged_labels: dict[int, list[int]], + ) -> "pd.DataFrame": + """ + Update pv measures from dataframe and remove corresponding entries from the + dataframe. + + Parameters + ---------- + dataframe : pd.DataFrame + The dataframe object with the PV values. + merged_labels : dict[int, list[int]] + Mapping from PVMeasure proxy label to list of labels it merges. + + Returns + ------- + pd.DataFrame + A dataframe object, where label 'groups' used for updates and in + `merged_labels` are removed, i.e. those labels added for PVMeasure objects. + + Raises + ------ + RuntimeError + """ + _lookup = {self.__to_lookup(ml): vl for vl, ml in merged_labels.items()} + filtered_df = dataframe + # go through the pv measures and find a measure that has the same list + for this in self.values(): + if isinstance(this, PVMeasure): + virtual_label = _lookup.get(self.__to_lookup(this.labels()), None) + if virtual_label is None: + raise RuntimeError(f"Could not find the virtual label for {this}.") + row = dataframe[dataframe["SegId"] == virtual_label] + if row.shape[0] != 1: + raise RuntimeError( + f"The search results in the dataframe for {this} failed: " + f"shape {row.shape}" + ) + this.update_data(row) + filtered_df = filtered_df[filtered_df["SegId"] != virtual_label] + + return filtered_df + + def wait_compute(self) -> Sequence[BaseException]: + """ + Wait for all pending computation processes and return their errors. + + Also resets the internal compute futures. + + Returns + ------- + Sequence[BaseException] + The errors raised in the computations. + """ + errors = [future.exception() for future in self._compute_futures] + self._compute_futures = [] + return [error for error in errors if error is not None] + + def wait_write_brainvolstats(self, brainvol_statsfile: Path): + """ + Wait for measure computation to finish and write results to brainvol_statsfile. + + Parameters + ---------- + brainvol_statsfile: Path + The file to write the measures to. + + Raises + ------ + RuntimeError + If errors occurred during measure computation. + """ + errors = list(self.wait_compute()) + if len(errors) != 0: + error_messages = ["Some errors occurred during measure computation:"] + error_messages.extend(map(lambda e: str(e.args[0]), errors)) + raise RuntimeError("\n - ".join(error_messages)) + + def fmt_measure(key: str, data: MeasureTuple) -> str: + return f"# Measure {key}, {data[0]}, {data[1]}, {data[2]:.12f}, {data[3]}" + + lines = self.format_measures(fmt_func=fmt_measure) + + with open(brainvol_statsfile, "w") as file: + for line in lines: + print(line, file=file) diff --git a/FastSurferCNN/utils/run_tools.py b/FastSurferCNN/utils/run_tools.py index 9b6b8737..a206420c 100644 --- a/FastSurferCNN/utils/run_tools.py +++ b/FastSurferCNN/utils/run_tools.py @@ -1,3 +1,20 @@ +#!/bin/python + +# Copyright 2023 Image Analysis Lab, German Center for Neurodegenerative Diseases(DZNE), Bonn +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from concurrent.futures import Executor, Future import subprocess from concurrent.futures import Executor, Future from dataclasses import dataclass diff --git a/doc/conf.py b/doc/conf.py index a33185a5..a9c70804 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -51,7 +51,10 @@ "sphinx_copybutton", "sphinx_design", "sphinx_issues", - "nbsphinx", + # sphinx.ext.autosectionlabel and nbsphinx together with sphinxarg.ext causes a + # duplicate label warning: https://github.com/spatialaudio/nbsphinx/issues/787 + # nbsphinx is currently not 'needed' as we do not include ipynb files. + # "nbsphinx", "IPython.sphinxext.ipython_console_highlighting", "myst_parser", "sphinxarg.ext", @@ -64,6 +67,7 @@ suppress_warnings = [ # "myst.xref_missing", "myst.duplicate_def", + "autosectionlabel", ] # create anchors for which headings? diff --git a/doc/sphinx_ext/fix_links/parser.py b/doc/sphinx_ext/fix_links/parser.py index 71f86824..9e492a11 100644 --- a/doc/sphinx_ext/fix_links/parser.py +++ b/doc/sphinx_ext/fix_links/parser.py @@ -39,7 +39,7 @@ def _wrapper(include_instance: Include): class Renderer(SphinxRenderer): """ - Renderer object to automatically fix headings that are not consequetive levels in + Renderer object to automatically fix headings that are not consecutive levels in (included) Markdown files. Also includes alternative targets into anchors that are rendered, but do not match a target. """ diff --git a/doc/sphinx_ext/fix_links/resolve.py b/doc/sphinx_ext/fix_links/resolve.py index 3e3c7d78..570e881d 100644 --- a/doc/sphinx_ext/fix_links/resolve.py +++ b/doc/sphinx_ext/fix_links/resolve.py @@ -110,27 +110,39 @@ def resolve_xref( doc_root = Path(env.srcdir) project_root = env.config.fix_links_project_root uri = node[attr] - if node["refdomain"] == "doc": - _uri_path, _uri_id = f"/{uri}", node.attributes.get("refid", None) or "" + if node["reftype"] == "doc": + if uri.startswith("/"): + node['reftarget'] = "index" + _uri_path = uri[1:] + else: + _uri_path = uri + _uri_id = node.attributes.get("refid", None) or "" _uri_sep = "#" if _uri_id else "" project_root = "." + reftype = "doc" else: _uri_path, _uri_sep, _uri_id = uri.partition("#") if not _uri_id and getattr(node, "reftargetid", None) is not None: _uri_sep, _uri_id = "#", node["reftargetid"] + + reftype = "ref" # resolve the target Path in the link w.r.t. the source it came from if _uri_path.startswith("/"): # absolute with respect to documentation root target_path = (doc_root / project_root / _uri_path[1:]).resolve() else: - sourcefile_path = Path(env.srcdir) / node["refdoc"] + sourcefile_path = doc_root / node.source.split(":")[0] target_path = (sourcefile_path.parent / _uri_path).resolve() - _uri_path = relpath(target_path, env.srcdir) + _uri_path = relpath(target_path, doc_root) _uri_hash = _uri_sep + _uri_id if not _uri_path.startswith("../"): # maybe this already fixed the path? - ref = _resolve_xref_with_(f"/{_uri_path}{_uri_hash}".lower(), uri) + ref = _resolve_xref_with_( + f"{_uri_path}{_uri_hash}".lower(), + node.source, + reftype=reftype, + ) if ref is not None: return ref @@ -141,13 +153,27 @@ def resolve_xref( env.found_docs, _uri_path, ) + _reftarget = node["reftarget"] + _reftype = "doc" if _uri_hash == "" else "ref" for potential_doc in potential_targets: potential_path = env.doc2path(potential_doc, False) - ref = _resolve_xref_with_(f"/{potential_path}{_uri_hash}".lower(), uri) + if potential_path.endswith(".rst"): + potential_path = potential_doc + potential_path = relpath( + doc_root / potential_path, + (doc_root / node["refdoc"]).parent, + ) + node["reftarget"] = potential_path + ref = _resolve_xref_with_( + (potential_path + _uri_hash).lower(), + node.source, + reftype=_reftype, + ) if ref is not None: return ref + node["reftarget"] = _reftarget - source = f"/{_uri_path}{_uri_sep}{_uri_id}" + source = f"{_uri_path}{_uri_sep}{_uri_id}" for key, (pat, repls) in subs.items(): # if this search string does not match, try next if not pat.match(source): @@ -169,7 +195,11 @@ def resolve_xref( break replaced = _replaced # search for a reference associated with the replaced link in std - ref = _resolve_xref_with_(str(replaced).lower(), uri) + ref = _resolve_xref_with_( + str(replaced).lower(), + node.source, + reftype=reftype, + ) # check and return the reference, if it is valid if ref is not None: @@ -204,13 +234,14 @@ def _resolve_xref_with( contnode: nodes.Element, target: str, source: str, + reftype: str = "ref", ) -> nodes.reference | None: std_domain = env.domains["std"] ref: nodes.reference | None = std_domain.resolve_xref( env, node["refdoc"], # fromdocname app.builder, - "ref", + reftype, target, node, contnode, diff --git a/recon_surf/functions.sh b/recon_surf/functions.sh index fd3a5205..f256444e 100644 --- a/recon_surf/functions.sh +++ b/recon_surf/functions.sh @@ -12,7 +12,7 @@ export binpath # also check for failure (e.g. on mac it fails) timecmd="${binpath}fs_time" $timecmd echo testing &> /dev/null -if [ ${PIPESTATUS[0]} -ne 0 ] ; then +if [ "${PIPESTATUS[0]}" -ne 0 ] ; then echo "time command failing, not using time..." timecmd="" fi @@ -35,9 +35,9 @@ function RunIt() echo "$timecmd $cmd" | tee -a $CMDF echo "if [ \${PIPESTATUS[0]} -ne 0 ] ; then exit 1 ; fi" >> $CMDF else - echo $cmd | tee -a $LF - $timecmd $cmd 2>&1 | tee -a $LF - if [ ${PIPESTATUS[0]} -ne 0 ] ; then exit 1 ; fi + echo "$cmd" | tee -a "$LF" + $timecmd $cmd 2>&1 | tee -a "$LF" + if [ "${PIPESTATUS[0]}" -ne 0 ] ; then exit 1 ; fi fi } @@ -59,31 +59,31 @@ function RunBatchJobs() shift local JOB local LOG - for cmdf in $*; do + for cmdf in "$@"; do echo "RunBatchJobs: CMDF: $cmdf" - chmod u+x $cmdf + chmod u+x "$cmdf" JOB="$cmdf" LOG=$cmdf.log - echo "" >& $LOG - echo " $JOB" >> $LOG - echo "" >> $LOG - bash "$JOB" >> $LOG 2>&1 & - PIDS=(${PIDS[@]} $!) - LOGS=(${LOGS[@]} $LOG) + echo "" >& "$LOG" + echo " $JOB" >> "$LOG" + echo "" >> "$LOG" + exec "$JOB" >> "$LOG" 2>&1 & + PIDS=("${PIDS[@]}" "$!") + LOGS=("${LOGS[@]}" "$LOG") done # wait till all processes have finished local PIDS_STATUS=() for pid in "${PIDS[@]}"; do echo "Waiting for PID $pid of (${PIDS[*]}) to complete..." - wait $pid - PIDS_STATUS=(${PIDS_STATUS[@]} $?) + wait "$pid" + PIDS_STATUS=("${PIDS_STATUS[@]}" "$?") done # now append their logs to the main log file for log in "${LOGS[@]}" do - cat $log >> $LOG_FILE - rm -f $log + cat "$log" >> "$LOG_FILE" + rm -f "$log" done echo "PIDs (${PIDS[*]}) completed and logs appended." # and check for failures @@ -103,27 +103,44 @@ function softlink_or_copy() # 3: logfile # 4: cmdf local LF="$3" - local ln_cmd="ln -sf $1 $2" - local cp_cmd="cp $1 $2" + local ln_cmd=(ln -sf "$1" "$2") + local cp_cmd=(cp "$1" "$2") if [[ $# -eq 4 ]] then local CMDF=$4 - echo "echo \"$ln_cmd\" " | tee -a $CMDF - echo "$timecmd $ln_cmd " | tee -a $CMDF - echo "if [ \${PIPESTATUS[0]} -ne 0 ]" | tee -a $CMDF - echo "then " | tee -a $CMDF - echo " echo \"$cp_cmd\" " | tee -a $CMDF - echo " $timecmd $cp_cmd " | tee -a $CMDF - echo " if [ \${PIPESTATUS[0]} -ne 0 ] ; then exit 1 ; fi" >> $CMDF - echo "fi" | tee -a $CMDF + { + echo "echo $(echo_quoted "${ln_cmd[@]}")" + echo "$timecmd $(echo_quoted "${ln_cmd[@]}")" + echo "if [ \${PIPESTATUS[0]} -ne 0 ]" + echo "then" + echo " echo $(echo_quoted "${cp_cmd[@]}")" + echo " $timecmd $(echo_quoted "${cp_cmd[@]}")" + echo " if [ \${PIPESTATUS[0]} -ne 0 ] ; then exit 1 ; fi" + echo "fi" + } | tee -a "$CMDF" else - echo $ln_cmd | tee -a $LF - $timecmd $ln_cmd 2>&1 | tee -a $LF - if [ ${PIPESTATUS[0]} -ne 0 ] - then - echo $cp_cmd | tee -a $LF - $timecmd $cp_cmd 2>&1 | tee -a $LF - if [ ${PIPESTATUS[0]} -ne 0 ] ; then exit 1 ; fi - fi + { + echo_quoted "${ln_cmd[@]}" + $timecmd "${ln_cmd[@]}" 2>&1 + if [ "${PIPESTATUS[0]}" -ne 0 ] + then + echo_quoted "${cp_cmd[@]}" + $timecmd "${cp_cmd[@]}" 2>&1 + if [ "${PIPESTATUS[0]}" -ne 0 ] ; then exit 1 ; fi + fi + } | tee -a "$LF" fi } + +function echo_quoted() +{ + # params ... 1-N + sep="" + for i in "$@" + do + if [[ "${i/ /}" != "$i" ]] ; then j="%q" ; else j="%s" ; fi + printf "%s$j" "$sep" "$i" + sep=" " + done + echo "" +} \ No newline at end of file diff --git a/recon_surf/lta.py b/recon_surf/lta.py index 07091873..3a6f32f1 100755 --- a/recon_surf/lta.py +++ b/recon_surf/lta.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -from typing import Dict # Copyright 2021 Image Analysis Lab, German Center for Neurodegenerative Diseases (DZNE), Bonn # @@ -24,9 +23,9 @@ def writeLTA( filename: str, T: npt.ArrayLike, src_fname: str, - src_header: Dict, + src_header: dict, dst_fname: str, - dst_header: Dict + dst_header: dict ) -> None: """ Write linear transform array info to an .lta file. diff --git a/recon_surf/recon-surf.sh b/recon_surf/recon-surf.sh index 33a24200..84b4a4aa 100755 --- a/recon_surf/recon-surf.sh +++ b/recon_surf/recon-surf.sh @@ -30,6 +30,7 @@ DoParallel=0 # if 1, run hemispheres in parallel threads="1" # number of threads to use for running FastSurfer allow_root="" # flag for allowing execution as root user atlas3T="false" # flag to use/do not use the 3t atlas for talairach registration/etiv +segstats_legacy="false" # flag to enable segstats legacy mode # Dev flags default check_version=1 # Check for supported FreeSurfer version (terminate if not detected) @@ -38,7 +39,8 @@ hires_voxsize_threshold=0.999 # Threshold below which the hires options are pas if [ -z "$FASTSURFER_HOME" ] then - binpath="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/" + binpath="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )/" + FASTSURFER_HOME="$(cd -- "$(dirname "$binpath")" >/dev/null 2>&1 ; pwd -P )/" else binpath="$FASTSURFER_HOME/recon_surf/" fi @@ -46,7 +48,7 @@ fi # check bash version > 3.1 (needed for printf %q) function version { echo "$@" | awk -F. '{ printf("%d%03d%03d%03d\n", $1,$2,$3,$4); }'; } -if [ $(version ${BASH_VERSION}) -lt $(version "3.1.0") ]; then +if [ "$(version "${BASH_VERSION}")" -lt "$(version "3.1.0")" ]; then echo "bash ${BASH_VERSION} is too old. Should be newer than 3.1, please upgrade!" exit 1 fi @@ -134,7 +136,7 @@ EOF } -# Load the RunIt and the RunBatchJobs functions +# Load the RunIt and the RunBatchJobs functions, also sets up timecmd source "$binpath/functions.sh" # PRINT USAGE if called without params @@ -152,128 +154,67 @@ while [[ $# -gt 0 ]] do # make key lowercase key=$(echo "$1" | tr '[:upper:]' '[:lower:]') +shift # past argument case $key in - --sid) - subject="$2" - shift # past argument - shift # past value - ;; - --sd) - export SUBJECTS_DIR="$2" - shift # past argument - shift # past value - ;; - --t1) - t1="$2" - shift # past argument - shift # past value - ;; - --asegdkt_segfile | --aparc_aseg_segfile | --seg) + --sid) subject="$1" ; shift ;; + --sd) export SUBJECTS_DIR="$1" ; shift ;; + --t1) t1="$1" ; shift ;; + --asegdkt_segfile | --aparc_aseg_segfile | --seg) if [ "$key" == "--seg" ] || [ "$key" == "--aparc_aseg_segfile" ]; then - echo "WARNING: $1 is deprecated and will be removed, use --asegdkt_segfile ." + echo "WARNING: $key is deprecated and will be removed, use --asegdkt_segfile ." fi - asegdkt_segfile="$2" - shift # past argument + asegdkt_segfile="$1" shift # past value ;; - --vol_segstats) + --vol_segstats) echo "WARNING: the --vol_segstats flag is obsolete and will be removed, --vol_segstats ignored." - shift # past argument - ;; - --fstess) - fstess=1 - shift # past argument - ;; - --fsqsphere) - fsqsphere=1 - shift # past argument - ;; - --fsaparc) - fsaparc=1 - shift # past argument - ;; - --no_surfreg) - fssurfreg=0 - shift # past argument - ;; - --3t) - atlas3T="true" - shift - ;; - --parallel) - DoParallel=1 - shift # past argument ;; - --threads) - threads="$2" - shift # past argument - shift # past value - ;; - --py) - python="$2" - shift # past argument - shift # past value - ;; - --fs_license) - if [ -f "$2" ]; then - export FS_LICENSE="$2" + --segstats_legacy) segstats_legacy="true" ;; + --fstess) fstess=1 ;; + --fsqsphere) fsqsphere=1 ;; + --fsaparc) fsaparc=1 ;; + --no_surfreg) fssurfreg=0 ;; + --3t) atlas3T="true" ;; + --parallel) DoParallel=1 ;; + --threads) threads="$1" ; shift ;; + --py) python="$1" ; shift ;; + --fs_license) + if [ -f "$1" ]; then + export FS_LICENSE="$1" else - echo "Provided FreeSurfer license file $2 could not be found. Make sure to provide the full path and name. Exiting..." - exit 1; + echo "Provided FreeSurfer license file $1 could not be found. Make sure to provide the full path and name. Exiting..." + exit 1; fi - shift # past argument shift # past value ;; - --ignore_fs_version) - check_version=0 - shift # past argument - ;; - --no_fs_t1 ) - get_t1=0 - shift # past argument - ;; - --allow_root) - allow_root="--allow_root" - shift # past argument - ;; - -h|--help) - usage - exit - ;; - *) # unknown option - echo ERROR: Flag $key unrecognized. - exit 1 - ;; + --ignore_fs_version) check_version=0 ;; + --no_fs_t1 ) get_t1=0 ;; + --allow_root) allow_root="--allow_root" ;; + -h|--help) usage ; exit ;; + # unknown option + *) echo "ERROR: Flag $key unrecognized." ; exit 1 ;; esac done set -- "${POSITIONAL[@]}" # restore positional parameters # CHECKS -echo -echo sid $subject -echo T1 $t1 -echo asegdkt_segfile $asegdkt_segfile -echo +echo "" +echo "sid $subject" +echo "T1 $t1" +echo "asegdkt_segfile $asegdkt_segfile" +echo "" # Warning if run as root user if [ -z "$allow_root" ] && [ "$(id -u)" == "0" ] - then - echo "You are trying to run '$0' as root. We advice to avoid running FastSurfer as root, " - echo "because it will lead to files and folders created as root." - echo "If you are running FastSurfer in a docker container, you can specify the user with " - echo "'-u \$(id -u):\$(id -g)' (see https://docs.docker.com/engine/reference/run/#user)." - echo "If you want to force running as root, you may pass --allow_root to recon-surf.sh." - exit 1; -fi - -if [ "$subject" == "subject" ] then - echo "Subject ID cannot be \"subject\", please choose a different sid" - # Explanation, see https://github.com/Deep-MI/FastSurfer/issues/186 - # this is a bug in FreeSurfer's argparse when calling "mri_brainvol_stats subject" - exit 1 + echo "You are trying to run '$0' as root. We advice to avoid running FastSurfer as root, " + echo "because it will lead to files and folders created as root." + echo "If you are running FastSurfer in a docker container, you can specify the user with " + echo "'-u \$(id -u):\$(id -g)' (see https://docs.docker.com/engine/reference/run/#user)." + echo "If you want to force running as root, you may pass --allow_root to recon-surf.sh." + exit 1; fi if [ -z "$SUBJECTS_DIR" ] @@ -295,9 +236,9 @@ export FREESURFER=$FREESURFER_HOME if [ "$check_version" == "1" ] then - if grep -q -v ${FS_VERSION_SUPPORT} $FREESURFER_HOME/build-stamp.txt + if grep -q -v "${FS_VERSION_SUPPORT}" "$FREESURFER_HOME/build-stamp.txt" then - echo "ERROR: You are trying to run recon-surf with FreeSurfer version $(cat $FREESURFER_HOME/build-stamp.txt)." + echo "ERROR: You are trying to run recon-surf with FreeSurfer version $(cat "$FREESURFER_HOME/build-stamp.txt")." echo "We are currently supporting only FreeSurfer $FS_VERSION_SUPPORT" echo "Therefore, make sure to export and source the correct FreeSurfer version before running recon-surf.sh: " echo "export FREESURFER_HOME=/path/to/your/local/fs$FS_VERSION_SUPPORT" @@ -349,7 +290,7 @@ then fsthreads="-threads $threads -itkthreads $threads" fi -if [ $(echo -n "${SUBJECTS_DIR}/${subject}" | wc -m) -gt 185 ] +if [ "$(echo -n "${SUBJECTS_DIR}/${subject}" | wc -m)" -gt 185 ] then echo "ERROR: subject directory path is very long." echo "This is known to cause errors due to some commands run by freesurfer versions built for Ubuntu." @@ -365,13 +306,13 @@ if [ -f "$SUBJECTS_DIR/$subject/mri/wm.mgz" ] || [ -f "$SUBJECTS_DIR/$subject/mr fi # collect info -StartTime=`date`; -tSecStart=`date '+%s'`; -year=`date +%Y` -month=`date +%m` -day=`date +%d` -hour=`date +%H` -min=`date +%M` +StartTime=$(date); +tSecStart=$(date '+%s') +year=$(date +%Y) +month=$(date +%m) +day=$(date +%d) +hour=$(date +%H) +min=$(date +%M) # Setup dirs @@ -384,6 +325,7 @@ mkdir -p "$SUBJECTS_DIR/$subject/stats" mdir="$SUBJECTS_DIR/$subject/mri" sdir="$SUBJECTS_DIR/$subject/surf" +statsdir="$SUBJECTS_DIR/$subject/stats" ldir="$SUBJECTS_DIR/$subject/label" mask="$mdir/mask.mgz" @@ -391,38 +333,41 @@ mask="$mdir/mask.mgz" # Set up log file DoneFile="$SUBJECTS_DIR/$subject/scripts/recon-surf.done" -if [ $DoneFile != /dev/null ] ; then rm -f $DoneFile ; fi +if [ "$DoneFile" != /dev/null ] ; then rm -f "$DoneFile" ; fi LF="$SUBJECTS_DIR/$subject/scripts/recon-surf.log" -if [ $LF != /dev/null ] ; then rm -f $LF ; fi -echo "Log file for recon-surf.sh" >> $LF -date 2>&1 | tee -a $LF -echo "" | tee -a $LF -echo "export SUBJECTS_DIR=$SUBJECTS_DIR" | tee -a $LF -echo "cd `pwd`" | tee -a $LF -echo $0 ${inputargs[*]} | tee -a $LF -echo "" | tee -a $LF -cat $FREESURFER_HOME/build-stamp.txt 2>&1 | tee -a $LF -echo $VERSION | tee -a $LF -uname -a 2>&1 | tee -a $LF - -echo " " | tee -a $LF -echo "==================== Checking validity of inputs =================================" | tee -a $LF -echo " " | tee -a $LF +if [ "$LF" != /dev/null ] ; then rm -f "$LF" ; fi +echo "Log file for recon-surf.sh" >> "$LF" +{ # all output tee -a "$LF" + date 2>&1 + echo " " + echo "export SUBJECTS_DIR=$SUBJECTS_DIR" + echo "cd $(pwd)" + echo_quoted "$0" "${inputargs[@]}" + echo " " + cat "$FREESURFER_HOME/build-stamp.txt" 2>&1 + echo "$VERSION" + uname -a 2>&1 + echo " " + echo " " + echo "==================== Checking validity of inputs =================================" + echo " " # Print parallelization parameters -echo " " | tee -a $LF -if [ "$DoParallel" == "1" ] -then - echo " RUNNING both hemis in PARALLEL " | tee -a $LF -else - echo " RUNNING both hemis SEQUENTIALLY " | tee -a $LF -fi -echo " RUNNING $OMP_NUM_THREADS number of OMP THREADS " | tee -a $LF -echo " RUNNING $ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS number of ITK THREADS " | tee -a $LF -echo " " | tee -a $LF + # Print parallelization parameters + if [ "$DoParallel" == "1" ] + then + echo " RUNNING both hemis in PARALLEL" + else + echo " RUNNING both hemis SEQUENTIALLY" + fi + echo " RUNNING $OMP_NUM_THREADS number of OMP THREADS" + echo " RUNNING $ITK_GLOBAL_DEFAULT_NUMBER_OF_THREADS number of ITK THREADS" + echo " " + + # Check input segmentation quality + echo "Checking Input Segmentation Quality ..." +} | tee -a "$LF" -# Check input segmentation quality -echo "Checking Input Segmentation Quality ..." | tee -a "$LF" cmd="$python $FASTSURFER_HOME/FastSurferCNN/quick_qc.py --asegdkt_segfile $asegdkt_segfile" RunIt "$cmd" "$LF" echo "" | tee -a "$LF" @@ -432,35 +377,37 @@ echo "" | tee -a "$LF" ########################################## START ######################################################## -echo " " | tee -a $LF -echo "================== Creating orig and rawavg from input =========================" | tee -a $LF -echo " " | tee -a $LF +{ + echo " " + echo "================== Creating orig and rawavg from input =========================" + echo " " +} | tee -a "$LF" CONFORM_LF=$SUBJECTS_DIR/$subject/scripts/conform.log -if [ $CONFORM_LF != /dev/null ] ; then rm -f $CONFORM_LF ; fi -echo "Log file for Conform test" > $CONFORM_LF +if [ "$CONFORM_LF" != /dev/null ] ; then rm -f "$CONFORM_LF" ; fi +echo "Log file for Conform test" > "$CONFORM_LF" # check for input conformance -cmd="$python ${binpath}../FastSurferCNN/data_loader/conform.py -i $t1 --check_only --vox_size min --verbose" -RunIt "$cmd" "$LF -a $CONFORM_LF" +cmd="$python $FASTSURFER_HOME/FastSurferCNN/data_loader/conform.py -i $t1 --check_only --vox_size min --verbose" +RunIt "$cmd" "$LF" |& tee -a "$CONFORM_LF" # look into the CONFORM_LF to find the voxel sizes, the second conform.py call will check the legality of vox_size -vox_size=`cat $CONFORM_LF | grep -E " - Voxel Size " | cut -d' ' -f5 | cut -d'x' -f1` +vox_size=$(grep -E " - Voxel Size " "$CONFORM_LF" | cut -d' ' -f5 | cut -d'x' -f1) # remove the temporary conform_log (all info is also in the recon-surf logfile) -if [ -f "$CONFORM_LF" ]; then rm -f $CONFORM_LF ; fi +if [ -f "$CONFORM_LF" ]; then rm -f "$CONFORM_LF" ; fi # here, we check the correct vox_size by passing it to the next conform, so errors in this line might be caused above -cmd="$python ${binpath}../FastSurferCNN/data_loader/conform.py -i $asegdkt_segfile --check_only --vox_size $vox_size --dtype any --verbose" -RunIt "$cmd" $LF +cmd="$python $FASTSURFER_HOME/FastSurferCNN/data_loader/conform.py -i $asegdkt_segfile --check_only --vox_size $vox_size --dtype any --verbose" +RunIt "$cmd" "$LF" if (( $(echo "$vox_size < $hires_voxsize_threshold" | bc -l) )) then - echo "The voxel size $vox_size is less than $hires_voxsize_threshold, so we are proceeding with hires options." | tee -a $LF + echo "The voxel size $vox_size is less than $hires_voxsize_threshold, so we are proceeding with hires options." | tee -a "$LF" hiresflag="-hires" noconform_if_hires=" -noconform" hires_surface_suffix=".predec" else - echo "The voxel size $vox_size is not less than $hires_voxsize_threshold, so we are proceeding with standard options." | tee -a $LF + echo "The voxel size $vox_size is not less than $hires_voxsize_threshold, so we are proceeding with standard options." | tee -a "$LF" hiresflag="" noconform_if_hires="" hires_surface_suffix="" @@ -469,15 +416,15 @@ fi # create orig.mgz and aparc.DKTatlas+aseg.orig.mgz (copy of T1 and segmentation) # also ensures .mgz format (in case inputs are nifti) cmd="mri_convert $t1 $mdir/orig.mgz" -RunIt "$cmd" $LF +RunIt "$cmd" "$LF" cmd="mri_convert $asegdkt_segfile $mdir/aparc.DKTatlas+aseg.orig.mgz" -RunIt "$cmd" $LF +RunIt "$cmd" "$LF" # link original T1 input to rawavg (needed by pctsurfcon) -pushd $mdir -softlink_or_copy "orig.mgz" "rawavg.mgz" "$LF" -popd +pushd "$mdir" > /dev/null || ( echo "Could not change to $mdir" ; exit 1 ) + softlink_or_copy "orig.mgz" "rawavg.mgz" "$LF" +popd > /dev/null || ( echo "Could not change to subject_dir" ; exit 1 ) @@ -490,23 +437,29 @@ popd # ============================= MASK & ASEG_noCC ======================================== if [ ! -f "$mask" ] || [ ! -f "$mdir/aseg.auto_noCCseg.mgz" ] ; then - # Mask or aseg.auto_noCCseg not found; create them from aparc.DKTatlas+aseg - echo " " | tee -a $LF - echo "============= Creating aseg.auto_noCCseg (map aparc labels back) ===============" | tee -a $LF - echo " " | tee -a $LF + { + # Mask or aseg.auto_noCCseg not found; create them from aparc.DKTatlas+aseg + echo " " + echo "============= Creating aseg.auto_noCCseg (map aparc labels back) ===============" + echo " " + } | tee -a "$LF" + # reduce labels to aseg, then create mask (dilate 5, erode 4, largest component), also mask aseg to remove outliers # output will be uchar (else mri_cc will fail below) - cmd="$python ${binpath}/../FastSurferCNN/reduce_to_aseg.py -i $mdir/aparc.DKTatlas+aseg.orig.mgz -o $mdir/aseg.auto_noCCseg.mgz --outmask $mask --fixwm" - RunIt "$cmd" $LF + cmd="$python $FASTSURFER_HOME/FastSurferCNN/reduce_to_aseg.py -i $mdir/aparc.DKTatlas+aseg.orig.mgz -o $mdir/aseg.auto_noCCseg.mgz --outmask $mask --fixwm" + RunIt "$cmd" "$LF" fi + # ============================= NU BIAS CORRECTION ======================================= if [ ! -f "$mdir/orig_nu.mgz" ]; then # only run the bias field correction, if the bias field corrected does not exist already - echo " " | tee -a $LF - echo "============= Computing NU (bias corrected) ============" | tee -a $LF - echo " " | tee -a $LF + { + echo " " + echo "============= Computing NU (bias corrected) ============" + echo " " + } | tee -a "$LF" # nu processing is changed here compared to recon-all: we use the brainmask from the # segmentation to improve the nu correction (and speedup) # orig_nu N3 in FS6 took 44 sec, FS 7.3.2 uses --ants-n4 (takes 3 min and does not accept @@ -517,73 +470,81 @@ if [ ! -f "$mdir/orig_nu.mgz" ]; then # frontal head), we don't. Also this avoids a second call to nu correct. # talairach.xfm is also not needed here at all, it can be dropped if other places in the # stream can be changed to avoid it. - pushd "$mdir" || ( echo "Cannot change to $mdir" | tee -a "$LF" || exit 1 ) - #cmd="mri_nu_correct.mni --no-rescale --i $mdir/orig.mgz --o $mdir/orig_nu.mgz --n 1 --proto-iters 1000 --distance 50 --mask $mdir/mask.mgz" - cmd="$python ${binpath}/N4_bias_correct.py --in $mdir/orig.mgz --rescale $mdir/orig_nu.mgz --aseg $mdir/aparc.DKTatlas+aseg.orig.mgz --threads $threads" - RunIt "$cmd" "$LF" - popd || return + pushd "$mdir" > /dev/null || ( echo "Cannot change to $mdir" ; exit 1 ) + #cmd="mri_nu_correct.mni --no-rescale --i $mdir/orig.mgz --o $mdir/orig_nu.mgz --n 1 --proto-iters 1000 --distance 50 --mask $mdir/mask.mgz" + cmd="$python ${binpath}/N4_bias_correct.py --in $mdir/orig.mgz --rescale $mdir/orig_nu.mgz --aseg $mdir/aparc.DKTatlas+aseg.orig.mgz --threads $threads" + RunIt "$cmd" "$LF" + popd > /dev/null || (echo "Could not popd" ; exit 1) fi # ============================= TALAIRACH ============================================== if [[ ! -f "$mdir/transforms/talairach.lta" ]] || [[ ! -f "$mdir/transforms/talairach_with_skull.lta" ]]; then - echo " " | tee -a $LF - echo "============= Computing Talairach Transform ============" | tee -a $LF - echo " " | tee -a $LF - echo "\"$binpath/talairach-reg.sh\" \"$mdir\" \"$atlas3T\" \"$LF\"" | tee -a "$LF" + { + echo " " + echo "============= Computing Talairach Transform ============" + echo " " + echo "\"$binpath/talairach-reg.sh\" \"$mdir\" \"$atlas3T\" \"$LF\"" + } | tee -a "$LF" "$binpath/talairach-reg.sh" "$mdir" "$atlas3T" "$LF" fi # ============================= BRAINMASK ============================================== +{ + echo " " + echo "============ Creating brainmask from aseg and nu or T1 ============" + echo " " +} | tee -a $LF -echo " " | tee -a $LF -echo "============ Creating brainmask from aseg and nu or T1 ============" | tee -a $LF -echo " " | tee -a $LF # create norm by masking nu cmd="mri_mask $mdir/nu.mgz $mdir/mask.mgz $mdir/norm.mgz" -RunIt "$cmd" $LF +RunIt "$cmd" "$LF" if [ "$get_t1" == "1" ] then # create T1.mgz from nu (!! here we could also try passing aseg?) cmd="mri_normalize -g 1 -seed 1234 -mprage $mdir/nu.mgz $mdir/T1.mgz $noconform_if_hires" - RunIt "$cmd" $LF + RunIt "$cmd" "$LF" # create brainmask by masking T1 cmd="mri_mask $mdir/T1.mgz $mdir/mask.mgz $mdir/brainmask.mgz" - RunIt "$cmd" $LF + RunIt "$cmd" "$LF" else # create brainmask by linkage to norm.mgz (masked nu.mgz) - pushd $mdir - softlink_or_copy "norm.mgz" "brainmask.mgz" $LF - popd + pushd "$mdir" > /dev/null || ( echo "Could not cd to $mdir" ; exit 1 ) + softlink_or_copy "norm.mgz" "brainmask.mgz" "$LF" + popd > /dev/null || (echo "Could not popd" ; exit 1 ) fi # ============================= CC SEGMENTATION ============================================ -echo " " | tee -a $LF -echo "============ Creating and adding CC Segmentation ============" | tee -a $LF -echo " " | tee -a $LF +{ + echo " " + echo "============ Creating and adding CC Segmentation ============" + echo " " +} | tee -a $LF # create aseg.auto including corpus callosum segmentation and 46 sec, requires norm.mgz # Note: if original input segmentation already contains CC, this will exit with ERROR # in the future maybe check and skip this step (and next) cmd="mri_cc -aseg aseg.auto_noCCseg.mgz -o aseg.auto.mgz -lta $mdir/transforms/cc_up.lta $subject" -RunIt "$cmd" $LF +RunIt "$cmd" "$LF" # add CC into aparc.DKTatlas+aseg.deep (not sure if this is really needed) cmd="$python ${binpath}paint_cc_into_pred.py -in_cc $mdir/aseg.auto.mgz -in_pred $asegdkt_segfile -out $mdir/aparc.DKTatlas+aseg.deep.withCC.mgz" -RunIt "$cmd" $LF +RunIt "$cmd" "$LF" # ============================= FILLED ===================================================== -echo " " | tee -a $LF -echo "========= Creating filled from brain (brainfinalsurfs, wm.asegedit, wm) =======" | tee -a $LF -echo " " | tee -a $LF +{ + echo " " + echo "========= Creating filled from brain (brainfinalsurfs, wm.asegedit, wm) =======" + echo " " +} | tee -a $LF # filled is needed to generate initial WM surfaces cmd="recon-all -s $subject -asegmerge -normalization2 -maskbfs -segmentation -fill $hiresflag $fsthreads" -RunIt "$cmd" $LF +RunIt "$cmd" "$LF" @@ -591,25 +552,27 @@ RunIt "$cmd" $LF # ================================================== SURFACES ============================================================== # ======= -CMDFS="" +CMDFS=() for hemi in lh rh; do CMDF="$SUBJECTS_DIR/$subject/scripts/$hemi.processing.cmdf" - CMDFS="$CMDFS $CMDF" - rm -rf $CMDF - echo "#!/bin/bash" > $CMDF + CMDFS+=("$CMDF") + rm -rf "$CMDF" + echo "#!/bin/bash" > "$CMDF" # ============================= TESSELATE - SMOOTH ===================================================== - echo "echo " | tee -a $CMDF - echo "echo \"================== Creating surfaces $hemi - orig.nofix ==================\"" | tee -a $CMDF - echo "echo " | tee -a $CMDF + { + echo "echo \" \"" + echo "echo \"================== Creating surfaces $hemi - orig.nofix ==================\"" + echo "echo \" \"" + } | tee -a "$CMDF" if [ "$fstess" == "1" ] then cmd="recon-all -subject $subject -hemi $hemi -tessellate -smooth1 -no-isrunning $hiresflag $fsthreads" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" else # instead of mri_tesselate lego land use marching cube @@ -621,57 +584,61 @@ for hemi in lh rh; do # extract initial surface "?h.orig.nofix" cmd="mri_pretess $mdir/filled.mgz $hemivalue $mdir/brain.mgz $mdir/filled-pretess$hemivalue.mgz" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" # Marching cube does not return filename and wrong volume info! outmesh=$sdir/$hemi.orig.nofix$hires_surface_suffix cmd="mri_mc $mdir/filled-pretess$hemivalue.mgz $hemivalue $outmesh" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" # Rewrite surface orig.nofix to fix vertex locs bug (scannerRAS instead of surfaceRAS set with mc) #cmd="$python ${binpath}rewrite_mc_surface.py --input $outmesh --output $outmesh --filename_pretess $mdir/filled-pretess$hemivalue.mgz" - #RunIt "$cmd" $LF $CMDF + #RunIt "$cmd" "$LF" "$CMDF" # Check if the surfaceRAS was correctly set and exit otherwise (sanity check in case nibabel changes their default header behaviour) - cmd="mris_info $outmesh | tr -s ' ' | grep -q 'vertex locs : surfaceRAS'" - echo "echo \"$cmd\" " | tee -a $CMDF - echo "$timecmd $cmd " | tee -a $CMDF - echo "if [ \${PIPESTATUS[1]} -ne 0 ] ; then echo \"Incorrect header information detected in $outmesh: vertex locs is not set to surfaceRAS. Exiting... \"; exit 1 ; fi" >> $CMDF + { + cmd="mris_info $outmesh | tr -s ' ' | grep -q 'vertex locs : surfaceRAS'" + echo "echo \"$cmd\"" + echo "$timecmd $cmd" + } | tee -a "$CMDF" + echo "if [ \${PIPESTATUS[1]} -ne 0 ] ; then echo \"Incorrect header information detected in $outmesh: vertex locs is not set to surfaceRAS. Exiting... \"; exit 1 ; fi" >> "$CMDF" # Reduce to largest component (usually there should only be one) cmd="mris_extract_main_component $outmesh $outmesh" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" # for hires decimate mesh - if [ ! -z "$hiresflag" ]; then + if [ -n "$hiresflag" ]; then DecimationFaceArea="0.5" # Reduce the number of faces such that the average face area is # DecimationFaceArea. If the average face area is already more # than DecimationFaceArea, then the surface is not changed. # set cmd = (mris_decimate -a $DecimationFaceArea ../surf/$hemi.orig.nofix.predec ../surf/$hemi.orig.nofix) cmd="mris_remesh --desired-face-area $DecimationFaceArea --input $outmesh --output $sdir/$hemi.orig.nofix" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" fi - # -smooth1 (explicitly state 10 iteration (default) but may change in future) cmd="mris_smooth -n 10 -nw -seed 1234 $sdir/$hemi.orig.nofix $sdir/$hemi.smoothwm.nofix" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" fi - # ============================= INFLATE1 - QSPHERE ===================================================== - echo "echo " | tee -a $CMDF - echo "echo \"=================== Creating surfaces $hemi - qsphere ====================\"" | tee -a $CMDF - echo "echo " | tee -a $CMDF + { + echo "echo \"\"" + echo "echo \"=================== Creating surfaces $hemi - qsphere ====================\"" + echo "echo \"\"" + } | tee -a "$CMDF" + #surface inflation (54sec both hemis) (needed for qsphere and for topo-fixer) cmd="recon-all -subject $subject -hemi $hemi -inflate1 -no-isrunning $hiresflag $fsthreads" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" + if [ "$fsqsphere" == "1" ] then # quick spherical mapping (2min48sec) cmd="recon-all -subject $subject -hemi $hemi -qsphere -no-isrunning $hiresflag $fsthreads" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" else # instead of mris_sphere, directly project to sphere with spectral approach # equivalent to -qsphere @@ -679,14 +646,17 @@ for hemi in lh rh; do cmd="$python ${binpath}spherically_project_wrapper.py --hemi $hemi --sdir $sdir" printf -v tmp %q "$python" cmd="$cmd --subject $subject --threads=$threads --py ${tmp} --binpath ${binpath}" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" fi # ============================= FIX - WHITEPREAPARC - CORTEXLABEL ============================================ - echo "echo " | tee -a $CMDF - echo "echo \"=================== Creating surfaces $hemi - fix ========================\"" | tee -a $CMDF - echo "echo " | tee -a $CMDF + { + echo "echo \"\"" + echo "echo \"=================== Creating surfaces $hemi - fix ========================\"" + echo "echo \"\"" + } | tee -a "$CMDF" + cmd="recon-all -subject $subject -hemi $hemi -fix -no-isrunning $hiresflag $fsthreads" RunIt "$cmd" $LF $CMDF @@ -697,7 +667,7 @@ for hemi in lh rh; do RunIt "$cmd" $LF $CMDF cmd="recon-all -subject $subject -hemi $hemi -autodetgwstats -white-preaparc -cortex-label -no-isrunning $hiresflag $fsthreads" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" ## copy nofix to orig and inflated for next step # -white (don't know how to call this from recon-all as it needs -whiteonly setting and by default it also creates the pial. # create first WM surface white.preaparc from topo fixed orig surf, also first cortex label (1min), (3min for deep learning surf) @@ -705,43 +675,51 @@ for hemi in lh rh; do # ============================= INFLATE2 - CURVHK =================================================== - echo "echo \" \"" | tee -a $CMDF - echo "echo \"================== Creating surfaces $hemi - inflate2 ====================\"" | tee -a $CMDF - echo "echo \" \"" | tee -a $CMDF + { + echo "echo \"\"" + echo "echo \"================== Creating surfaces $hemi - inflate2 ====================\"" + echo "echo \"\"" + } | tee -a "$CMDF" + # create nicer inflated surface from topo fixed (not needed, just later for visualization) cmd="recon-all -subject $subject -hemi $hemi -smooth2 -inflate2 -curvHK -no-isrunning $hiresflag $fsthreads" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" # ============================= MAP-DKT ========================================================== - echo "echo \" \"" | tee -a $CMDF - echo "echo \"=========== Creating surfaces $hemi - map input asegdkt_segfile to surf ===============\"" | tee -a $CMDF - echo "echo \" \"" | tee -a $CMDF + { + echo "echo \" \"" + echo "echo \"=========== Creating surfaces $hemi - map input asegdkt_segfile to surf ===============\"" + echo "echo \" \"" + } | tee -a "$CMDF" + # sample input segmentation (aparc.DKTatlas+aseg orig) onto wm surface: # map input aparc to surface (requires thickness (and thus pail) to compute projfrac 0.5), here we do projmm which allows us to compute based only on white # this is dangerous, as some cortices could be < 0.6 mm, but then there is no volume label probably anyway. # Also note that currently we cannot mask non-cortex regions here, should be done in mris_anatomical stats later # the smoothing helps #cmd="mris_sample_parc -ct $FREESURFER_HOME/average/colortable_desikan_killiany.txt -file ${binpath}$hemi.DKTatlaslookup.txt -projmm 0.6 -f 5 -surf white.preaparc $subject $hemi aparc.DKTatlas+aseg.orig.mgz aparc.DKTatlas.mapped.prefix.annot" - #RunIt "$cmd" $LF $CMDF + #RunIt "$cmd" "$LF" "$CMDF" #cmd="$python ${binpath}smooth_aparc.py --insurf $sdir/$hemi.white.preaparc --inaparc $ldir/$hemi.aparc.DKTatlas.mapped.prefix.annot --incort $ldir/$hemi.cortex.label --outaparc $ldir/$hemi.aparc.DKTatlas.mapped.annot" - #RunIt "$cmd" $LF $CMDF + #RunIt "$cmd" "$LF" "$CMDF" cmd="$python ${binpath}sample_parc.py --inseg $mdir/aparc.DKTatlas+aseg.orig.mgz --insurf $sdir/$hemi.white.preaparc --incort $ldir/$hemi.cortex.label --outaparc $ldir/$hemi.aparc.DKTatlas.mapped.annot --seglut ${binpath}$hemi.DKTatlaslookup.txt --surflut ${binpath}DKTatlaslookup.txt --projmm 0.6 --radius 2" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" # ============================= SPHERE - SURFREG (optional) ============================================== # if we segment with FS or if surface registration is requested do it here: if [ "$fsaparc" == "1" ] || [ "$fssurfreg" == "1" ] ; then - echo "echo \" \"" | tee -a $CMDF - echo "echo \"============ Creating surfaces $hemi - FS sphere, surfreg ===============\"" | tee -a $CMDF - echo "echo \" \"" | tee -a $CMDF + { + echo "echo \" \"" + echo "echo \"============ Creating surfaces $hemi - FS sphere, surfreg ===============\"" + echo "echo \" \"" + } | tee -a "$CMDF" # Surface registration for cross-subject correspondence (registration to fsaverage) cmd="recon-all -subject $subject -hemi $hemi -sphere $hiresflag -no-isrunning $fsthreads" - RunIt "$cmd" $LF "$CMDF" + RunIt "$cmd" "$LF" "$CMDF" # (mr) FIX: sometimes FreeSurfer Sphere Reg. fails and moves pre and post central # one gyrus too far posterior, FastSurferCNN's image-based segmentation does not @@ -751,18 +729,18 @@ for hemi in lh rh; do # (note the former fix, initializing with pre-central label, is not working in FS7.2 # as they broke the label initialization in mris_register) cmd="$python ${binpath}/rotate_sphere.py \ - --srcsphere $sdir/${hemi}.sphere \ - --srcaparc $ldir/$hemi.aparc.DKTatlas.mapped.annot \ - --trgsphere $FREESURFER_HOME/subjects/fsaverage/surf/${hemi}.sphere \ - --trgaparc $FREESURFER_HOME/subjects/fsaverage/label/${hemi}.aparc.annot \ - --out $sdir/${hemi}.angles.txt" - RunIt "$cmd" $LF "$CMDF" + --srcsphere $sdir/${hemi}.sphere \ + --srcaparc $ldir/$hemi.aparc.DKTatlas.mapped.annot \ + --trgsphere $FREESURFER_HOME/subjects/fsaverage/surf/${hemi}.sphere \ + --trgaparc $FREESURFER_HOME/subjects/fsaverage/label/${hemi}.aparc.annot \ + --out $sdir/${hemi}.angles.txt" + RunIt "$cmd" "$LF" "$CMDF" # 2. use global rotation as initialization to non-linear registration: cmd="mris_register -curv -norot -rotate \`cat $sdir/${hemi}.angles.txt\` \ - $sdir/${hemi}.sphere \ - $FREESURFER_HOME/average/${hemi}.folding.atlas.acfb40.noaparc.i12.2016-08-02.tif \ - $sdir/${hemi}.sphere.reg" - RunIt "$cmd" $LF "$CMDF" + $sdir/${hemi}.sphere \ + $FREESURFER_HOME/average/${hemi}.folding.atlas.acfb40.noaparc.i12.2016-08-02.tif \ + $sdir/${hemi}.sphere.reg" + RunIt "$cmd" "$LF" "$CMDF" # command to generate new aparc to check if registration was OK # run only for debugging #cmd="mris_ca_label -l $SUBJECTS_DIR/$subject/label/${hemi}.cortex.label \ @@ -774,52 +752,57 @@ for hemi in lh rh; do # ============================= WHITE & PIAL & (FSSURFSEG optional) =============================================== if [ "$fsaparc" == "1" ] ; then - echo "echo \" \"" | tee -a $CMDF - echo "echo \"============ Creating surfaces $hemi - FS asegdkt_segfile..pial ===============\"" | tee -a $CMDF - echo "echo \" \"" | tee -a $CMDF + { + echo "echo \" \"" + echo "echo \"============ Creating surfaces $hemi - FS asegdkt_segfile..pial ===============\"" + echo "echo \" \"" + } | tee -a "$CMDF" + # 20-25 min for traditional surface segmentation (each hemi) # this creates aparc and creates pial using aparc, also computes jacobian cmd="recon-all -subject $subject -hemi $hemi -jacobian_white -avgcurv -cortparc -white -pial -no-isrunning $hiresflag $fsthreads" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" # Here insert DoT2Pial later! else - echo "echo \" \"" | tee -a $CMDF - echo "echo \"================ Creating surfaces $hemi - white and pial direct ===================\"" | tee -a $CMDF - echo "echo \" \"" | tee -a $CMDF + { + echo "echo \" \"" + echo "echo \"================ Creating surfaces $hemi - white and pial direct ===================\"" + echo "echo \" \"" + } | tee -a "$CMDF" # 4 min compute white : - echo "pushd $mdir" >> $CMDF + echo "pushd $mdir > /dev/null" >> "$CMDF" cmd="mris_place_surface --adgws-in ../surf/autodet.gw.stats.$hemi.dat --seg aseg.presurf.mgz --wm wm.mgz --invol brain.finalsurfs.mgz --$hemi --i ../surf/$hemi.white.preaparc --o ../surf/$hemi.white --white --nsmooth 0 --rip-label ../label/$hemi.cortex.label --rip-bg --rip-surf ../surf/$hemi.white.preaparc --aparc ../label/$hemi.aparc.DKTatlas.mapped.annot" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" # 4 min compute pial : cmd="mris_place_surface --adgws-in ../surf/autodet.gw.stats.$hemi.dat --seg aseg.presurf.mgz --wm wm.mgz --invol brain.finalsurfs.mgz --$hemi --i ../surf/$hemi.white --o ../surf/$hemi.pial.T1 --pial --nsmooth 0 --rip-label ../label/$hemi.cortex+hipamyg.label --pin-medial-wall ../label/$hemi.cortex.label --aparc ../label/$hemi.aparc.DKTatlas.mapped.annot --repulse-surf ../surf/$hemi.white --white-surf ../surf/$hemi.white" - RunIt "$cmd" $LF $CMDF - echo "popd" >> $CMDF + RunIt "$cmd" "$LF" "$CMDF" + echo "popd > /dev/null" >> "$CMDF" # Here insert DoT2Pial later --> if T2pial is not run, need to softlink pial.T1 to pial! - echo "pushd $sdir" >> $CMDF - softlink_or_copy "$hemi.pial.T1" "$hemi.pial" $LF $CMDF - echo "popd" >> $CMDF + echo "pushd $sdir > /dev/null" >> "$CMDF" + softlink_or_copy "$hemi.pial.T1" "$hemi.pial" "$LF" "$CMDF" + echo "popd > /dev/null" >> "$CMDF" - echo "pushd $mdir" >> $CMDF + echo "pushd $mdir > /dev/null" >> "$CMDF" # these are run automatically in fs7* recon-all and cannot be called directly without -pial flag (or other t2 flags) if [ "$fssurfreg" == "1" ] ; then # jacobian needs sphere reg which might be turned off by user (on by default) cmd="mris_jacobian ../surf/$hemi.white ../surf/$hemi.sphere.reg ../surf/$hemi.jacobian_white" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" fi cmd="mris_place_surface --curv-map ../surf/$hemi.white 2 10 ../surf/$hemi.curv" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" cmd="mris_place_surface --area-map ../surf/$hemi.white ../surf/$hemi.area" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" cmd="mris_place_surface --curv-map ../surf/$hemi.pial 2 10 ../surf/$hemi.curv.pial" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" cmd="mris_place_surface --area-map ../surf/$hemi.pial ../surf/$hemi.area.pial" - RunIt "$cmd" $LF $CMDF + RunIt "$cmd" "$LF" "$CMDF" cmd="mris_place_surface --thickness ../surf/$hemi.white ../surf/$hemi.pial 20 5 ../surf/$hemi.thickness" - RunIt "$cmd" $LF $CMDF - echo "popd" >> $CMDF + RunIt "$cmd" "$LF" "$CMDF" + echo "popd > /dev/null" >> "$CMDF" fi @@ -827,15 +810,17 @@ for hemi in lh rh; do # in FS7 curvstats moves here cmd="recon-all -subject $subject -hemi $hemi -curvstats -no-isrunning $hiresflag $fsthreads" - RunIt "$cmd" $LF "$CMDF" + RunIt "$cmd" "$LF" "$CMDF" if [ "$DoParallel" == "0" ] ; then - echo " " | tee -a $LF - echo " RUNNING $hemi sequentially ... " | tee -a $LF - echo " " | tee -a $LF + { + echo " " + echo " RUNNING $hemi sequentially ... " + echo " " + } | tee -a "$LF" chmod u+x $CMDF RunIt "$CMDF" $LF fi @@ -846,80 +831,90 @@ done # hemi loop ---------------------------------- if [ "$DoParallel" == 1 ] ; then - echo " " | tee -a $LF - echo " RUNNING HEMIs in PARALLEL !!! " | tee -a $LF - echo " " | tee -a $LF - RunBatchJobs $LF $CMDFS + { + echo "" + echo " RUNNING HEMIs in PARALLEL !!! " + echo "" + } |& tee -a "$LF" + RunBatchJobs "$LF" "${CMDFS[@]}" fi - # ============================= RIBBON =============================================== - echo " " | tee -a $LF - echo "============================ Creating surfaces - ribbon ===========================" | tee -a $LF - echo " " | tee -a $LF - # -cortribbon 4 minutes, ribbon is used in mris_anatomical stats to remove voxels from surface-based volumes that should not be cortex - # anatomical stats can run without ribbon, but will omit some surface-based measures then +{ + echo "" + echo "============================ Creating surfaces - ribbon ===========================" + echo "" +} | tee -a "$LF" + # -cortribbon 4 minutes, ribbon is used in mris_anatomical stats to remove voxels from surface based volumes that should not be cortex + # anatomical stats can run without ribbon, but will omit some surface based measures then # wmparc needs ribbon, probably other stuff (aparc to aseg etc). # So lets run it to have these measures below. cmd="recon-all -subject $subject -cortribbon $hiresflag $fsthreads" - RunIt "$cmd" $LF + RunIt "$cmd" "$LF" # ============================= FSAPARC - parc23 surfcon hypo ... ========================================= if [ "$fsaparc" == "1" ] ; then - echo " " | tee -a $LF - echo "============= Creating surfaces - other FS asegdkt_segfile and stats =======================" | tee -a $LF - echo " " | tee -a $LF + { + echo "" + echo "============= Creating surfaces - other FS asegdkt_segfile and stats =======================" + echo "" + } | tee -a "$LF" cmd="recon-all -subject $subject -cortparc2 -cortparc3 -pctsurfcon -hyporelabel $hiresflag $fsthreads" - RunIt "$cmd" $LF - cmd="recon-all -subject $subject -apas2aseg -aparc2aseg -wmparc -parcstats -parcstats2 -parcstats3 -segstats $hiresflag $fsthreads" - RunIt "$cmd" $LF + RunIt "$cmd" "$LF" + + cmd="recon-all -subject $subject -apas2aseg -aparc2aseg -wmparc -parcstats -parcstats2 -parcstats3 $hiresflag $fsthreads" + RunIt "$cmd" "$LF" # removed -balabels here and do that below independent of fsaparc flag + # removed -segstats here (now part of mri_segstats.py/segstats.py fi # (FS-APARC) # ============================= MAPPED SURF-STATS ========================================= - echo " " | tee -a $LF - echo "===================== Creating surfaces - mapped stats =========================" | tee -a $LF - echo " " | tee -a $LF +{ + echo "" + echo "===================== Creating surfaces - mapped stats =========================" + echo "" +} | tee -a "$LF" + # 2x18sec create stats from mapped aparc for hemi in lh rh; do - cmd="mris_anatomical_stats -th3 -mgz -cortex $ldir/$hemi.cortex.label -f $sdir/../stats/$hemi.aparc.DKTatlas.mapped.stats -b -a $ldir/$hemi.aparc.DKTatlas.mapped.annot -c $ldir/aparc.annot.mapped.ctab $subject $hemi white" - RunIt "$cmd" $LF + cmd="mris_anatomical_stats -th3 -mgz -cortex $ldir/$hemi.cortex.label -f $statsdir/$hemi.aparc.DKTatlas.mapped.stats -b -a $ldir/$hemi.aparc.DKTatlas.mapped.annot -c $ldir/aparc.annot.mapped.ctab $subject $hemi white" + RunIt "$cmd" "$LF" done # ============================= FASTSURFER - surfcon hypo stats ========================================= if [ "$fsaparc" == "0" ] ; then - - echo " " | tee -a $LF - echo "============= Creating surfaces - pctsurfcon, hypo, segstats ====================" | tee -a $LF - echo " " | tee -a $LF - + { + echo "" + echo "============= Creating surfaces - pctsurfcon, hypo, segstats ====================" + echo "" + } | tee -a "$LF" # pctsurfcon (has no way to specify which annot to use, so we need to link ours as aparc is not available) - pushd $ldir - softlink_or_copy "lh.aparc.DKTatlas.mapped.annot" "lh.aparc.annot" $LF - softlink_or_copy "rh.aparc.DKTatlas.mapped.annot" "rh.aparc.annot" $LF - popd + pushd "$ldir" > /dev/null || (echo "Could not cd to $ldir" ; exit 1) + softlink_or_copy "lh.aparc.DKTatlas.mapped.annot" "lh.aparc.annot" "$LF" + softlink_or_copy "rh.aparc.DKTatlas.mapped.annot" "rh.aparc.annot" "$LF" + popd > /dev/null || (echo "Could not popd" ; exit 1) for hemi in lh rh; do cmd="pctsurfcon --s $subject --$hemi-only" - RunIt "$cmd" $LF + RunIt "$cmd" "$LF" done - pushd $ldir - cmd="rm *h.aparc.annot" - RunIt "$cmd" $LF - popd + pushd "$ldir" > /dev/null || (echo "Could not cd to $ldir" ; exit 1) + cmd="rm *h.aparc.annot" + RunIt "$cmd" "$LF" + popd > /dev/null || (echo "Could not popd" ; exit 1) # 25 sec hyporelabel run whatever else can be done without sphere, cortical ribbon and segmentations # -hyporelabel creates aseg.presurf.hypos.mgz from aseg.presurf.mgz # -apas2aseg creates aseg.mgz by editing aseg.presurf.hypos.mgz with surfaces cmd="recon-all -subject $subject -hyporelabel -apas2aseg $hiresflag $fsthreads" - RunIt "$cmd" $LF + RunIt "$cmd" "$LF" fi @@ -928,36 +923,123 @@ fi # creating aparc.DKTatlas+aseg.mapped.mgz by mapping aparc.DKTatlas.mapped from surface to aseg.mgz # (should be a nicer aparc+aseg compared to orig CNN segmentation, due to surface updates) cmd="mri_surf2volseg --o $mdir/aparc.DKTatlas+aseg.mapped.mgz --label-cortex --i $mdir/aseg.mgz --threads $threads --lh-annot $ldir/lh.aparc.DKTatlas.mapped.annot 1000 --lh-cortex-mask $ldir/lh.cortex.label --lh-white $sdir/lh.white --lh-pial $sdir/lh.pial --rh-annot $ldir/rh.aparc.DKTatlas.mapped.annot 2000 --rh-cortex-mask $ldir/rh.cortex.label --rh-white $sdir/rh.white --rh-pial $sdir/rh.pial" - RunIt "$cmd" $LF + RunIt "$cmd" "$LF" # ============================= FASTSURFER - STATS ========================================= if [ "$fsaparc" == "0" ] ; then # get stats for the aseg (note these are surface fine tuned, that may be good or bad, below we also do the stats for the input aseg (plus some processing) - cmd="recon-all -subject $subject -segstats $hiresflag $fsthreads" - RunIt "$cmd" $LF + # cmd="recon-all -subject $subject -segstats $hiresflag $fsthreads" + if [[ "$segstats_legacy" == "true" ]] ; then + cmd=($python "$FASTSURFER_HOME/FastSurferCNN/mri_brainvol_stats.py" + --subject "$subject") + RunIt "$(echo_quoted "${cmd[@]}")" "$LF" + + cmd=($python "$FASTSURFER_HOME/FastSurferCNN/mri_segstats.py" --seed 1234 + --seg "$mdir/aseg.mgz" --sum "$statsdir/aseg.stats" --pv "$mdir/norm.mgz" + "--in-intensity-name" norm "--in-intensity-units" MR --subject "$subject" + --surf-wm-vol --ctab "$FREESURFER_HOME/ASegStatsLUT.txt" --etiv + --threads "$threads") +# cmd="$python $FASTSURFER_HOME/FastSurferCNN/mri_segstats.py --seed 1234 --seg $mdir/wmparc.mgz --sum $statsdir/wmparc.stats --pv $mdir/norm.mgz --in-intensity-name norm --in-intensity-units MR --subject $subject --surf-wm-vol --ctab $FREESURFER_HOME/WMParcStatsLUT.txt --etiv" + else + # calculate brainvol stats and aseg stats with segstats.py + cmd=($python "$FASTSURFER_HOME/FastSurferCNN/segstats.py" --sid "$subject" + --segfile "$mdir/aseg.mgz" --segstatsfile "$statsdir/aseg.stats" + --pvfile "$mdir/norm.mgz" --normfile "$mdir/norm.mgz" --threads "$threads" + # --excl-ctxgmwm: exclude Left/Right WM / Cortex despite ASegStatsLUT.txt + --excludeid 0 2 3 41 42 + --lut "$FREESURFER_HOME/ASegStatsLUT.txt" --empty + measures --compute "BrainSeg" "BrainSegNotVent" "VentricleChoroidVol" + "lhCortex" "rhCortex" "Cortex" "lhCerebralWhiteMatter" + "rhCerebralWhiteMatter" "CerebralWhiteMatter" + "SubCortGray" "TotalGray" "SupraTentorial" + "SupraTentorialNotVent" "Mask($mdir/mask.mgz)" + "BrainSegVol-to-eTIV" "MaskVol-to-eTIV" "lhSurfaceHoles" + "rhSurfaceHoles" "SurfaceHoles" + "EstimatedTotalIntraCranialVol") + RunIt "$(echo_quoted "${cmd[@]}")" "$LF" + echo "Extract the brainvol stats section from segstats output." | tee -a "$LF" + # ... so stats/brainvol.stats also exists (but it is slightly different +# cmd="recon-all -subject $subject -segstats $hiresflag $fsthreads" +# RunIt "$cmd" "$LF" + + # this call is only "required" to "compute" brainvol.stats, so --normfile/--pvfile + # are not required + cmd=($python "$FASTSURFER_HOME/FastSurferCNN/segstats.py" --sid "$subject" + --segfile "$mdir/aseg.mgz" --pvfile "$mdir/norm.mgz" + --measure_only --threads "$threads" --segstatsfile "$statsdir/brainvol.stats" + measures --file "$statsdir/aseg.stats" + --import "BrainSeg" "BrainSegNotVent" "SupraTentorial" + "SupraTentorialNotVent" "SubCortGray" "lhCortex" "rhCortex" + "Cortex" "TotalGray" "lhCerebralWhiteMatter" + "rhCerebralWhiteMatter" "CerebralWhiteMatter" "Mask" + --compute "SupraTentorialNotVentVox" "BrainSegNotVentSurf" + "VentricleChoroidVol") + fi + RunIt "$(echo_quoted "${cmd[@]}")" "$LF" fi - # ============================= MAPPED-WMPARC ========================================= - -echo " " | tee -a $LF -echo "===================== Creating wmparc from mapped =======================" | tee -a $LF -echo " " | tee -a $LF - - # 1m 11sec also create stats for aseg.presurf.hypos (which is basically the aseg derived from the input with CC and hypos) - # difference between this and the surface improved one above are probably tiny, so the surface improvement above can probably be skipped to save time - cmd="mri_segstats --seed 1234 --seg $mdir/aseg.presurf.hypos.mgz --sum $mdir/../stats/aseg.presurf.hypos.stats --pv $mdir/norm.mgz --empty --brainmask $mdir/brainmask.mgz --brain-vol-from-seg --excludeid 0 --excl-ctxgmwm --supratent --subcortgray --in $mdir/norm.mgz --in-intensity-name norm --in-intensity-units MR --etiv --surf-wm-vol --surf-ctx-vol --totalgray --euler --ctab /$FREESURFER_HOME/ASegStatsLUT.txt --subject $subject" - RunIt "$cmd" $LF - +{ + echo "" + echo "===================== Creating wmparc from mapped =======================" + echo "" +} | tee -a "$LF" + + if [[ "$segstats_legacy" == "true" ]] ; then + # 1m 11sec also create stats for aseg.presurf.hypos (which is basically the aseg derived from the input with CC and + # hypos) difference between this and the surface improved one above are probably tiny, so the surface improvement + # above can probably be skipped to save time + cmd=($python "$FASTSURFER_HOME/FastSurferCNN/mri_segstats.py" --seed 1234 + --seg "$mdir/aseg.presurf.hypos.mgz" --sum "$statsdir/aseg.presurf.hypos.stats" + --pv "$mdir/norm.mgz" --empty --brainmask "$mdir/brainmask.mgz" + --brain-vol-from-seg --excludeid 0 --excl-ctxgmwm --supratent --subcortgray + "--in" "$mdir/norm.mgz" "--in-intensity-name" norm "--in-intensity-units" MR + --etiv --surf-wm-vol --surf-ctx-vol --totalgray --euler + --ctab "$FREESURFER_HOME/ASegStatsLUT.txt" --subject "$subject") + else + # segstats.py version of the mri_segstats call + cmd=($python "$FASTSURFER_HOME/FastSurferCNN/segstats.py" --sid "$subject" + --segfile "$mdir/aseg.presurf.hypos.mgz" --normfile "$mdir/norm.mgz" + --pvfile "$mdir/norm.mgz" --segstatsfile "$statsdir/aseg.presurf.hypos.stats" + # --excl-ctxgmwm: exclude Left/Right WM / Cortex despite ASegStatsLUT.txt + --excludeid 0 2 3 41 42 + --lut "$FREESURFER_HOME/ASegStatsLUT.txt" --threads "$threads" --empty + measures --file "$statsdir/aseg.stats" --import "all") + # --compute "Mask($mask_name)" "BrainSeg" "BrainSegNotVent" "SupraTentorial" + # "SupraTentorialNotVent" "SubCortGray" "EstimatedTotalIntraCranialVol" + # "rhCerebralWhiteMatter" "lhCerebralWhiteMatter" "CerebralWhiteMatter" + # "rhCortex" "lhCortex" "Cortex" "TotalGray" "rhSurfaceHoles" + # "lhSurfaceHoles" "SurfaceHoles" "BrainSegVol-to-eTIV" "MaskVol-to-eTIV" + fi + RunIt "$(echo_quoted "${cmd[@]}")" "$LF" # -wmparc based on mapped aparc labels (from input asegdkt_segfile) (1min40sec) needs ribbon and we need to point it to aparc.mapped: cmd="mri_surf2volseg --o $mdir/wmparc.DKTatlas.mapped.mgz --label-wm --i $mdir/aparc.DKTatlas+aseg.mapped.mgz --threads $threads --lh-annot $ldir/lh.aparc.DKTatlas.mapped.annot 3000 --lh-cortex-mask $ldir/lh.cortex.label --lh-white $sdir/lh.white --lh-pial $sdir/lh.pial --rh-annot $ldir/rh.aparc.DKTatlas.mapped.annot 4000 --rh-cortex-mask $ldir/rh.cortex.label --rh-white $sdir/rh.white --rh-pial $sdir/rh.pial" - RunIt "$cmd" $LF + RunIt "$cmd" "$LF" # takes a few mins - cmd="mri_segstats --seed 1234 --seg $mdir/wmparc.DKTatlas.mapped.mgz --sum $mdir/../stats/wmparc.DKTatlas.mapped.stats --pv $mdir/norm.mgz --excludeid 0 --brainmask $mdir/brainmask.mgz --in $mdir/norm.mgz --in-intensity-name norm --in-intensity-units MR --subject $subject --surf-wm-vol --ctab $FREESURFER_HOME/WMParcStatsLUT.txt" - RunIt "$cmd" $LF + #cmd="mri_segstats --seed 1234 --seg $mdir/wmparc.DKTatlas.mapped.mgz --sum $mdir/../stats/wmparc.DKTatlas.mapped.stats --pv $mdir/norm.mgz --excludeid 0 --brainmask $mdir/brainmask.mgz --in $mdir/norm.mgz --in-intensity-name norm --in-intensity-units MR --subject $subject --surf-wm-vol --ctab $FREESURFER_HOME/WMParcStatsLUT.txt" + if [[ "$segstats_legacy" == "true" ]] ; then + cmd=($python "$FASTSURFER_HOME/FastSurferCNN/mri_segstats.py" + --seed 1234 --seg "$mdir/wmparc.DKTatlas.mapped.mgz" + --sum "$statsdir/wmparc.DKTatlas.mapped.stats" --pv "$mdir/norm.mgz" + --excludeid 0 --brainmask "$mdir/brainmask.mgz" "--in" "$mdir/norm.mgz" + "--in-intensity-name" norm "--in-intensity-units" MR + --subject "$subject" --surf-wm-vol + --ctab "$FREESURFER_HOME/WMParcStatsLUT.txt") + else + # + cmd=($python "$FASTSURFER_HOME/FastSurferCNN/segstats.py" + --sid "$subject" --sd "$SUBJECTS_DIR" --pvfile "$mdir/norm.mgz" + --segfile "$mdir/wmparc.DKTatlas.mapped.mgz" --normfile "$mdir/norm.mgz" + --lut "$FREESURFER_HOME/WMParcStatsLUT.txt" --threads "$threads" + --segstatsfile "$statsdir/wmparc.DKTatlas.mapped.stats" --empty + measures --file "$statsdir/brainvol.stats" --import "Mask" + "VentricleChoroidVol" "rhCerebralWhiteMatter" "lhCerebralWhiteMatter" + "CerebralWhiteMatter") + fi + RunIt "$(echo_quoted "${cmd[@]}")" "$LF" # ============================= FASTSURFER - SYMLINKS ========================================= @@ -965,20 +1047,19 @@ echo " " | tee -a $LF # Create symlinks for downstream analysis (sub-segmentations, TRACULA, etc.) if [ "$fsaparc" == "0" ] ; then # Symlink of aparc.DKTatlas+aseg.mapped.mgz - pushd $mdir - softlink_or_copy "aparc.DKTatlas+aseg.mapped.mgz" "aparc.DKTatlas+aseg.mgz" $LF - softlink_or_copy "aparc.DKTatlas+aseg.mapped.mgz" "aparc+aseg.mgz" $LF - popd - - # Symlink of wmparc.mapped - pushd $mdir - softlink_or_copy "wmparc.DKTatlas.mapped.mgz" "wmparc.mgz" $LF - popd - - # Symbolic link for mapped surface parcellations - pushd $ldir - softlink_or_copy "lh.aparc.DKTatlas.mapped.annot" "lh.aparc.DKTatlas.annot" $LF - softlink_or_copy "rh.aparc.DKTatlas.mapped.annot" "rh.aparc.DKTatlas.annot" $LF + pushd "$mdir" > /dev/null || (echo "Could not cd to $mdir" ; exit 1) + softlink_or_copy "aparc.DKTatlas+aseg.mapped.mgz" "aparc.DKTatlas+aseg.mgz" "$LF" + softlink_or_copy "aparc.DKTatlas+aseg.mapped.mgz" "aparc+aseg.mgz" "$LF" + + # Symlink of wmparc.mapped + softlink_or_copy "wmparc.DKTatlas.mapped.mgz" "wmparc.mgz" "$LF" + popd > /dev/null || ( echo "Could not popd" ; exit 1 ) + + # Symbolic link for mapped surface parcellations + pushd "$ldir" > /dev/null || (echo "Could not cd to $ldir" ; exit 1) + softlink_or_copy "lh.aparc.DKTatlas.mapped.annot" "lh.aparc.DKTatlas.annot" "$LF" + softlink_or_copy "rh.aparc.DKTatlas.mapped.annot" "rh.aparc.DKTatlas.annot" "$LF" + popd > /dev/null || ( echo "Could not popd" ; exit 1 ) fi @@ -988,45 +1069,54 @@ echo " " | tee -a $LF if [ "$fssurfreg" == "1" ] ; then # can be produced if surf registration exists #cmd="recon-all -subject $subject -balabels $hiresflag $fsthreads" - #RunIt "$cmd" $LF + #RunIt "$cmd" "$LF" # here we run our version of balabels: mapping and annot creation is very fast # time is used in mris_anatomical_stats (called 4 times, BA and BA-thresh for each hemi) cmd="$python ${binpath}/fs_balabels.py --sd $SUBJECTS_DIR --sid $subject" - RunIt "$cmd" $LF + RunIt "$cmd" "$LF" fi - -echo " " | tee -a $LF -echo "================= DONE =========================================================" | tee -a $LF -echo " " | tee -a $LF - # Collect info -EndTime=`date` -tSecEnd=`date '+%s'` -tRunHours=`echo \($tSecEnd - $tSecStart\)/3600|bc -l` -tRunHours=`printf %6.3f $tRunHours` +EndTime=$(date) +tSecEnd=$(date '+%s') +tRunHours=$(($((tSecEnd - tSecStart))/3600)) +tRunHours=$(printf %6.3f "$tRunHours") -echo "Started at $StartTime " | tee -a $LF -echo "Ended at $EndTime" | tee -a $LF -echo "#@#%# recon-surf-run-time-hours $tRunHours" | tee -a $LF +{ + echo "" + echo "================= DONE =========================================================" + echo "" + + echo "Started at $StartTime" + echo "Ended at $EndTime" + echo "#@#%# recon-surf-run-time-hours $tRunHours" +} |& tee -a "$LF" # Create the Done File -echo "------------------------------" > $DoneFile -echo "SUBJECT $subject" >> $DoneFile -echo "START_TIME $StartTime" >> $DoneFile -echo "END_TIME $EndTime" >> $DoneFile -echo "RUNTIME_HOURS $tRunHours" >> $DoneFile -echo "USER `id -un`" >> $DoneFile 2> /dev/null -echo "HOST `hostname`" >> $DoneFile -echo "PROCESSOR `uname -m`" >> $DoneFile -echo "OS `uname -s`" >> $DoneFile -echo "UNAME `uname -a`" >> $DoneFile -echo "VERSION $VERSION" >> $DoneFile -echo "CMDPATH $0" >> $DoneFile -echo "CMDARGS ${inputargs[*]}" >> $DoneFile - -echo "recon-surf.sh $subject finished without error at `date`" | tee -a $LF +{ + echo "------------------------------" + echo "SUBJECT $subject" + echo "START_TIME $StartTime" + echo "END_TIME $EndTime" + echo "RUNTIME_HOURS $tRunHours" + # id -n sends an error message in docker (no user name), fall back to the USER environment variable or + username=$(id -un 2>&1) + if echo "$username" | grep -q "^id: " ; then + if [[ -n "$USER" ]] ; then username="$USER" + else username="$(id -u)" + fi + fi + echo "USER $username" + echo "HOST $(hostname)" + echo "PROCESSOR $(uname -m)" + echo "OS $(uname -s)" + echo "UNAME $(uname -a)" + echo "VERSION $VERSION" + echo "CMDPATH $0" + echo "CMDARGS ${inputargs[*]}" +} > "$DoneFile" +echo "recon-surf.sh $subject finished without error at $(date)" |& tee -a "$LF" cmd="$python ${binpath}utils/extract_recon_surf_time_info.py -i $LF -o $SUBJECTS_DIR/$subject/scripts/recon-surf_times.yaml" RunIt "$cmd" "/dev/null" diff --git a/recon_surf/talairach-reg.sh b/recon_surf/talairach-reg.sh index 5a9859a3..8f199a34 100755 --- a/recon_surf/talairach-reg.sh +++ b/recon_surf/talairach-reg.sh @@ -57,7 +57,7 @@ source "$binpath/functions.sh" mkdir -p $mdir/transforms mkdir -p $mdir/tmp -pushd "$mdir" || ( echo "Could not change to $mdir!" | tee -a "$LF" && exit 1) +pushd "$mdir" > /dev/null || ( echo "Could not change to $mdir!" | tee -a "$LF" && exit 1) # talairach.xfm: compute talairach full head (25sec) if [[ "$atlas3T" == "true" ]] @@ -88,12 +88,12 @@ RunIt "$cmd" $LF # all this is basically useless, as we did a good orig_nu already, including WM normalization # Since we do not run mri_em_register we sym-link other talairach transform files here -pushd $mdir/transforms || ( echo "ERROR: Could not change to the transforms directory $mdir/transforms!" | tee -a "$LF" && exit 1 ) -cmd="ln -sf talairach.xfm.lta talairach_with_skull.lta" -RunIt "$cmd" $LF -cmd="ln -sf talairach.xfm.lta talairach.lta" -RunIt "$cmd" $LF -popd || exit 1 +pushd "$mdir/transforms" > /dev/null || ( echo "ERROR: Could not change to the transforms directory $mdir/transforms!" | tee -a "$LF" && exit 1 ) + cmd="ln -sf talairach.xfm.lta talairach_with_skull.lta" + RunIt "$cmd" $LF + cmd="ln -sf talairach.xfm.lta talairach.lta" + RunIt "$cmd" $LF +popd > /dev/null || exit 1 # Add xfm to nu # (use orig_nu, if nu.mgz does not exist already); by default, it should exist @@ -103,4 +103,4 @@ fi cmd="mri_add_xform_to_header -c $mdir/transforms/talairach.xfm $src_nu_file $mdir/nu.mgz" RunIt "$cmd" $LF -popd || return \ No newline at end of file +popd > /dev/null || return \ No newline at end of file diff --git a/run_fastsurfer.sh b/run_fastsurfer.sh index 4a3ff5ca..8487eba8 100755 --- a/run_fastsurfer.sh +++ b/run_fastsurfer.sh @@ -301,289 +301,153 @@ do # make key lowercase key=$(echo "$1" | tr '[:upper:]' '[:lower:]') +shift # past argument + case $key in - --fs_license) - if [[ -f "$2" ]] + ############################################################## + # general options + ############################################################## + --fs_license) + if [[ -f "$1" ]] then - export FS_LICENSE="$2" + export FS_LICENSE="$1" else - echo "Provided FreeSurfer license file $2 could not be found. Make sure to provide the full path and name. Exiting..." - exit 1; + echo "ERROR: Provided FreeSurfer license file $1 could not be found. Make sure to provide the full path and name. Exiting..." + exit 1 fi - shift # past argument - shift # past value - ;; - --sid) - subject="$2" - shift # past argument - shift # past value - ;; - --sd) - sd="$2" - shift # past argument shift # past value ;; - --t1) - t1="$2" - shift # past argument - shift # past value - ;; - --t2) - t2="$2" - shift # past argument - shift # past value - ;; - --merged_segfile) - merged_segfile="$2" - shift # past argument - shift # past value - ;; - --seg | --asegdkt_segfile | --aparc_aseg_segfile) - if [[ "$key" == "--seg" ]] - then - echo "WARNING: --seg is deprecated and will be removed, use --asegdkt_segfile ." - fi - if [[ "$key" == "--aparc_aseg_segfile" ]] + + # options that *just* set a flag + #============================================================= + --allow_root) allow_root=("--allow_root") ;; + # options that set a variable + --sid) subject="$1" ; shift ;; + --sd) sd="$1" ; shift ;; + --t1) t1="$1" ; shift ;; + --t2) t2="$1" ; shift ;; + --seg_log) seg_log="$1" ; shift ;; + --conformed_name) conformed_name="$1" ; shift ;; + --norm_name) norm_name="$1" ; shift ;; + --norm_name_t2) norm_name_t2="$1" ; shift ;; + --seg|--asegdkt_segfile|--aparc_aseg_segfile) + if [[ "$key" != "--asegdkt_segfile" ]] then - echo "WARNING: --aparc_aseg_segfile is deprecated and will be removed, use --asegdkt_segfile " + echo "WARNING: --$key is deprecated and will be removed, use --asegdkt_segfile ." fi - asegdkt_segfile="$2" - shift # past argument - shift # past value - ;; - --asegdkt_statsfile) - asegdkt_statsfile="$2" - shift # past argument - shift # past value - ;; - --cereb_segfile) - cereb_segfile="$2" - shift # past argument - shift # past value - ;; - --cereb_statsfile) - cereb_statsfile="$2" - shift # past argument + asegdkt_segfile="$1" shift # past value ;; - --hypo_segfile) - hypo_segfile="$2" - shift # past argument - shift # past value - ;; - --hypo_statsfile) - hypo_statsfile="$2" - shift # past argument - shift # past value - ;; - --reg_mode) - mode=$(echo "$2" | tr "[:upper:]" "[:lower:]") - if [[ "$mode" =~ ^(none|coreg|robust)$ ]] ; then - hypvinn_flags+=(--regmode "$mode") + --vox_size) vox_size="$1" ; shift ;; + # --3t: both for surface pipeline and the --tal_reg flag + --3t) surf_flags=("${surf_flags[@]}" "--3T") ; atlas3T="true" ;; + --threads) threads="$1" ; shift ;; + --py) python="$1" ; shift ;; + -h|--help) usage ; exit ;; + --version) + if [[ "$#" -lt 1 ]] || [[ "$1" =~ ^-- ]]; then + # no more args or next arg starts with -- + version_and_quit="1" else - echo "Invalid --reg_mode option, must be 'none', 'coreg' or 'robust'." - exit 1 + case "$(echo "$1" | tr '[:upper:]' '[:lower:]')" in + all) version_and_quit="+checkpoints+git+pip" ;; + +*) version_and_quit="$1" ;; + *) echo "ERROR: Invalid option for --version: '$1', must be 'all' or [+checkpoints][+git][+pip]" + exit 1 + ;; + esac + shift fi - shift # past argument - shift # past value - ;; - --qc_snap) - hypvinn_flags+=(--qc_snap) - shift # past argument - ;; - --mask_name) - mask_name="$2" - shift # past argument - shift # past value - ;; - --norm_name) - norm_name="$2" - shift # past argument - shift # past value - ;; - --norm_name_t2) - norm_name_t2="$2" - shift # past argument - shift # past value - ;; - --aseg_segfile) - aseg_segfile="$2" - shift # past argument - shift # past value - ;; - --conformed_name) - conformed_name="$2" - shift # past argument - shift # past value ;; - --conformed_name_t2) - conformed_name_t2="$2" - shift # past argument - shift # past value - ;; - --seg_log) - seg_log="$2" - shift # past argument - shift # past value - ;; - --viewagg_device | --run_viewagg_on) + + ############################################################## + # seg-pipeline options + ############################################################## + + # common options for seg + #============================================================= + --surf_only) run_seg_pipeline="0" ;; + --no_biasfield) run_biasfield="0" ;; + --tal_reg) run_talairach_registration="true" ;; + --device) device="$1" ; shift ;; + --batch) batch_size="$1" ; shift ;; + --viewagg_device|--run_viewagg_on) if [[ "$key" == "--run_viewagg_on" ]] then echo "WARNING: --run_viewagg_on (cpu|gpu|check) is deprecated and will be removed, use --viewagg_device ." fi - case "$2" in + case "$1" in check) - echo "WARNING: the option \"check\" is deprecated for --viewagg_device , use \"auto\"." - viewagg="auto" - ;; - gpu) - viewagg="cuda" - ;; - *) - viewagg="$2" - ;; + echo "WARNING: the option \"check\" is deprecated for --viewagg_device , use \"auto\"." + viewagg="auto" + ;; + gpu) viewagg="cuda" ;; + *) viewagg="$1" ;; esac - shift # past argument shift # past value ;; - --no_cuda) + --no_cuda) echo "WARNING: --no_cuda is deprecated and will be removed, use --device cpu." device="cpu" - shift # past argument ;; - --no_biasfield) - run_biasfield="0" - shift # past argument - ;; - --no_asegdkt | --no_aparc) + + # asegdkt module options + #============================================================= + --no_asegdkt|--no_aparc) if [[ "$key" == "--no_aparc" ]] then echo "WARNING: --no_aparc is deprecated and will be removed, use --no_asegdkt." fi run_asegdkt_module="0" - shift # past argument - ;; - --no_cereb) - run_cereb_module="0" - shift # past argument - ;; - --no_hypothal) - run_hypvinn_module="0" - shift # past argument - ;; - --tal_reg) - run_talairach_registration="true" - shift - ;; - --device) - device=$2 - shift # past argument - shift # past value ;; - --batch) - batch_size="$2" - shift # past argument - shift # past value - ;; - --seg_only) - run_surf_pipeline="0" - shift # past argument - ;; - --surf_only) - run_seg_pipeline="0" - shift # past argument - ;; - --fstess) - surf_flags=("${surf_flags[@]}" "--fstess") - shift # past argument - ;; - --fsqsphere) - surf_flags=("${surf_flags[@]}" "--fsqsphere") - shift # past argument - ;; - --fsaparc) - surf_flags=("${surf_flags[@]}" "--fsaparc") - shift # past argument - ;; - --no_surfreg) - surf_flags=("${surf_flags[@]}" "--no_surfreg") - shift # past argument - ;; - --vox_size) - vox_size="$2" - shift # past argument - shift # past value - ;; - --3t) - surf_flags=("${surf_flags[@]}" "--3T") - atlas3T="true" - shift - ;; - --parallel) - surf_flags=("${surf_flags[@]}" "--parallel") - shift # past argument - ;; - --threads) - threads="$2" - shift # past argument - shift # past value - ;; - --py) - python="$2" - shift # past argument + --asegdkt_statsfile) asegdkt_statsfile="$1" ; shift ;; + --aseg_segfile) aseg_segfile="$1" ; shift ;; + --mask_name) mask_name="$1" ; shift ;; + --merged_segfile) merged_segfile="$1" ; shift ;; + + # cereb module options + #============================================================= + --no_cereb) run_cereb_module="0" ;; + # several options that set a variable + --cereb_segfile) cereb_segfile="$1" ; shift ;; + --cereb_statsfile) cereb_statsfile="$1" ; shift ;; + + # hypothal module options + #============================================================= + --no_hypothal) run_hypvinn_module="0" ;; + # several options that set a variable + --hypo_segfile) hypo_segfile="$1" ; shift ;; + --hypo_statsfile) hypo_statsfile="$1" ; shift ;; + --reg_mode) + mode=$(echo "$1" | tr "[:upper:]" "[:lower:]") + if [[ "$mode" =~ ^(none|coreg|robust)$ ]] ; then + hypvinn_flags+=(--regmode "$mode") + else + echo "Invalid --reg_mode option, must be 'none', 'coreg' or 'robust'." + exit 1 + fi shift # past value ;; - --ignore_fs_version) - # Dev flag - surf_flags=("${surf_flags[@]}" "--ignore_fs_version") - shift # past argument - ;; - --no_fs_t1 ) - # Dev flag - surf_flags=("${surf_flags[@]}" "--no_fs_T1") - shift # past argument - ;; - --allow_root) - allow_root=("--allow_root") - shift # past argument - ;; - -h|--help) - usage - exit + # several options that set a variable + --qc_snap) hypvinn_flags+=(--qc_snap) ;; + + ############################################################## + # surf-pipeline options + ############################################################## + --seg_only) run_surf_pipeline="0" ;; + # several flag options that are *just* passed through to recon-surf.sh + --fstess|--fsqsphere|--fsaparc|--no_surfreg|--parallel|--ignore_fs_version) + surf_flags=("${surf_flags[@]}" "$key") ;; - --version) - if [[ "$#" -lt 2 ]]; then - version_and_quit="1" - else - case "$2" in - all) - version_and_quit="+checkpoints+git+pip" - shift - ;; - +*) - version_and_quit="$2" - shift - ;; - --*) - version_and_quit="1" - ;; - *) - echo "Invalid option for --version: '$2', must be 'all' or [+checkpoints][+git][+pip]" - exit 1 - ;; - esac - fi - shift + --no_fs_t1) surf_flags=("${surf_flags[@]}" "--no_fs_T1") ;; + + # temporary segstats development flag + --segstats_legacy) + surf_flags=("${surf_flags[@]}" "$key") ;; - *) # unknown option - if [[ "$1" == "" ]] - then - # skip empty arguments - shift - else - echo "ERROR: Flag '$1' unrecognized." - exit 1 - fi + *) # unknown option + # if not empty arguments, error & exit + if [[ "$key" != "" ]] ; then echo "ERROR: Flag '$key' unrecognized." ; exit 1 ; fi ;; esac done @@ -600,45 +464,45 @@ fi ########################################## VERSION AND QUIT HERE ######################################## version_args=() if [[ -f "$FASTSURFER_HOME/BUILD.info" ]] - then - version_args=(--build_cache "$FASTSURFER_HOME/BUILD.info" --prefer_cache) +then + version_args=(--build_cache "$FASTSURFER_HOME/BUILD.info" --prefer_cache) fi if [[ -n "$version_and_quit" ]] +then + # if version_and_quit is 1, it should only print the version number+git branch + if [[ "$version_and_quit" != "1" ]] then - # if version_and_quit is 1, it should only print the version number+git branch - if [[ "$version_and_quit" != "1" ]] - then - version_args=("${version_args[@]}" --sections "$version_and_quit") - fi - $python "$FASTSURFER_HOME/FastSurferCNN/version.py" "${version_args[@]}" - exit + version_args=("${version_args[@]}" --sections "$version_and_quit") + fi + $python "$FASTSURFER_HOME/FastSurferCNN/version.py" "${version_args[@]}" + exit fi # make sure the python executable is valid and found if [[ -z "$(which "${python/ */}")" ]]; then - echo "Cannot find the python interpreter ${python/ */}." - exit 1 + echo "Cannot find the python interpreter ${python/ */}." + exit 1 fi # Warning if run as root user if [[ "${#allow_root}" == 0 ]] && [[ "$(id -u)" == "0" ]] - then - echo "You are trying to run '$0' as root. We advice to avoid running FastSurfer as root, " - echo "because it will lead to files and folders created as root." - echo "If you are running FastSurfer in a docker container, you can specify the user with " - echo "'-u \$(id -u):\$(id -g)' (see https://docs.docker.com/engine/reference/run/#user)." - echo "If you want to force running as root, you may pass --allow_root to run_fastsurfer.sh." - exit 1; +then + echo "You are trying to run '$0' as root. We advice to avoid running FastSurfer as root, " + echo "because it will lead to files and folders created as root." + echo "If you are running FastSurfer in a docker container, you can specify the user with " + echo "'-u \$(id -u):\$(id -g)' (see https://docs.docker.com/engine/reference/run/#user)." + echo "If you want to force running as root, you may pass --allow_root to run_fastsurfer.sh." + exit 1; fi # CHECKS if [[ "$run_seg_pipeline" == "1" ]] && { [[ -z "$t1" ]] || [[ ! -f "$t1" ]]; } - then - echo "ERROR: T1 image ($t1) could not be found. Must supply an existing T1 input (full head) via " - echo "--t1 (absolute path and name) for generating the segmentation." - echo "NOTES: If running in a container, make sure symlinks are valid!" - exit 1; +then + echo "ERROR: T1 image ($t1) could not be found. Must supply an existing T1 input (full head) via " + echo "--t1 (absolute path and name) for generating the segmentation." + echo "NOTES: If running in a container, make sure symlinks are valid!" + exit 1; fi if [[ -z "${sd}" ]] @@ -659,59 +523,59 @@ then fi if [[ -z "$subject" ]] - then - echo "ERROR: must supply subject name via --sid" - exit 1; +then + echo "ERROR: must supply subject name via --sid" + exit 1; fi if [[ -z "$merged_segfile" ]] - then - merged_segfile="${sd}/${subject}/mri/fastsurfer.merged.mgz" +then + merged_segfile="${sd}/${subject}/mri/fastsurfer.merged.mgz" fi if [[ -z "$asegdkt_segfile" ]] - then - asegdkt_segfile="${sd}/${subject}/mri/aparc.DKTatlas+aseg.deep.mgz" +then + asegdkt_segfile="${sd}/${subject}/mri/aparc.DKTatlas+aseg.deep.mgz" fi if [[ -z "$aseg_segfile" ]] - then - aseg_segfile="${sd}/${subject}/mri/aseg.auto_noCCseg.mgz" +then + aseg_segfile="${sd}/${subject}/mri/aseg.auto_noCCseg.mgz" fi if [[ -z "$asegdkt_statsfile" ]] - then - asegdkt_statsfile="${sd}/${subject}/stats/aseg+DKT.stats" +then + asegdkt_statsfile="${sd}/${subject}/stats/aseg+DKT.stats" fi if [[ -z "$cereb_segfile" ]] - then - cereb_segfile="${sd}/${subject}/mri/cerebellum.CerebNet.nii.gz" +then + cereb_segfile="${sd}/${subject}/mri/cerebellum.CerebNet.nii.gz" fi if [[ -z "$cereb_statsfile" ]] - then - cereb_statsfile="${sd}/${subject}/stats/cerebellum.CerebNet.stats" +then + cereb_statsfile="${sd}/${subject}/stats/cerebellum.CerebNet.stats" fi if [[ -z "$hypo_segfile" ]] - then - hypo_segfile="${sd}/${subject}/mri/hypothalamus.HypVINN.nii.gz" +then + hypo_segfile="${sd}/${subject}/mri/hypothalamus.HypVINN.nii.gz" fi if [[ -z "$hypo_statsfile" ]] - then - hypo_statsfile="${sd}/${subject}/stats/hypothalamus.HypVINN.stats" +then + hypo_statsfile="${sd}/${subject}/stats/hypothalamus.HypVINN.stats" fi if [[ -z "$mask_name" ]] - then - mask_name="${sd}/${subject}/mri/mask.mgz" +then + mask_name="${sd}/${subject}/mri/mask.mgz" fi if [[ -z "$conformed_name" ]] - then - conformed_name="${sd}/${subject}/mri/orig.mgz" +then + conformed_name="${sd}/${subject}/mri/orig.mgz" fi if [[ -z "$conformed_name_t2" ]] @@ -720,33 +584,33 @@ if [[ -z "$conformed_name_t2" ]] fi if [[ -z "$norm_name" ]] - then - norm_name="${sd}/${subject}/mri/orig_nu.mgz" +then + norm_name="${sd}/${subject}/mri/orig_nu.mgz" fi if [[ -z "$norm_name_t2" ]] - then - norm_name_t2="${sd}/${subject}/mri/T2_nu.mgz" +then + norm_name_t2="${sd}/${subject}/mri/T2_nu.mgz" fi if [[ -z "$seg_log" ]] - then - seg_log="${sd}/${subject}/scripts/deep-seg.log" +then + seg_log="${sd}/${subject}/scripts/deep-seg.log" fi if [[ -z "$build_log" ]] - then - build_log="${sd}/${subject}/scripts/build.log" +then + build_log="${sd}/${subject}/scripts/build.log" fi if [[ -n "$t2" ]] - then - if [[ ! -f "$t2" ]] - then - echo "ERROR: T2 file $t2 does not exist!" - exit 1; - fi - copy_name_T2="${sd}/${subject}/mri/orig/T2.001.mgz" +then + if [[ ! -f "$t2" ]] + then + echo "ERROR: T2 file $t2 does not exist!" + exit 1; + fi + copy_name_T2="${sd}/${subject}/mri/orig/T2.001.mgz" fi if [[ -z "$PYTHONUNBUFFERED" ]] @@ -783,55 +647,71 @@ fi #fi if [[ "${asegdkt_segfile: -3}" != "${conformed_name: -3}" ]] - then - echo "ERROR: Specified segmentation output and conformed image output do not have same file type." - echo "You passed --asegdkt_segfile ${asegdkt_segfile} and --conformed_name ${conformed_name}." - echo "Make sure these have the same file-format and adjust the names passed to the flags accordingly!" - exit 1; +then + echo "ERROR: Specified segmentation output and conformed image output do not have same file type." + echo "You passed --asegdkt_segfile ${asegdkt_segfile} and --conformed_name ${conformed_name}." + echo "Make sure these have the same file-format and adjust the names passed to the flags accordingly!" + exit 1; fi if [[ "$run_surf_pipeline" == "1" ]] && { [[ "$run_asegdkt_module" == "0" ]] || [[ "$run_seg_pipeline" == "0" ]]; } +then + if [[ ! -f "$asegdkt_segfile" ]] then - if [[ ! -f "$asegdkt_segfile" ]] - then - echo "ERROR: To run the surface pipeline, a whole brain segmentation must already exist." - echo "You passed --surf_only or --no_asegdkt, but the whole-brain segmentation ($asegdkt_segfile) could not be found." - echo "If the segmentation is not saved in the default location ($asegdkt_segfile_default), specify the absolute path and name via --asegdkt_segfile" - exit 1; - fi - if [[ ! -f "$conformed_name" ]] - then - echo "ERROR: To run the surface pipeline only, a conformed T1 image must already exist." - echo "You passed --surf_only but the conformed image ($conformed_name) could not be found." - echo "If the conformed image is not saved in the default location (\$SUBJECTS_DIR/\$SID/mri/orig.mgz)," - echo "specify the absolute path and name via --conformed_name." - exit 1; - fi + echo "ERROR: To run the surface pipeline, a whole brain segmentation must already exist." + echo "You passed --surf_only or --no_asegdkt, but the whole-brain segmentation ($asegdkt_segfile) could not be found." + echo "If the segmentation is not saved in the default location ($asegdkt_segfile_default), specify the absolute path and name via --asegdkt_segfile" + exit 1; + fi + if [[ ! -f "$conformed_name" ]] + then + echo "ERROR: To run the surface pipeline only, a conformed T1 image must already exist." + echo "You passed --surf_only but the conformed image ($conformed_name) could not be found." + echo "If the conformed image is not saved in the default location (\$SUBJECTS_DIR/\$SID/mri/orig.mgz)," + echo "specify the absolute path and name via --conformed_name." + exit 1; + fi fi if [[ "$run_seg_pipeline" == "1" ]] && { [[ "$run_asegdkt_module" == "0" ]] && [[ "$run_cereb_module" == "1" ]]; } +then + if [[ ! -f "$asegdkt_segfile" ]] then - if [[ ! -f "$asegdkt_segfile" ]] - then - echo "ERROR: To run the cerebellum segmentation but no asegdkt, the aseg segmentation must already exist." - echo "You passed --no_asegdkt but the asegdkt segmentation ($asegdkt_segfile) could not be found." - echo "If the segmentation is not saved in the default location ($asegdkt_segfile_default), specify the absolute path and name via --asegdkt_segfile" - exit 1; - fi + echo "ERROR: To run the cerebellum segmentation but no asegdkt, the aseg segmentation must already exist." + echo "You passed --no_asegdkt but the asegdkt segmentation ($asegdkt_segfile) could not be found." + echo "If the segmentation is not saved in the default location ($asegdkt_segfile_default), specify the absolute path and name via --asegdkt_segfile" + exit 1; + fi fi if [[ "$run_surf_pipeline" == "0" ]] && [[ "$run_seg_pipeline" == "0" ]] +then + echo "ERROR: You specified both --surf_only and --seg_only. Therefore neither part of the pipeline will be run." + echo "To run the whole FastSurfer pipeline, omit both flags." + exit 1; +fi + +if [[ "$run_surf_pipeline" == "1" ]] || [[ "$run_talairach_registration" == "true" ]] +then + msg="The surface pipeline and the talairach-registration in the segmentation pipeline require a FreeSurfer License" + if [[ -z "$FS_LICENSE" ]] then - echo "ERROR: You specified both --surf_only and --seg_only. Therefore neither part of the pipeline will be run." - echo "To run the whole FastSurfer pipeline, omit both flags." + echo "ERROR: $msg, but no license was provided via --fs_license or the FS_LICENSE environment variable." exit 1; + elif [[ ! -f "$FS_LICENSE" ]] + then + echo "ERROR: $msg, but the provided path is not a file: $FS_LICENSE." + exit 1; + fi fi ########################################## START ######################################################## mkdir -p "$(dirname "$seg_log")" +source "${reconsurfdir}/functions.sh" + if [[ -f "$seg_log" ]]; then log_existed="true" else log_existed="false" fi @@ -847,178 +727,206 @@ printf "%s %s\n%s\n" "$THIS_SCRIPT" "${inputargs[*]}" "$(date -R)" >> "$build_lo $python "$FASTSURFER_HOME/FastSurferCNN/version.py" "${version_args[@]}" >> "$build_log" & if [[ "$run_seg_pipeline" != "1" ]] - then - echo "Running run_fastsurfer.sh without segmentation ; expecting previous --seg_only run in ${sd}/${subject}" | tee -a "$seg_log" +then + echo "Running run_fastsurfer.sh without segmentation ; expecting previous --seg_only run in ${sd}/${subject}" | tee -a "$seg_log" fi if [[ "$run_seg_pipeline" == "1" ]] +then + # "============= Running FastSurferCNN (Creating Segmentation aparc.DKTatlas.aseg.mgz) ===============" + # use FastSurferCNN to create cortical parcellation + anatomical segmentation into 95 classes. + echo "Log file for segmentation FastSurferCNN/run_prediction.py" >> "$seg_log" + { date 2>&1 ; echo "" ; } | tee -a "$seg_log" + + if [[ "$run_asegdkt_module" == "1" ]] then - # "============= Running FastSurferCNN (Creating Segmentation aparc.DKTatlas.aseg.mgz) ===============" - # use FastSurferCNN to create cortical parcellation + anatomical segmentation into 95 classes. - echo "Log file for segmentation FastSurferCNN/run_prediction.py" >> "$seg_log" - date 2>&1 | tee -a "$seg_log" - echo "" | tee -a "$seg_log" + cmd=($python "$fastsurfercnndir/run_prediction.py" --t1 "$t1" + --asegdkt_segfile "$asegdkt_segfile" --conformed_name "$conformed_name" + --brainmask_name "$mask_name" --aseg_name "$aseg_segfile" --sid "$subject" + --seg_log "$seg_log" --vox_size "$vox_size" --batch_size "$batch_size" + --viewagg_device "$viewagg" --device "$device" "${allow_root[@]}") + # specify the subject dir $sd, if asegdkt_segfile explicitly starts with it + if [[ "$sd" == "${asegdkt_segfile:0:${#sd}}" ]]; then cmd=("${cmd[@]}" --sd "$sd"); fi + echo_quoted "${cmd[@]}" | tee -a "$seg_log" + "${cmd[@]}" + exit_code="${PIPESTATUS[0]}" + if [[ "${exit_code}" == 2 ]] + then + echo "ERROR: FastSurfer asegdkt segmentation failed QC checks." | tee -a "$seg_log" + exit 1 + elif [[ "${exit_code}" -ne 0 ]] + then + echo "ERROR: FastSurfer asegdkt segmentation failed." | tee -a "$seg_log" + exit 1 + fi + fi + if [[ -n "$t2" ]] + then + { + echo "INFO: Copying T2 file to ${copy_name_T2}..." + cmd=("nib-convert" "$t2" "$copy_name_T2") + echo_quoted "${cmd[@]}" + "${cmd[@]}" 2>&1 + + echo "INFO: Robust scaling (partial conforming) of T2 image..." + cmd=($python "${fastsurfercnndir}/data_loader/conform.py" --no_strict_lia + --no_vox_size --no_img_size "$t2" "$conformed_name_t2") + echo_quoted "${cmd[@]}" + "${cmd[@]}" 2>&1 + echo "Done." + } | tee -a "$seg_log" + fi - if [[ "$run_asegdkt_module" == "1" ]] - then - cmd=($python "$fastsurfercnndir/run_prediction.py" --t1 "$t1" - --asegdkt_segfile "$asegdkt_segfile" --conformed_name "$conformed_name" - --brainmask_name "$mask_name" --aseg_name "$aseg_segfile" --sid "$subject" - --seg_log "$seg_log" --vox_size "$vox_size" --batch_size "$batch_size" - --viewagg_device "$viewagg" --device "$device" "${allow_root[@]}") - # specify the subject dir $sd, if asegdkt_segfile explicitly starts with it - if [[ "$sd" == "${asegdkt_segfile:0:${#sd}}" ]]; then cmd=("${cmd[@]}" --sd "$sd"); fi - echo "${cmd[@]}" | tee -a "$seg_log" - "${cmd[@]}" - exit_code="${PIPESTATUS[0]}" - if [[ "${exit_code}" == 2 ]] - then - echo "ERROR: FastSurfer asegdkt segmentation failed QC checks." - exit 1 - elif [[ "${exit_code}" -ne 0 ]] - then - echo "ERROR: FastSurfer asegdkt segmentation failed." - exit 1 - fi + if [[ "$run_biasfield" == "1" ]] + then + { + # this will always run, since norm_name is set to subject_dir/mri/orig_nu.mgz, if it is not passed/empty + cmd=($python "${reconsurfdir}/N4_bias_correct.py" "--in" "$conformed_name" + --rescale "$norm_name" --aseg "$asegdkt_segfile" --threads "$threads") + echo "INFO: Running N4 bias-field correction" + echo_quoted "${cmd[@]}" + "${cmd[@]}" 2>&1 + } | tee -a "$seg_log" + if [[ "${PIPESTATUS[0]}" -ne 0 ]] + then + echo "ERROR: Biasfield correction failed" | tee -a "$seg_log" + exit 1 fi - if [[ -n "$t2" ]] + + if [[ "$run_talairach_registration" == "true" ]] + then + cmd=("$reconsurfdir/talairach-reg.sh" "$sd/$subject/mri" "$atlas3T" "$seg_log") + { + echo "INFO: Running talairach registration" + echo_quoted "${cmd[@]}" + } | tee -a "$seg_log" + "${cmd[@]}" + if [[ "${PIPESTATUS[0]}" -ne 0 ]] then - printf "INFO: Copying T2 file to %s..." "${copy_name_T2}" | tee -a "$seg_log" - cmd=("nib-convert" "$t2" "$copy_name_T2") - "${cmd[@]}" 2>&1 | tee -a "$seg_log" - - echo "INFO: Robust scaling (partial conforming) of T2 image..." | tee -a "$seg_log" - cmd=($python "${fastsurfercnndir}/data_loader/conform.py" --no_strict_lia - --no_vox_size --no_img_size "$t2" "$conformed_name_t2") - "${cmd[@]}" 2>&1 | tee -a "$seg_log" - echo "Done." | tee -a "$seg_log" + echo "ERROR: talairach registration failed" | tee -a "$seg_log" + exit 1 + fi fi - if [[ "$run_biasfield" == "1" ]] + if [[ "$run_asegdkt_module" ]] + then + cmd=($python "${fastsurfercnndir}/segstats.py" --segfile "$asegdkt_segfile" + --segstatsfile "$asegdkt_statsfile" --normfile "$norm_name" + --threads "$threads" "${allow_root[@]}" --empty --excludeid 0 + --sd "${sd}" --sid "${subject}" + --ids 2 4 5 7 8 10 11 12 13 14 15 16 17 18 24 26 28 31 41 43 44 46 47 + 49 50 51 52 53 54 58 60 63 77 251 252 253 254 255 1002 1003 1005 + 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 + 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 + 1034 1035 2002 2003 2005 2006 2007 2008 2009 2010 2011 2012 2013 + 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 + 2027 2028 2029 2030 2031 2034 2035 + --lut "$fastsurfercnndir/config/FreeSurferColorLUT.txt" + measures --compute "Mask($mask_name)" "BrainSeg" "BrainSegNotVent" + "SupraTentorial" "SupraTentorialNotVent" + "SubCortGray" "rhCerebralWhiteMatter" + "lhCerebralWhiteMatter" "CerebralWhiteMatter" + # make sure to read white matter hypointensities from the + ) + if [[ "$run_talairach_registration" == "true" ]] then - # this will always run, since norm_name is set to subject_dir/mri/orig_nu.mgz, if it is not passed/empty - echo "INFO: Running N4 bias-field correction" | tee -a "$seg_log" - cmd=($python "${reconsurfdir}/N4_bias_correct.py" "--in" "$conformed_name" - --rescale "$norm_name" --aseg "$asegdkt_segfile" --threads "$threads") - echo "${cmd[@]}" | tee -a "$seg_log" - "${cmd[@]}" 2>&1 | tee -a "$seg_log" - if [[ "${PIPESTATUS[0]}" -ne 0 ]] - then - echo "ERROR: Biasfield correction failed" | tee -a "$seg_log" - exit 1 - fi - - if [[ "$run_talairach_registration" == "true" ]] - then - echo "INFO: Running talairach registration" | tee -a "$seg_log" - cmd=("$reconsurfdir/talairach-reg.sh" "$sd/$subject/mri" "$atlas3T" "$seg_log") - echo "${cmd[@]}" | tee -a "$seg_log" - "${cmd[@]}" - if [[ "${PIPESTATUS[0]}" -ne 0 ]] - then - echo "ERROR: talairach registration failed" | tee -a "$seg_log" - exit 1 - fi - fi - - if [[ "$run_asegdkt_module" ]] - then - cmd=($python "${fastsurfercnndir}/segstats.py" --segfile "$asegdkt_segfile" - --segstatsfile "$asegdkt_statsfile" --normfile "$norm_name" - --threads "$threads" "${allow_root[@]}" --empty --excludeid 0 - --ids 2 4 5 7 8 10 11 12 13 14 15 16 17 18 24 26 28 31 41 43 44 46 47 - 49 50 51 52 53 54 58 60 63 77 251 252 253 254 255 1002 1003 1005 - 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 - 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 - 1034 1035 2002 2003 2005 2006 2007 2008 2009 2010 2011 2012 2013 - 2014 2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 - 2027 2028 2029 2030 2031 2034 2035 - --lut "$fastsurfercnndir/config/FreeSurferColorLUT.txt") - echo "${cmd[@]}" | tee -a "$seg_log" - "${cmd[@]}" 2>&1 | tee -a "$seg_log" - if [[ "${PIPESTATUS[0]}" -ne 0 ]] - then - echo "ERROR: asegdkt statsfile generation failed" | tee -a "$seg_log" - exit 1 - fi - fi - - if [[ -n "$t2" ]] - then - # ... we have a t2 image, bias field-correct it - # (use the T2 image, not the conformed save and robustly scaled uchar) - echo "INFO: Running N4 bias-field correction of the t2" | tee -a "$seg_log" - cmd=($python "${reconsurfdir}/N4_bias_correct.py" "--in" "$copy_name_T2" - --out "$norm_name_t2" --threads "$threads" --uchar) - echo "${cmd[@]}" | tee -a "$seg_log" - "${cmd[@]}" - if [[ "${PIPESTATUS[0]}" -ne 0 ]] - then - echo "ERROR: T2 Biasfield correction failed" | tee -a "$seg_log" - exit 1 - fi - fi - else - if [[ -n "$t2" ]] + cmd=("${cmd[@]}" "EstimatedTotalIntraCranialVol" + "BrainSegVol-to-eTIV" "MaskVol-to-eTIV") + fi + { + echo_quoted "${cmd[@]}" + "${cmd[@]}" 2>&1 + } | tee -a "$seg_log" + if [[ "${PIPESTATUS[0]}" -ne 0 ]] then - # no biasfield, but a t2 is passed; presumably, this is biasfield corrected - echo "INFO: Linking $norm_name_t2 to $t2, which is assumed to already be biasfield corrected." | tee -a "$seg_log" - source "${reconsurfdir}/functions.sh" - softlink_or_copy "$norm_name_t2" "$t2" "$seg_log" + echo "ERROR: asegdkt statsfile generation failed" | tee -a "$seg_log" + exit 1 fi fi + fi # [[ "$run_biasfield" == "1" ]] - if [[ "$run_cereb_module" == "1" ]] + if [[ -n "$t2" ]] + then + if [[ "$run_biasfield" == "1" ]] + then + # ... we have a t2 image, bias field-correct it (save robustly scaled uchar) + cmd=($python "${reconsurfdir}/N4_bias_correct.py" "--in" "$copy_name_T2" + --out "$norm_name_t2" --threads "$threads" --uchar) + { + echo "INFO: Running N4 bias-field correction of the t2" + echo_quoted "${cmd[@]}" + } | tee -a "$seg_log" + "${cmd[@]}" 2>&1 | tee -a "$seg_log" + if [[ "${PIPESTATUS[0]}" -ne 0 ]] then - if [[ "$run_biasfield" == "1" ]] - then - cereb_flags=("${cereb_flags[@]}" --norm_name "$norm_name" - --cereb_statsfile "$cereb_statsfile") - else - echo "INFO: Running CerebNet without generating a statsfile, since biasfield correction deactivated '--no_biasfield'." | tee -a $seg_log - fi - - cmd=($python "$cerebnetdir/run_prediction.py" --t1 "$t1" - --asegdkt_segfile "$asegdkt_segfile" --conformed_name "$conformed_name" - --cereb_segfile "$cereb_segfile" --seg_log "$seg_log" --async_io - --batch_size "$batch_size" --viewagg_device "$viewagg" --device "$device" - --threads "$threads" "${cereb_flags[@]}" "${allow_root[@]}") - # specify the subject dir $sd, if asegdkt_segfile explicitly starts with it - if [[ "$sd" == "${cereb_segfile:0:${#sd}}" ]] ; then cmd=("${cmd[@]}" --sd "$sd"); fi - echo "${cmd[@]}" | tee -a "$seg_log" - "${cmd[@]}" - if [[ "${PIPESTATUS[0]}" -ne 0 ]] - then - echo "ERROR: Cerebellum Segmentation failed" | tee -a "$seg_log" - exit 1 - fi + echo "ERROR: T2 Biasfield correction failed" | tee -a "$seg_log" + exit 1 + fi + else + # no biasfield, but a t2 is passed; presumably, this is biasfield corrected + cmd=($python "${fastsurfercnndir}/data_loader/conform.py" --no_strict_lia + --no_iso_vox --no_img_size "$t2" "$norm_name_t2") + { + echo "INFO: Robustly rescaling $t2 to uchar ($norm_name_t2), which is assumed to already be biasfield corrected." + echo "WARNING: --no_biasfield is activated, but FastSurfer does not check, if " + echo " passed T2 image is properly scaled and typed. T2 needs to be uchar and" + echo " robustly scaled (see FastSurferCNN/utils/data_loader/conform.py)!" + } | tee -a "$seg_log" + "${cmd[@]}" 2>&1 | tee -a "$seg_log" fi + fi - if [[ "$run_hypvinn_module" == "1" ]] - then + if [[ "$run_cereb_module" == "1" ]] + then + if [[ "$run_biasfield" == "1" ]] + then + cereb_flags=("${cereb_flags[@]}" --norm_name "$norm_name" + --cereb_statsfile "$cereb_statsfile") + else + echo "INFO: Running CerebNet without generating a statsfile, since biasfield correction deactivated '--no_biasfield'." | tee -a "$seg_log" + fi + + cmd=($python "$cerebnetdir/run_prediction.py" --t1 "$t1" + --asegdkt_segfile "$asegdkt_segfile" --conformed_name "$conformed_name" + --cereb_segfile "$cereb_segfile" --seg_log "$seg_log" --async_io + --batch_size "$batch_size" --viewagg_device "$viewagg" --device "$device" + --threads "$threads" "${cereb_flags[@]}" "${allow_root[@]}") + # specify the subject dir $sd, if asegdkt_segfile explicitly starts with it + if [[ "$sd" == "${cereb_segfile:0:${#sd}}" ]] ; then cmd=("${cmd[@]}" --sd "$sd"); fi + echo_quoted "${cmd[@]}" | tee -a "$seg_log" + "${cmd[@]}" # no tee, directly logging to $seg_log + if [[ "${PIPESTATUS[0]}" -ne 0 ]] + then + echo "ERROR: Cerebellum Segmentation failed" | tee -a "$seg_log" + exit 1 + fi + fi + + if [[ "$run_hypvinn_module" == "1" ]] + then # currently, the order of the T2 preprocessing only is registration to T1w - cmd=($python "$hypvinndir/run_prediction.py" --sd "${sd}" --sid "${subject}" - "${hypvinn_flags[@]}" "${allow_root[@]}" --threads "$threads" --async_io - --batch_size "$batch_size" --seg_log "$seg_log" --device "$device" - --viewagg_device "$viewagg" --t1) - if [[ "$run_biasfield" == "1" ]] - then - cmd+=("$norm_name") - if [[ -n "$t2" ]] ; then cmd+=(--t2 "$norm_name_t2"); fi - else - echo "WARNING: We strongly recommend to *not* exclude the biasfield (--no_biasfield) with the hypothal module!" - cmd+=("$t1") - if [[ -n "$t2" ]] ; then cmd+=(--t2 "$t2"); fi - fi - echo "${cmd[@]}" | tee -a "$seg_log" - "${cmd[@]}" - if [[ "${PIPESTATUS[0]}" -ne 0 ]] - then - echo "ERROR: Hypothalamus Segmentation failed" | tee -a "$seg_log" - exit 1 - fi + cmd=($python "$hypvinndir/run_prediction.py" --sd "${sd}" --sid "${subject}" + "${hypvinn_flags[@]}" "${allow_root[@]}" --threads "$threads" --async_io + --batch_size "$batch_size" --seg_log "$seg_log" --device "$device" + --viewagg_device "$viewagg" --t1) + if [[ "$run_biasfield" == "1" ]] + then + cmd+=("$norm_name") + if [[ -n "$t2" ]] ; then cmd+=(--t2 "$norm_name_t2"); fi + else + echo "WARNING: We strongly recommend to *not* exclude the biasfield (--no_biasfield) with the hypothal module!" + cmd+=("$t1") + if [[ -n "$t2" ]] ; then cmd+=(--t2 "$t2"); fi + fi + echo_quoted "${cmd[@]}" | tee -a "$seg_log" + "${cmd[@]}" + if [[ "${PIPESTATUS[0]}" -ne 0 ]] + then + echo "ERROR: Hypothalamus Segmentation failed" | tee -a "$seg_log" + exit 1 fi + fi # if [[ ! -f "$merged_segfile" ]] # then @@ -1027,17 +935,17 @@ if [[ "$run_seg_pipeline" == "1" ]] fi if [[ "$run_surf_pipeline" == "1" ]] - then - # ============= Running recon-surf (surfaces, thickness etc.) =============== - # use recon-surf to create surface models based on the FastSurferCNN segmentation. - pushd "$reconsurfdir" || exit 1 - cmd=("./recon-surf.sh" --sid "$subject" --sd "$sd" --t1 "$conformed_name" - --asegdkt_segfile "$asegdkt_segfile" --threads "$threads" --py "$python" - "${surf_flags[@]}" "${allow_root[@]}") - echo "${cmd[@]}" | tee -a "$seg_log" - "${cmd[@]}" - if [[ "${PIPESTATUS[0]}" -ne 0 ]] ; then exit 1 ; fi - popd || return +then + # ============= Running recon-surf (surfaces, thickness etc.) =============== + # use recon-surf to create surface models based on the FastSurferCNN segmentation. + pushd "$reconsurfdir" > /dev/null || exit 1 + cmd=("./recon-surf.sh" --sid "$subject" --sd "$sd" --t1 "$conformed_name" + --asegdkt_segfile "$asegdkt_segfile" --threads "$threads" --py "$python" + "${surf_flags[@]}" "${allow_root[@]}") + echo_quoted "${cmd[@]}" | tee -a "$seg_log" + "${cmd[@]}" + if [[ "${PIPESTATUS[0]}" -ne 0 ]] ; then exit 1 ; fi + popd > /dev/null || return fi ########################################## End ########################################################