From 23eed72933dfb42db69664cb857ea846a1298c77 Mon Sep 17 00:00:00 2001 From: gferraro Date: Tue, 1 Oct 2024 16:29:38 +0200 Subject: [PATCH] and support for therml norm diff --- src/ml_tools/hyperparams.py | 6 +++ src/ml_tools/interpreter.py | 96 ++++++++++++++++++++++++++----------- src/ml_tools/preprocess.py | 13 +++-- 3 files changed, 85 insertions(+), 30 deletions(-) diff --git a/src/ml_tools/hyperparams.py b/src/ml_tools/hyperparams.py index b4b57055..90bcc362 100644 --- a/src/ml_tools/hyperparams.py +++ b/src/ml_tools/hyperparams.py @@ -27,6 +27,8 @@ def insert_defaults(self): self["segment_type"] = self.segment_type self["multi_label"] = True self["diff_norm"] = self.diff_norm + self["thermal_diff_norm"] = self.thermal_diff_norm + self["smooth_predictions"] = self.smooth_predictions self["channels"] = self.channels @@ -58,6 +60,10 @@ def excluded_labels(self): def remapped_labels(self): return self.get("remapped_labels", None) + @property + def thermal_diff_norm(self): + return self.get("thermal_diff_norm", False) + @property def diff_norm(self): return self.get("diff_norm", True) diff --git a/src/ml_tools/interpreter.py b/src/ml_tools/interpreter.py index 2763957b..e4c0a99e 100644 --- a/src/ml_tools/interpreter.py +++ b/src/ml_tools/interpreter.py @@ -183,9 +183,12 @@ def preprocess_frames( data = [] frames_used = [] filtered_norm_limits = None - if self.params.diff_norm: + thermal_norm_limits = None + if self.params.diff_norm or self.params.thermal_diff_norm: min_diff = None max_diff = 0 + thermal_max_diff = None + thermal_min_diff = None for i, region in enumerate(reversed(track.bounds_history)): if region.blank: continue @@ -201,16 +204,30 @@ def preprocess_frames( continue f.float_arrays() - diff_frame = region.subimage(f.thermal) - region.subimage( - clip.background - ) - new_max = np.amax(diff_frame) - new_min = np.amin(diff_frame) - if min_diff is None or new_min < min_diff: - min_diff = new_min - if new_max > max_diff: - max_diff = new_max - filtered_norm_limits = (min_diff, max_diff) + + if self.params.thermal_diff_norm: + diff_frame = f.thermal - np.median(f.thermal) + new_max = np.amax(diff_frame) + new_min = np.amin(diff_frame) + if thermal_min_diff is None or new_min < thermal_min_diff: + thermal_min_diff = new_min + if thermal_max_diff is None or new_max > thermal_max_diff: + thermal_max_diff = new_max + if self.params.diff_norm: + diff_frame = region.subimage(f.thermal) - region.subimage( + clip.background + ) + new_max = np.amax(diff_frame) + new_min = np.amin(diff_frame) + if min_diff is None or new_min < min_diff: + min_diff = new_min + if new_max > max_diff: + max_diff = new_max + if self.params.thermal_diff_norm: + thermal_norm_limits = (thermal_min_diff, thermal_max_diff) + + if self.params.diff_norm: + filtered_norm_limits = (min_diff, max_diff) for i, region in enumerate(reversed(track.bounds_history)): if region.blank: continue @@ -249,6 +266,7 @@ def preprocess_frames( clip.background, clip.crop_rectangle, filtered_norm_limits=filtered_norm_limits, + thermal_norm_limits=thermal_norm_limits, ) preprocessed = preprocess_single_frame( cropped_frame, @@ -293,30 +311,52 @@ def preprocess_segments( # should really be over whole track buts let just do the indices we predict of # seems to make little different to just doing a min max normalization + thermal_norm_limits = None filtered_norm_limits = None - if self.params.diff_norm: + if self.params.diff_norm or self.params.thermal_diff_norm: min_diff = None max_diff = 0 - for frame_index in frame_indices: - region = track.bounds_history[frame_index - track.start_frame] - f = clip.get_frame(region.frame_number) - if f is None: - logging.warn("Could not get frame {}", region.frame_number) + thermal_max_diff = None + thermal_min_diff = None + for i, region in enumerate(reversed(track.bounds_history)): + if region.blank: continue + if region.width == 0 or region.height == 0: + logging.warn( + "No width or height for frame %s regoin %s", + region.frame_number, + region, + ) + continue + f = clip.get_frame(region.frame_number) if region.blank or region.width <= 0 or region.height <= 0: continue f.float_arrays() - diff_frame = region.subimage(f.thermal) - region.subimage( - clip.background - ) - new_max = np.amax(diff_frame) - new_min = np.amin(diff_frame) - if min_diff is None or new_min < min_diff: - min_diff = new_min - if new_max > max_diff: - max_diff = new_max - filtered_norm_limits = (min_diff, max_diff) + + if self.params.thermal_diff_norm: + diff_frame = f.thermal - np.median(f.thermal) + new_max = np.amax(diff_frame) + new_min = np.amin(diff_frame) + if thermal_min_diff is None or new_min < thermal_min_diff: + thermal_min_diff = new_min + if thermal_max_diff is None or new_max > thermal_max_diff: + thermal_max_diff = new_max + if self.params.diff_norm: + diff_frame = region.subimage(f.thermal) - region.subimage( + clip.background + ) + new_max = np.amax(diff_frame) + new_min = np.amin(diff_frame) + if min_diff is None or new_min < min_diff: + min_diff = new_min + if new_max > max_diff: + max_diff = new_max + if self.params.thermal_diff_norm: + thermal_norm_limits = (thermal_min_diff, thermal_max_diff) + + if self.params.diff_norm: + filtered_norm_limits = (min_diff, max_diff) for frame_index in frame_indices: region = track.bounds_history[frame_index - track.start_frame] @@ -341,6 +381,7 @@ def preprocess_segments( clip.background, clip.crop_rectangle, filtered_norm_limits=filtered_norm_limits, + thermal_norm_limits=thermal_norm_limits, ) track_data[frame.frame_number] = cropped_frame features = None @@ -365,6 +406,7 @@ def preprocess_segments( self.params.frame_size, self.params.channels, self.preprocess_fn, + sample=f"{clip.get_id()}-{track.get_id()}", ) if frames is None: logging.warn("No frames to predict on") diff --git a/src/ml_tools/preprocess.py b/src/ml_tools/preprocess.py index b3186127..89214614 100644 --- a/src/ml_tools/preprocess.py +++ b/src/ml_tools/preprocess.py @@ -61,6 +61,7 @@ def preprocess_frame( crop_rectangle=None, calculate_filtered=True, filtered_norm_limits=None, + thermal_norm_limits=None, ): median = np.median(frame.thermal) cropped_frame = frame.crop_by_region(region, only_thermal=True) @@ -79,7 +80,8 @@ def preprocess_frame( True, ) cropped_frame.thermal -= median - np.clip(cropped_frame.thermal, 0, None, out=cropped_frame.thermal) + if thermal_norm_limits is None: + np.clip(cropped_frame.thermal, 0, None, out=cropped_frame.thermal) if calculate_filtered and filtered_norm_limits is not None: cropped_frame.filtered, stats = imageprocessing.normalize( cropped_frame.filtered, @@ -88,8 +90,13 @@ def preprocess_frame( new_max=255, ) if frame.thermal is not None: + thermal_min = None + thermal_max = None + if thermal_norm_limits is not None: + thermal_min, thermal_max = thermal_norm_limits + logging.info("Using therml min max %s, %s", thermal_min, thermal_max) cropped_frame.thermal, _ = imageprocessing.normalize( - cropped_frame.thermal, new_max=255 + cropped_frame.thermal, min=thermal_min, max=thermal_max, new_max=255 ) else: cropped_frame.normalize() @@ -161,7 +168,7 @@ def preprocess_movement( # index += 1 # tools.saveclassify_image( # data, - # f"samples/{index}", + # f"samples/{sample}-{index}", # ) if preprocess_fn: