Merge pull request #81 from gferraro/bad-tracking

Changed how tracking temp threshold works and added alternative region match scoring

classifier_TEMPLATE.yaml

@@ -1,3 +1,6 @@
+# Output debug information e.g. heat numbers on preview
+debug: False
+
 # root folder where the source and output folders belong
 base_data_folder: "<full path>"
 
@@ -39,6 +42,9 @@ tracking:
 
     # these parameters are associated with the preview background method
     preview:
+        # Temp_thresh is calculated based on the preview min, max and mean, and temp_thresh value
+        dynamic_thresh: True
+
         # When calculating the background ignore the last frames of the preview as the motion detection
         # will still be triggering during this time.
         ignore_frames: 2
@@ -113,6 +119,9 @@ tracking:
         cropped_regions_strategy: "cautious"
 
     filters:
+        # regions with a movement less than this have their tracking scored based of x,y matching, instead of than mid points
+        moving_vel_thresh: 4
+
         # discard any tracks that overlap with other tracks more than this.   T than this length in seconds
         track_overlap_ratio: 0.5
 

ml_tools/config.py

@@ -29,6 +29,7 @@ class Config:
     previews_colour_map = attr.ib()
     use_gpu = attr.ib()
     worker_threads = attr.ib()
+    debug = attr.ib()
 
     @classmethod
     def load_from_file(cls, filename=None):
@@ -59,6 +60,7 @@ def load_from_stream(cls, stream):
             previews_colour_map=raw["previews_colour_map"],
             use_gpu=raw["use_gpu"],
             worker_threads=raw["worker_threads"],
+            debug=raw["debug"],
         )
 
 

ml_tools/previewer.py

@@ -11,17 +11,6 @@
 from track.region import Region
 from .dataset import TrackChannels
 
-LOCAL_RESOURCES = path.join(path.dirname(path.dirname(__file__)), "resources")
-GLOBAL_RESOURCES = "/usr/lib/classifier-pipeline/resources"
-
-
-def resource_path(name):
-    for base in [LOCAL_RESOURCES, GLOBAL_RESOURCES]:
-        p = path.join(base, name)
-        if path.exists(p):
-            return p
-    raise OSError(f"unable to locate {name!r} resource")
-
 
 class Previewer:
 
@@ -44,6 +33,7 @@ class Previewer:
     ]
 
     TRACK_COLOURS = [(255, 0, 0), (0, 255, 0), (255, 255, 0), (128, 255, 255)]
+    FILTERED_COLOURS = [(128, 128, 128)]
 
     def __init__(self, config, preview_type):
         self.config = config
@@ -55,6 +45,7 @@ def __init__(self, config, preview_type):
         self.font_title
         self.preview_type = preview_type
         self.frame_scale = 1
+        self.debug = config.debug
 
     @classmethod
     def create_if_required(self, config, preview_type):
@@ -64,15 +55,15 @@ def create_if_required(self, config, preview_type):
     def _load_colourmap(self):
         colourmap_path = self.config.previews_colour_map
         if not path.exists(colourmap_path):
-            colourmap_path = resource_path("colourmap.dat")
+            colourmap_path = tools.resource_path("colourmap.dat")
         return tools.load_colourmap(colourmap_path)
 
     @property
     def font(self):
         """ gets default font. """
         if not globs._previewer_font:
             globs._previewer_font = ImageFont.truetype(
-                resource_path("Ubuntu-R.ttf"), 12
+                tools.resource_path("Ubuntu-R.ttf"), 12
             )
         return globs._previewer_font
 
@@ -81,7 +72,7 @@ def font_title(self):
         """ gets default title font. """
         if not globs._previewer_font_title:
             globs._previewer_font_title = ImageFont.truetype(
-                resource_path("Ubuntu-B.ttf"), 14
+                tools.resource_path("Ubuntu-B.ttf"), 14
             )
         return globs._previewer_font_title
 
@@ -103,6 +94,8 @@ def export_clip_preview(
             logging.error("Do not have temperatures to use.")
             return
 
+        if self.debug:
+            footer = Previewer.stats_footer(tracker.stats)
         if bool(track_predictions) and self.preview_type == self.PREVIEW_CLASSIFIED:
             self.create_track_descriptions(tracker, track_predictions)
 
@@ -122,9 +115,25 @@ def export_clip_preview(
                 )
                 image = self.convert_and_resize(image, 3.0, mode=Image.NEAREST)
                 draw = ImageDraw.Draw(image)
-                regions = tracker.region_history[frame_number]
-                self.add_regions(draw, regions)
-                self.add_regions(draw, regions, v_offset=120)
+
+                filtered = [track[1] for track in tracker.filtered_tracks]
+                filtered_reasons = [track[0] for track in tracker.filtered_tracks]
+
+                self.add_tracks(
+                    draw,
+                    filtered,
+                    frame_number,
+                    colours=Previewer.FILTERED_COLOURS,
+                    tracks_text=filtered_reasons,
+                )
+                self.add_tracks(
+                    draw,
+                    filtered,
+                    frame_number,
+                    v_offset=120,
+                    colours=Previewer.FILTERED_COLOURS,
+                    tracks_text=filtered_reasons,
+                )
                 self.add_tracks(draw, tracker.tracks, frame_number)
 
             if self.preview_type == self.PREVIEW_BOXES:
@@ -143,6 +152,8 @@ def export_clip_preview(
                     draw, tracker.tracks, frame_number, track_predictions, screen_bounds
                 )
 
+            if self.debug:
+                self.add_footer(draw, image.width, image.height, footer)
             mpeg.next_frame(np.asarray(image))
 
             # we store the entire video in memory so we need to cap the frame count at some point.
@@ -170,10 +181,11 @@ def create_individual_track_previews(self, filename, tracker: TrackExtractor):
             logging.info("creating preview %s", filename_format.format(id + 1))
             tools.write_mpeg(filename_format.format(id + 1), video_frames)
 
-    def convert_and_resize(self, image, size=None, mode=Image.BILINEAR):
+    def convert_and_resize(self, frame, size=None, mode=Image.BILINEAR):
         """ Converts the image to colour using colour map and resize """
+        thermal = frame[:120, :160].copy()
         image = tools.convert_heat_to_img(
-            image, self.colourmap, self.auto_min, self.auto_max
+            frame, self.colourmap, self.auto_min, self.auto_max
         )
         if size:
             self.frame_scale = size
@@ -184,6 +196,9 @@ def convert_and_resize(self, image, size=None, mode=Image.BILINEAR):
                 ),
                 mode,
             )
+
+        if self.debug:
+            tools.add_heat_number(image, thermal, self.frame_scale)
         return image
 
     def create_track_descriptions(self, tracker, track_predictions):
@@ -232,27 +247,77 @@ def add_tracks(
         track_predictions=None,
         screen_bounds=None,
         colours=TRACK_COLOURS,
+        tracks_text=None,
+        v_offset=0,
     ):
+
         # look for any tracks that occur on this frame
         for index, track in enumerate(tracks):
             frame_offset = frame_number - track.start_frame
             if frame_offset >= 0 and frame_offset < len(track.bounds_history) - 1:
-                rect = track.bounds_history[frame_offset]
+                region = track.bounds_history[frame_offset]
                 draw.rectangle(
-                    self.rect_points(rect), outline=colours[index % len(colours)]
+                    self.rect_points(region, v_offset),
+                    outline=colours[index % len(colours)],
                 )
                 if track_predictions:
                     self.add_class_results(
                         draw,
                         track,
                         frame_offset,
-                        rect,
+                        region,
                         track_predictions,
                         screen_bounds,
+                        v_offset=v_offset,
+                    )
+                if self.debug:
+                    text = None
+                    if tracks_text and len(tracks_text) > index:
+                        text = tracks_text[index]
+                    self.add_debug_text(
+                        draw,
+                        track,
+                        frame_offset,
+                        region,
+                        screen_bounds,
+                        text=text,
+                        v_offset=v_offset,
                     )
 
+    def add_footer(self, draw, width, height, text):
+        footer_size = self.font.getsize(text)
+        center = (width / 2 - footer_size[0] / 2.0, height - footer_size[1])
+        draw.text((center[0], center[1]), text, font=self.font)
+
+    def add_debug_text(
+        self, draw, track, frame_offset, region, screen_bounds, text=None, v_offset=0
+    ):
+        if text is None:
+            text = (
+                f"id {track.id} mass {region.mass} var {round(region.pixel_variance,2)}"
+            )
+        footer_size = self.font.getsize(text)
+        footer_center = ((region.width * self.frame_scale) - footer_size[0]) / 2
+
+        footer_rect = Region(
+            region.left * self.frame_scale + footer_center,
+            (v_offset + region.bottom) * self.frame_scale,
+            footer_size[0],
+            footer_size[1],
+        )
+        self.fit_to_image(footer_rect, screen_bounds)
+
+        draw.text((footer_rect.x, footer_rect.y), text, font=self.font)
+
     def add_class_results(
-        self, draw, track, frame_offset, rect, track_predictions, screen_bounds
+        self,
+        draw,
+        track,
+        frame_offset,
+        rect,
+        track_predictions,
+        screen_bounds,
+        v_offset=0,
     ):
         prediction = track_predictions[track]
         if track not in track_predictions:
@@ -270,14 +335,14 @@ def add_class_results(
         # figure out where to draw everything
         header_rect = Region(
             rect.left * self.frame_scale,
-            rect.top * self.frame_scale - header_size[1],
+            (v_offset + rect.top) * self.frame_scale - header_size[1],
             header_size[0],
             header_size[1],
         )
         footer_center = ((rect.width * self.frame_scale) - footer_size[0]) / 2
         footer_rect = Region(
             rect.left * self.frame_scale + footer_center,
-            rect.bottom * self.frame_scale,
+            (v_offset + rect.bottom) * self.frame_scale,
             footer_size[0],
             footer_size[1],
         )
@@ -296,6 +361,8 @@ def add_class_results(
 
     def fit_to_image(self, rect: Region, screen_bounds: Region):
         """ Modifies rect so that rect is visible within bounds. """
+        if screen_bounds is None:
+            return
         if rect.left < screen_bounds.left:
             rect.x = screen_bounds.left
         if rect.top < screen_bounds.top:
@@ -316,3 +383,14 @@ def rect_points(self, rect, v_offset=0, h_offset=0):
             s * (rect.bottom + v_offset) - 1,
         ]
         return
+
+    @staticmethod
+    def stats_footer(stats):
+        return "max {}, min{}, mean{}, back delta {}, avg delta{}, temp_thresh {}".format(
+            round(stats["max_temp"], 2),
+            round(stats["min_temp"], 2),
+            round(stats["mean_temp"], 2),
+            round(stats["average_background_delta"], 2),
+            round(stats["average_delta"], 2),
+            stats["temp_thresh"],
+        )

ml_tools/tools.py

@@ -10,25 +10,22 @@
 import math
 import matplotlib.pyplot as plt
 import logging
-import io
-import itertools
-import gzip
 from sklearn import metrics
 import json
 import dateutil
 import binascii
-import time
 import datetime
 import glob
 import cv2
 from matplotlib.colors import LinearSegmentedColormap
 import subprocess
+from PIL import ImageFont
+from PIL import ImageDraw
 
 EPISON = 1e-5
 
-# the coldest value to display when rendering previews
-TEMPERATURE_MIN = 2800
-TEMPERATURE_MAX = 4200
+LOCAL_RESOURCES = os.path.join(os.path.dirname(os.path.dirname(__file__)), "resources")
+GLOBAL_RESOURCES = "/usr/lib/classifier-pipeline/resources"
 
 
 class Rectangle:
@@ -692,3 +689,42 @@ def get_confusion_matrix(pred_class, true_class, classes, normalize=True):
     "blue": ((0.0, 0.3, 0.3), (0.5, 0.0, 0.0), (1.0, 0.1, 0.1)),
 }
 cm_blue_red = LinearSegmentedColormap("BlueRed2", color_dict)
+
+
+def frame_to_img(frame, filename, colourmap_file, min, max):
+    colourmap = _load_colourmap(colourmap_file)
+    img = convert_heat_to_img(frame, colourmap, min, max)
+    img.save(filename + ".jpg", "JPEG")
+
+
+def _load_colourmap(colourmap_path):
+    if not os.path.exists(colourmap_path):
+        colourmap_path = resource_path("colourmap.dat")
+    return load_colourmap(colourmap_path)
+
+
+def resource_path(name):
+
+    for base in [LOCAL_RESOURCES, GLOBAL_RESOURCES]:
+        p = os.path.join(base, name)
+        if os.path.exists(p):
+            return p
+    raise OSError(f"unable to locate {name!r} resource")
+
+
+def add_heat_number(img, frame, scale):
+    draw = ImageDraw.Draw(img)
+    font = ImageFont.truetype("FreeSans.ttf", 8)
+
+    for y, row in enumerate(frame):
+        if y % 4 == 0:
+            min_v = np.amin(row)
+            min_i = np.where(row == min_v)[0][0]
+            max_v = np.amax(row)
+            max_i = np.where(row == max_v)[0][0]
+            draw.text((min_i * scale, y * scale), str(int(min_v)), (0, 0, 0), font=font)
+            draw.text((max_i * scale, y * scale), str(int(max_v)), (0, 0, 0), font=font)
+
+
+def eucl_distance(first, second):
+    return ((first[0] - second[0]) ** 2 + (first[1] - second[1]) ** 2) ** 0.5